diff --git a/.gitignore b/.gitignore index cab0667..8845d6b 100644 --- a/.gitignore +++ b/.gitignore @@ -31,8 +31,12 @@ share/python-wheels/ MANIFEST # PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. +# Usually these files are written by a python +script from a template + +# before PyInstaller builds the exe, so as to +inject date/other infos into it. + *.manifest *.spec @@ -88,13 +92,22 @@ ipython_config.py .python-version # pipenv -# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. -# However, in case of collaboration, if having platform-specific dependencies or dependencies -# having no cross-platform support, pipenv may install dependencies that don't work, or not +# According to pypa/pipenv#598, it is recomme +nded to include Pipfile.lock in version control +. + +# However, in case of collaboration, if havin +g platform-specific dependencies or dependencie +s + +# having no cross-platform support, pipenv ma +y install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock -# PEP 582; used by e.g. github.com/David-OConnor/pyflow +# PEP 582; used by e.g. github.com/David-OConno +r/pyflow + __pypackages__/ # Celery stuff @@ -150,10 +163,15 @@ metadata/ # Chroot environments /var/lib/deb-mock/ /tmp/deb-mock-* +chroot/ +chroots/ +mock-chroot-* # Build logs *.log logs/ +build.log +mock.log # IDE files .vscode/ @@ -164,5 +182,116 @@ logs/ # OS files .DS_Store -Thumbs.db +Thumbs.db +._* +.Spotlight-V100 +.Trashes +ehthumbs.db + +# Temporary files +*.tmp +*.temp +*.bak +*.backup +*.old +*.orig + +# Configuration overrides +config.local.yaml +config.local.yml +.env.local +.env.*.local + +# Test artifacts +.pytest_cache/ +test-results/ +test-output/ +coverage/ +htmlcov/ + +# Mock-specific build artifacts +mock-build-* +mock-result-* +mock-*.log +mock-*.txt + +# Package build artifacts +*.build +*.buildinfo +*.changes +*.dsc +*.deb +*.udeb +*.tar.gz +*.tar.xz +*.tar.bz2 +*.diff.gz +*.orig.tar.gz +*.debian.tar.gz + +# Chroot and build environment files +/var/lib/mock/ +/var/cache/mock/ +mock-* +mockroot/ + +# Development tools +.coverage +.pytest_cache/ +.tox/ +.nox/ +.mypy_cache/ +.pyre/ + +# Documentation builds +docs/_build/ +site/ +docs/build/ + +# Cache directories +.cache/ +cache/ +__pycache__/ + +# Backup and temporary files +*~ +*.swp +*.swo +*.bak +*.backup +*.old +*.orig +*.tmp +*.temp + +# Local development files +local/ +dev/ +development/ +local_config.py +local_settings.py + +# Database files +*.db +*.sqlite +*.sqlite3 + +# Log files +*.log +logs/ +log/ +*.log.* + +# Archive files +*.zip +*.rar +*.7z +*.tar +*.gz +*.bz2 +*.xz + +# System files +.fuse_hidden* +.nfs* diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..623b625 --- /dev/null +++ b/LICENSE @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/Makefile b/Makefile index 4f2db7a..20721c7 100644 --- a/Makefile +++ b/Makefile @@ -1,95 +1,91 @@ -.PHONY: help install install-dev test clean lint format docs +# deb-mock Makefile +# Debian's equivalent to Fedora's Mock build environment manager -help: ## Show this help message - @echo "Deb-Mock - Debian Package Build Environment" +.PHONY: all install clean test lint format dev-setup help + +# Default target +all: install + +# Install the package +install: + @echo "Installing deb-mock..." + @pip install -e . + +# Install development dependencies +dev-setup: install + @echo "Installing development dependencies..." + @pip install -e ".[dev]" + +# Run tests +test: + @echo "Running tests..." + @python -m pytest tests/ -v + +# Run tests with coverage +test-cov: + @echo "Running tests with coverage..." + @python -m pytest tests/ --cov=deb_mock --cov-report=html + +# Lint the code +lint: + @echo "Linting code..." + @flake8 deb_mock/ tests/ + @mypy deb_mock/ + +# Format the code +format: + @echo "Formatting code..." + @black deb_mock/ tests/ + +# Clean build artifacts +clean: + @echo "Cleaning build artifacts..." + @rm -rf build/ + @rm -rf dist/ + @rm -rf *.egg-info/ + @rm -rf .pytest_cache/ + @rm -rf htmlcov/ + @find . -type f -name "*.pyc" -delete + @find . -type d -name "__pycache__" -delete + +# Development helpers +dev-install: dev-setup + @echo "Development environment ready!" + +dev-test: dev-setup test + +dev-lint: dev-setup lint + +dev-format: dev-setup format + +# Run the CLI +run: + @echo "Running deb-mock CLI..." + @python -m deb_mock.cli --help + +# Create virtual environment (optional) +venv: + @echo "Creating virtual environment..." + @python3 -m venv venv + @echo "Virtual environment created. Activate with: source venv/bin/activate" + +# Help +help: + @echo "deb-mock Makefile" @echo "" - @echo "Available targets:" - @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf " \033[36m%-15s\033[0m %s\n", $$1, $$2}' - -install: ## Install deb-mock (for Debian package build) - @echo "Installation handled by dh-python" - -install-dev: ## Install deb-mock with development dependencies - pip install -e . - pip install -r requirements-dev.txt - -test: ## Run tests - python3 -m pytest tests/ -v - -test-coverage: ## Run tests with coverage - python3 -m pytest tests/ --cov=deb_mock --cov-report=html --cov-report=term - -lint: ## Run linting checks - @echo "=== Running all linting checks with Docker container ===" - docker run --rm -v $(PWD):/app git.raines.xyz/robojerk/deb-mock-linter:latest /bin/bash -c "\ - echo '=== Linting YAML files ===' && \ - yamllint .forgejo/workflows/ deb_mock/configs/ test-config.yaml && \ - echo '=== Linting Python files ===' && \ - source /opt/venv/bin/activate && \ - flake8 deb_mock/ tests/ --max-line-length=120 --ignore=E203,W503 && \ - black --check --line-length=120 deb_mock/ tests/ && \ - isort --check-only --profile=black deb_mock/ tests/ && \ - echo '=== Linting shell scripts ===' && \ - find . -name '*.sh' -exec shellcheck {} \; && \ - echo '✅ All linting checks passed!'" - -lint-local: ## Run linting checks locally (requires tools installed) - @echo "=== Running Python linting ===" - venv/bin/flake8 deb_mock/ tests/ --max-line-length=120 --ignore=E203,W503 - venv/bin/black --check --line-length=120 deb_mock/ tests/ - venv/bin/isort --check-only --profile=black deb_mock/ tests/ - @echo "=== Running YAML linting ===" - yamllint .forgejo/workflows/ deb_mock/configs/ test-config.yaml - @echo "=== Running shell linting ===" - find . -name "*.sh" -exec shellcheck {} \; - @echo "✅ All linting checks passed!" - -format: ## Format code with black - black deb_mock/ tests/ - -clean: ## Clean build artifacts - rm -rf build/ - rm -rf dist/ - rm -rf *.egg-info/ - # rm -rf output/ # Temporarily disabled to preserve build artifacts - rm -rf metadata/ - find . -type d -name __pycache__ -exec rm -rf {} + - find . -type f -name "*.pyc" -delete - -docs: ## Build documentation - cd docs && make html - -install-system-deps: ## Install system dependencies (requires sudo) - sudo apt update - sudo apt install -y sbuild schroot debhelper build-essential debootstrap yamllint shellcheck - -install-lint-deps: ## Install linting dependencies - python3 -m venv venv - venv/bin/pip install flake8 black isort bandit - sudo apt install -y yamllint shellcheck nodejs npm - sudo npm install -g markdownlint-cli - -setup-chroot: ## Setup initial chroot environment (requires sudo) - sudo mkdir -p /var/lib/deb-mock/chroots - sudo mkdir -p /etc/schroot/chroot.d - sudo chown -R $$USER:$$USER /var/lib/deb-mock - -build-example: ## Build an example package (requires setup) - deb-mock init-chroot bookworm-amd64 - deb-mock build examples/hello_1.0.dsc - -check: ## Run all checks (lint, test, format) - $(MAKE) lint - $(MAKE) test - $(MAKE) format - -dist: ## Build distribution package - python3 setup.py sdist bdist_wheel - -upload: ## Upload to PyPI (requires twine) - twine upload dist/* - -dev-setup: ## Complete development setup - $(MAKE) install-system-deps - $(MAKE) setup-chroot - $(MAKE) install-dev \ No newline at end of file + @echo "Targets:" + @echo " all - Install the package" + @echo " install - Install the package" + @echo " dev-setup - Install with development dependencies" + @echo " test - Run tests" + @echo " test-cov - Run tests with coverage" + @echo " lint - Lint the code" + @echo " format - Format the code" + @echo " clean - Clean build artifacts" + @echo " dev-install - Set up development environment" + @echo " dev-test - Run tests in development environment" + @echo " dev-lint - Lint code in development environment" + @echo " dev-format - Format code in development environment" + @echo " run - Run the CLI" + @echo " venv - Create virtual environment" + @echo " help - Show this help" \ No newline at end of file diff --git a/README.md b/README.md deleted file mode 100644 index ea72165..0000000 --- a/README.md +++ /dev/null @@ -1,355 +0,0 @@ -# Deb-Mock - -![Build Status](https://git.raines.xyz/robojerk/deb-mock/actions/workflows/build.yml/badge.svg) - -A low-level utility to create clean, isolated build environments for single Debian packages. This tool is a direct functional replacement for Fedora's Mock, adapted specifically for Debian-based ecosystems. - -**Last updated: 2025-01-22 12:00:00 UTC** - -## Purpose - -Deb-Mock provides: -- **sbuild Integration**: A wrapper around the native Debian sbuild tool to standardize its command-line arguments and behavior -- **Chroot Management**: Handles the creation, maintenance, and cleanup of the base chroot images used for building -- **Build Metadata Capture**: Captures and standardizes all build output, including logs, .deb files, and .changes files -- **Reproducible Build Enforcement**: Ensures that all build dependencies are satisfied within the isolated environment - -## Features - -- ✅ Isolated build environments using chroot -- ✅ Integration with Debian's native sbuild tool -- ✅ Standardized build metadata capture -- ✅ Reproducible build verification -- ✅ Clean environment management and cleanup -- ✅ **Chain building** for dependent packages (like Mock's `--chain`) -- ✅ **Shell access** to chroot environments (like Mock's `--shell`) -- ✅ **File operations** between host and chroot (like Mock's `--copyin`/`--copyout`) -- ✅ **Chroot scrubbing** for cleanup without removal (like Mock's `--scrub`) -- ✅ **Core configurations** for popular distributions (like Mock's `mock-core-configs`) - -## CI/CD Status - -This project uses Forgejo Actions for continuous integration and deployment: - -- **Build**: Automatically builds and tests the package on every push -- **Test**: Comprehensive testing of all CLI commands and functionality -- **Release**: Automated releases when tags are pushed -- **Documentation**: Auto-updates README with build status - -### Build Status -![Build Status](https://git.raines.xyz/robojerk/deb-mock/actions/workflows/build.yml/badge.svg) -![Test Status](https://git.raines.xyz/robojerk/deb-mock/actions/workflows/test.yml/badge.svg) -![Package Build Status](https://git.raines.xyz/robojerk/deb-mock/actions/workflows/build-deb.yml/badge.svg) - -## Installation - -### From Forgejo Package Registry (Recommended) - -```bash -# Add the Deb-Mock repository from Forgejo -wget -O - https://git.raines.xyz/api/packages/robojerk/debian/gpg.key | sudo apt-key add - -echo 'deb [signed-by=/usr/share/keyrings/forgejo-robojerk.gpg] https://git.raines.xyz/api/packages/robojerk/debian unstable main' | sudo tee /etc/apt/sources.list.d/deb-mock.list -sudo apt update - -# Install mock -sudo apt install -y mock -``` - -### From Debian Repository (Alternative) - -```bash -# Add the Mock repository -wget -O - http://debian.raines.xyz/mock.gpg.key | sudo apt-key add - -echo 'deb http://debian.raines.xyz unstable main' | sudo tee /etc/apt/sources.list.d/mock.list -sudo apt update - -# Install mock -sudo apt install -y mock -``` - -### From Source - -```bash -# Clone the repository -git clone https://git.raines.xyz/robojerk/deb-mock.git -cd deb-mock - -# Install dependencies -sudo apt install sbuild schroot debhelper build-essential debootstrap python3-venv python3-pip - -# Create virtual environment and install -python3 -m venv venv -source venv/bin/activate -pip install -e . -``` - -### Building Debian Package - -```bash -# Install build dependencies -sudo apt install -y build-essential devscripts debhelper dh-python python3-all python3-setuptools - -# Build the package -dpkg-buildpackage -us -uc -b - -# Install the built package -sudo dpkg -i ../deb-mock_*.deb -``` - -## Usage - -### Basic Package Build (Similar to Mock) - -```bash -# Build a source package (like: mock -r fedora-35-x86_64 package.src.rpm) -mock build package.dsc - -# Build with specific chroot config (like: mock -r debian-bookworm-amd64 package.src.rpm) -mock -r debian-bookworm-amd64 build package.dsc - -# Build with specific chroot -mock build --chroot=bookworm-amd64 package.dsc - -# Build with specific architecture -mock build --arch=amd64 package.dsc -``` - -### Advanced Build Options (Mock's advanced CLI options) - -```bash -# Skip running tests (like: mock --nocheck) -mock build --no-check package.dsc - -# Build in offline mode (like: mock --offline) -mock build --offline package.dsc - -# Set build timeout (like: mock --rpmbuild_timeout) -mock build --build-timeout 3600 package.dsc - -# Force architecture (like: mock --forcearch) -mock build --force-arch amd64 package.dsc - -# Unique extension for buildroot (like: mock --uniqueext) -mock build --unique-ext mybuild package.dsc - -# Clean chroot after build (like: mock --cleanup-after) -mock build --cleanup-after package.dsc - -# Don't clean chroot after build (like: mock --no-cleanup-after) -deb-mock build --no-cleanup-after package.dsc -``` - -### Core Configurations (Mock's `mock-core-configs` equivalent) - -```bash -# List available core configurations -deb-mock list-configs - -# Use core configurations (similar to Mock's -r option) -deb-mock -r debian-bookworm-amd64 build package.dsc -deb-mock -r debian-sid-amd64 build package.dsc -deb-mock -r ubuntu-jammy-amd64 build package.dsc -deb-mock -r ubuntu-noble-amd64 build package.dsc -``` - -### Chain Building (Mock's `--chain` equivalent) - -```bash -# Build multiple packages that depend on each other -deb-mock chain package1.dsc package2.dsc package3.dsc - -# Continue building even if one package fails -deb-mock chain --continue-on-failure package1.dsc package2.dsc package3.dsc - -# Use core config with chain building -deb-mock -r debian-bookworm-amd64 chain package1.dsc package2.dsc -``` - -### Package Management (Mock's package management commands) - -```bash -# Install build dependencies (like: mock --installdeps package.src.rpm) -deb-mock install-deps package.dsc - -# Install packages in chroot (like: mock --install package) -deb-mock install package1 package2 package3 - -# Update packages in chroot (like: mock --update) -deb-mock update -deb-mock update package1 package2 - -# Remove packages from chroot (like: mock --remove package) -deb-mock remove package1 package2 - -# Execute APT commands (like: mock --pm-cmd "command") -deb-mock apt-cmd "update" -deb-mock apt-cmd "install package" -``` - -### Chroot Management (Similar to Mock) - -```bash -# Initialize a new chroot (like: mock -r fedora-35-x86_64 --init) -deb-mock init-chroot bookworm-amd64 - -# List available chroots (like: mock --list-chroots) -deb-mock list-chroots - -# Clean up a chroot (like: mock -r fedora-35-x86_64 --clean) -deb-mock clean-chroot bookworm-amd64 - -# Scrub a chroot without removing it (like: mock -r fedora-35-x86_64 --scrub) -deb-mock scrub-chroot bookworm-amd64 - -# Scrub all chroots (like: mock --scrub-all-chroots) -deb-mock scrub-all-chroots -``` - -### Debugging and Configuration (Mock's debugging commands) - -```bash -# Show current configuration (like: mock --debug-config) -deb-mock config - -# Show detailed configuration (like: mock --debug-config-expanded) -deb-mock debug-config -deb-mock debug-config --expand - -# Show cache statistics -deb-mock cache-stats - -# Clean up old cache files -deb-mock cleanup-caches -``` - -### Shell Access (Mock's `--shell` equivalent) - -```bash -# Open a shell in the chroot environment -deb-mock shell - -# Open a shell in a specific chroot -deb-mock shell --chroot=sid-amd64 - -# Use core config for shell access -deb-mock -r debian-sid-amd64 shell -``` - -### File Operations (Mock's `--copyin`/`--copyout` equivalents) - -```bash -# Copy files from host to chroot (like: mock --copyin file.txt /tmp/) -deb-mock copyin file.txt /tmp/ - -# Copy files from chroot to host (like: mock --copyout /tmp/file.txt .) -deb-mock copyout /tmp/file.txt . - -# Use core config with file operations -deb-mock -r debian-bookworm-amd64 copyin file.txt /tmp/ -``` - -### Advanced Usage - -```bash -# Build with custom configuration -deb-mock build --config=custom.conf package.dsc - -# Build with verbose output -deb-mock build --verbose package.dsc - -# Build with debug output -deb-mock build --debug package.dsc - -# Keep chroot after build (for debugging) -deb-mock build --keep-chroot package.dsc -``` - -## Core Configurations - -Deb-Mock includes pre-configured build environments for popular Debian-based distributions, similar to Mock's `mock-core-configs` package: - -### **Debian Family** -- `debian-bookworm-amd64` - Debian 12 (Bookworm) - AMD64 -- `debian-sid-amd64` - Debian Unstable (Sid) - AMD64 - -### **Ubuntu Family** -- `ubuntu-jammy-amd64` - Ubuntu 22.04 LTS (Jammy) - AMD64 -- `ubuntu-noble-amd64` - Ubuntu 24.04 LTS (Noble) - AMD64 - -### **Usage Examples** -```bash -# Build for Debian Bookworm -deb-mock -r debian-bookworm-amd64 build package.dsc - -# Build for Ubuntu Jammy -deb-mock -r ubuntu-jammy-amd64 build package.dsc - -# Build for Debian Sid (unstable) -deb-mock -r debian-sid-amd64 build package.dsc -``` - -## Configuration - -Deb-Mock uses YAML configuration files to define build environments. See `docs/configuration.md` for detailed configuration options. - -### Example Configuration (Similar to Mock configs) - -```yaml -# Basic configuration -chroot_name: bookworm-amd64 -architecture: amd64 -suite: bookworm -output_dir: ./output -keep_chroot: false -verbose: false -debug: false - -# Build environment -build_env: - DEB_BUILD_OPTIONS: parallel=4,nocheck - DEB_BUILD_PROFILES: nocheck - -# Build options -build_options: - - --verbose - - --no-run-lintian -``` - -## Comparison with Fedora Mock - -| Mock Feature | Deb-Mock Equivalent | Status | -|--------------|-------------------|--------| -| `mock -r config package.src.rpm` | `deb-mock -r config package.dsc` | ✅ | -| `mock --chain` | `deb-mock chain package1.dsc package2.dsc` | ✅ | -| `mock --shell` | `deb-mock shell` | ✅ | -| `mock --copyin` | `deb-mock copyin` | ✅ | -| `mock --copyout` | `deb-mock copyout` | ✅ | -| `mock --scrub` | `deb-mock scrub-chroot` | ✅ | -| `mock --init` | `deb-mock init-chroot` | ✅ | -| `mock --clean` | `deb-mock clean-chroot` | ✅ | -| `mock --list-chroots` | `deb-mock list-chroots` | ✅ | -| `mock --installdeps` | `deb-mock install-deps` | ✅ | -| `mock --install` | `deb-mock install` | ✅ | -| `mock --update` | `deb-mock update` | ✅ | -| `mock --remove` | `deb-mock remove` | ✅ | -| `mock --pm-cmd` | `deb-mock apt-cmd` | ✅ | -| `mock --nocheck` | `deb-mock --no-check` | ✅ | -| `mock --offline` | `deb-mock --offline` | ✅ | -| `mock --forcearch` | `deb-mock --force-arch` | ✅ | -| `mock --debug-config` | `deb-mock debug-config` | ✅ | -| `mock-core-configs` | `deb-mock list-configs` | ✅ | - -## Development - -This project is part of the three-tool system for Debian build and assembly: -- **Deb-Mock** (this project): Low-level build environment utility -- **Deb-Orchestrator**: Central build management system -- **Tumbi-Assembler**: Distribution composition tool - -## License - -[License information to be added] - -## Contributing - -[Contribution guidelines to be added] \ No newline at end of file diff --git a/README.md b/README.md new file mode 120000 index 0000000..f1348c5 --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +mock/README.md \ No newline at end of file diff --git a/behave/.vcs-diff-lint.yml b/behave/.vcs-diff-lint.yml new file mode 100644 index 0000000..e69de29 diff --git a/behave/README.md b/behave/README.md new file mode 100644 index 0000000..f078414 --- /dev/null +++ b/behave/README.md @@ -0,0 +1,14 @@ +BDD for Mock +============ + +This test-suite can destroy your system! Not intentionally, but some steps +require us to use root (e.g. install or remove packages). **Never** execute +this test suite on your host system, allocate some disposable machine. + +How to run the tests +-------------------- + +1. Install the Mock RPM that you want to test. + +2. Run `$ behave` command in this directory, with `--tags tagname` if you want + to test only subset of all provided scenarios. diff --git a/behave/environment.py b/behave/environment.py new file mode 100644 index 0000000..0884517 --- /dev/null +++ b/behave/environment.py @@ -0,0 +1,79 @@ +""" +Global configuration for Mock's behave tests +""" + +import os +import pwd +import random +import shutil +import string +import tempfile + +import requests + +from testlib.mock import Mock +from testlib.commands import no_output + + +def _random_string(length): + return ''.join(random.choices(string.ascii_lowercase + string.digits, + k=length)) + + +def _download(context, url): + print(f'Downloading {url}') + req = requests.get(url, timeout=60) + filename = os.path.join(context.workdir, os.path.basename(url)) + with open(filename, 'wb') as dfd: + dfd.write(req.content) + return filename + + +def _download_rpm(context, rpm): + files = { + "always-installable": + "repo/always-installable-1-0.fc32.noarch.rpm", + "buildrequires-always-installable": + "buildrequires-always-installable-1-0.src.rpm", + } + return _download(context, "/".join([context.test_storage, files[rpm]])) + + +def before_all(context): + """ executed before all the testing starts, only once per behave run """ + context.uniqueext = _random_string(8) + context.uniqueext_used = False + + # detect the default used chroot from default.cfg link + default_config = os.readlink("/etc/mock/default.cfg") + context.chroot = default_config[:-4] # drop cfg suffix + + context.test_storage = ( + "https://github.com/" + "rpm-software-management/mock-test-data/raw/main/") + + context.download = lambda url: _download(context, url) + context.download_rpm = lambda rpm: _download_rpm(context, rpm) + context.next_mock_options = [] + + +def _cleanup_workdir(context): + shutil.rmtree(context.workdir) + context.workdir = None + context.custom_config = "" + + +def before_scenario(context, _scenario): + """ execute before - once for each - scenario """ + context.workdir = tempfile.mkdtemp(prefix="mock-behave-tests-") + context.custom_config = "" + context.add_cleanup(_cleanup_workdir, context) + context.mock = Mock(context) + context.add_repos = [] + context.current_user = pwd.getpwuid(os.getuid())[0] + + +def after_scenario(context, _scenario): + """ execute after - and for each - scenario """ + with no_output(): + context.mock.clean() diff --git a/behave/features/__init__.py b/behave/features/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/behave/features/add-repo.feature b/behave/features/add-repo.feature new file mode 100644 index 0000000..218354b --- /dev/null +++ b/behave/features/add-repo.feature @@ -0,0 +1,17 @@ +Feature: The --addrepo commandline option. + + Background: + Given an unique mock namespace + And pre-intitialized chroot + + Scenario: Test that --addrepo works + Given a custom third-party repository is used for builds + When a build is depending on third-party repo requested + Then the build succeeds + + Scenario: Test that --addrepo LOCAL_DIR works + Given a created local repository + And the local repo contains a "always-installable" RPM + And the local repo is used for builds + When a build which requires the "always-installable" RPM is requested + Then the build succeeds diff --git a/behave/features/auto-download-of-srpm.feature b/behave/features/auto-download-of-srpm.feature new file mode 100644 index 0000000..886f03a --- /dev/null +++ b/behave/features/auto-download-of-srpm.feature @@ -0,0 +1,8 @@ +Feature: Check that we download source RPMs URLs + + @autodownload + Scenario: Mock downloads SRPMs in --rebuild mode + Given an unique mock namespace + And pre-intitialized chroot + When an online source RPM is rebuilt + Then the build succeeds diff --git a/behave/features/buildroot-image.feature b/behave/features/buildroot-image.feature new file mode 100644 index 0000000..8c4edf1 --- /dev/null +++ b/behave/features/buildroot-image.feature @@ -0,0 +1,19 @@ +Feature: Mock 6.0+ supports --bootstrap-image feature and OCI buildroot exports + + @buildroot_image + Scenario: Use image from registry for buildroot preparation + Given an unique mock namespace + Given mock is always executed with "--buildroot-image registry.fedoraproject.org/fedora:rawhide" + When an online source RPM is rebuilt against fedora-rawhide-x86_64 + Then the build succeeds + + @buildroot_image + Scenario: Image from 'export_buildroot_image' works with --buildroot-image + Given an unique mock namespace + Given next mock call uses --enable-plugin=export_buildroot_image option + # No need to do a full build here! + When deps for python-copr-999-1.src.rpm are calculated against fedora-rawhide-x86_64 + And OCI tarball from fedora-rawhide-x86_64 backed up and will be used + And the fedora-rawhide-x86_64 chroot is scrubbed + And an online SRPM python-copr-999-1.src.rpm is rebuilt against fedora-rawhide-x86_64 + Then the build succeeds diff --git a/behave/features/chroot-scan.feature b/behave/features/chroot-scan.feature new file mode 100644 index 0000000..d78990c --- /dev/null +++ b/behave/features/chroot-scan.feature @@ -0,0 +1,19 @@ +Feature: The chroot_scan plugin + +@chroot_scan +Scenario: Check that chroot_scan works and file permissions are correct + Given chroot_scan is enabled for dnf5.log + And an unique mock namespace + When an online source RPM is rebuilt + Then the build succeeds + And dnf5.log file is in chroot_scan result dir + And ownership of all chroot_scan files is correct + +@chroot_scan +Scenario: Check that chroot_scan tarball is created correctly + Given an unique mock namespace + And chroot_scan is enabled for dnf5.log + And chroot_scan is configured to produce tarball + When an online source RPM is rebuilt + Then the build succeeds + And chroot_scan tarball has correct perms and provides dnf5.log diff --git a/behave/features/cmdline-errors.feature b/behave/features/cmdline-errors.feature new file mode 100644 index 0000000..dbf9ca1 --- /dev/null +++ b/behave/features/cmdline-errors.feature @@ -0,0 +1,7 @@ +Feature: Test error reporting from argument parser + + @errors + Scenario: The --resultdir option is incompatible with --chain + When mock is run with "--resultdir /tmp/dir --chain" options + Then the exit code is 5 + And the one-liner error contains "ERROR: The --chain mode doesn't support --resultdir" diff --git a/behave/features/dnf4.feature b/behave/features/dnf4.feature new file mode 100644 index 0000000..7a17827 --- /dev/null +++ b/behave/features/dnf4.feature @@ -0,0 +1,24 @@ +Feature: Mock is able to work with dnf4 chroots + + @dnf4 @no-bootstrap + Scenario: Building a DNF4 chroot without bootstrap chroot + Given an unique mock namespace + And mock is always executed with "--no-bootstrap-chroot --config-opts=dnf_warning=False" + When an online source RPM is rebuilt against centos-stream+epel-9-x86_64 + Then the build succeeds + + @dnf4 @no-bootstrap-image + Scenario: Building in DNF4 chroot with dnf4 on host, without bootstrap image + Given an unique mock namespace + And the python3-dnf package is installed on host + And mock is always executed with "--no-bootstrap-image" + When an online source RPM is rebuilt against centos-stream+epel-9-x86_64 + Then the build succeeds + + @dnf4 @no-bootstrap-image @with-dnf4 + Scenario: Building a DNF4 chroot without dnf4 on host, without bootstrap image + Given an unique mock namespace + And the python3-dnf package not installed on host + And mock is always executed with "--no-bootstrap-image" + When an online source RPM is rebuilt against centos-stream+epel-9-x86_64 + Then the build succeeds diff --git a/behave/features/dnf5.feature b/behave/features/dnf5.feature new file mode 100644 index 0000000..931a6ee --- /dev/null +++ b/behave/features/dnf5.feature @@ -0,0 +1,15 @@ +Feature: Mock correctly works with DNF5 + + @dnf5 @no-bootstrap + Scenario: Building in Rawhide with DNF5, without bootstrap chroot + Given mock is always executed with "--no-bootstrap-chroot" + And an unique mock namespace + When an online source RPM is rebuilt + Then the build succeeds + + @dnf5 @no-bootstrap-image + Scenario: Building in Rawhide with DNF5 with DNF5 on host + Given mock is always executed with "--no-bootstrap-image" + And an unique mock namespace + When an online source RPM is rebuilt + Then the build succeeds diff --git a/behave/features/hermetic-build.feature b/behave/features/hermetic-build.feature new file mode 100644 index 0000000..e16db7c --- /dev/null +++ b/behave/features/hermetic-build.feature @@ -0,0 +1,19 @@ +Feature: Mock 5.7+ supports hermetic builds + + @hermetic_build + Scenario: Hermetic build against a DNF5 distribution + Given an unique mock namespace + When deps for python-copr-999-1.src.rpm are calculated against fedora-rawhide-x86_64 + And a local repository is created from lockfile + And a hermetic build is retriggered with the lockfile and repository + Then the build succeeds + And the produced lockfile is validated properly + + @hermetic_build + Scenario: Hermetic build against a DNF4 distribution + Given an unique mock namespace + When deps for mock-test-bump-version-1-0.src.rpm are calculated against centos-stream+epel-9-x86_64 + And a local repository is created from lockfile + And a hermetic build is retriggered with the lockfile and repository + Then the build succeeds + And the produced lockfile is validated properly diff --git a/behave/features/library.feature b/behave/features/library.feature new file mode 100644 index 0000000..6c06f26 --- /dev/null +++ b/behave/features/library.feature @@ -0,0 +1,6 @@ +Feature: Test the "library" methods + + @library @simple_load_config + Scenario: The --resultdir option is incompatible with --chain + When simple_load_config method from mockbuild.config is called with fedora-rawhide-x86_64 args + Then the return value contains a field "description=Fedora Rawhide" diff --git a/behave/features/list-configs.feature b/behave/features/list-configs.feature new file mode 100644 index 0000000..90a009b --- /dev/null +++ b/behave/features/list-configs.feature @@ -0,0 +1,8 @@ +Feature: The --list-chroots commandline option + + @list_chroots + Scenario: Test --list-chroots + When mock is run with "--list-chroots" options + Then the exit code is 0 + And stdout contains "fedora-rawhide-x86_64 Fedora Rawhide" + And stdout contains "rhel+epel-8-x86_64 RHEL 8 + EPEL" diff --git a/behave/features/scrub-all-chroots.feature b/behave/features/scrub-all-chroots.feature new file mode 100644 index 0000000..bbe0b4d --- /dev/null +++ b/behave/features/scrub-all-chroots.feature @@ -0,0 +1,8 @@ +Feature: Clean all chroots + + @clan_all_chroots + Scenario: The --scrub-all-chroots works as expected + When mock is run with "--shell true" options + And mock is run with "--scrub-all-chroots" options + Then the directory /var/lib/mock is empty + And the directory /var/cache/mock is empty diff --git a/behave/pylintrc b/behave/pylintrc new file mode 100644 index 0000000..e528337 --- /dev/null +++ b/behave/pylintrc @@ -0,0 +1,15 @@ +# mock pylint configuration for behave/ subdir + +[MESSAGES CONTROL] + +# Reasoning for wide warning ignore +# --------------------------------- +# import-error +# This is here to silence Pylint in CI where we do not have all the +# build/runtime dependencies installed. +# cyclic-import +# Seems like cyclic-import is just a style check which is not going to be +# fixed: https://github.com/PyCQA/pylint/issues/6983 +# function-redefined +# This is a Behave's policy to create all step methods as `step_impl()`. +disable=import-error,cyclic-import,function-redefined diff --git a/behave/steps/other.py b/behave/steps/other.py new file mode 100644 index 0000000..8acc4fe --- /dev/null +++ b/behave/steps/other.py @@ -0,0 +1,333 @@ +""" Generic testing steps """ + +import glob +import importlib +import json +import os +import shutil +import tarfile +import tempfile +from pathlib import Path + +from hamcrest import ( + assert_that, + contains_string, + ends_with, + equal_to, + has_item, + has_entries, + has_length, + not_, +) +import jsonschema +from behave import given, when, then # pylint: disable=no-name-in-module + +from testlib.commands import run, no_output + +# flake8: noqa +# pylint: disable=missing-function-docstring,function-redefined +# mypy: disable-error-code="no-redef" + + +def _first_int(string, max_lines=20): + for line in string.split("\n")[:max_lines]: + if not line: + continue + first_word = line.split()[0] + if first_word.isdigit(): + return first_word + raise RuntimeError("unexpected dnf history output") + + +def add_cleanup_last_transaction(context): + # DNF5 support https://github.com/rpm-software-management/dnf5/issues/140 + dnf = ["sudo", "/usr/bin/dnf", "history"] + _, out, _ = run(dnf + ["list"]) + transaction_id = _first_int(out) + + def _revert_transaction(_context): + cmd = dnf + ["undo", transaction_id, "-y"] + with no_output(): + assert_that(run(cmd)[0], equal_to(0)) + + context.add_cleanup(_revert_transaction, context) + + +@given('an unique mock namespace') +def step_impl(context): + print(f"using uniqueext {context.uniqueext}") + context.uniqueext_used = True + + +@given('the {package} package {state} installed on host') +def step_impl(context, package, state): + """ + Install the package, and uninstall in post- action. If state is "not", then + just check it is not installed. + """ + is_installed, _, _ = run(["rpm", "-q", package]) + # exit_status 0 => installed + is_installed = bool(not is_installed) + + if "not" in state: + if not is_installed: + return # nothing to do + + # Remove the package and schedule its removal + cmd = ["sudo", "dnf", "-y", "remove", package] + assert_that(run(cmd)[0], equal_to(0)) + # schedule removal + add_cleanup_last_transaction(context) + return + + if is_installed: + return + + # install the package, and schedule removal + def _uninstall_pkg(_context): + cmd = ["sudo", "dnf", "-y", "remove", package] + with no_output(): + assert_that(run(cmd)[0], equal_to(0)) + + cmd = ["sudo", "dnf", "-y", "install", package] + assert_that(run(cmd)[0], equal_to(0)) + context.add_cleanup(_uninstall_pkg, context) + + +@given('pre-intitialized chroot') +def step_impl(context): + context.mock.init() + + +@given('a custom third-party repository is used for builds') +def step_impl(context): + context.add_repos.append( + "https://raw.githubusercontent.com/rpm-software-management/" + "mock-test-data/main/repo/" + ) + + +@given("a created local repository") +def step_impl(context): + context.local_repo = tempfile.mkdtemp(prefix="mock-tests-local-repo-") + run(["createrepo_c", context.local_repo]) + + +@given('the local repo contains a "{rpm}" RPM') +def step_impl(context, rpm): + rpm = context.download_rpm(rpm) + shutil.move(rpm, context.local_repo) + run(["createrepo_c", context.local_repo]) + + +@given("the local repo is used for builds") +def step_impl(context): + context.add_repos.append(context.local_repo) + + +@when('a build is depending on third-party repo requested') +@when('a build which requires the "always-installable" RPM is requested') +def step_impl(context): + local_file = context.download_rpm("buildrequires-always-installable") + context.mock.rebuild([local_file]) + + +@then('the build succeeds') +def step_impl(context): + assert os.path.exists(context.mock.resultdir) + rpms = glob.glob(os.path.join(context.mock.resultdir, "*.rpm")) + print("Found RPMs: " + ", ".join(rpms)) + assert_that(rpms, has_item(ends_with(".src.rpm"))) + assert_that(rpms, has_item(not_(ends_with(".src.rpm")))) + + +@when('mock is run with "{options}" options') +def step_impl(context, options): + options = options.split() + context.last_cmd = run(['mock'] + options) + + +@given('mock is always executed with "{options}"') +def step_impl(context, options): + options = options.split() + context.mock.common_opts += options + + +@then('the exit code is {code}') +def step_impl(context, code): + code = int(code) + assert_that(context.last_cmd[0], equal_to(code)) + + +@then('the one-liner error contains "{expected_message}"') +def step_impl(context, expected_message): + err = context.last_cmd[2].splitlines() + assert_that(err, has_length(1)) + assert_that(err[0], contains_string(expected_message)) + + +def _rebuild_online(context, chroot=None, package=None): + package = package or "mock-test-bump-version-1-0.src.rpm" + url = context.test_storage + package + if chroot: + context.mock.chroot = chroot + context.mock.chroot_opt = chroot + context.mock.rebuild([url]) + + +@when('an online source RPM is rebuilt') +def step_impl(context): + _rebuild_online(context) + + +@when('an online source RPM is rebuilt against {chroot}') +def step_impl(context, chroot): + _rebuild_online(context, chroot) + + +@when('an online SRPM {package} is rebuilt against {chroot}') +def step_impl(context, package, chroot): + _rebuild_online(context, chroot, package) + + +@then('{output} contains "{text}"') +def step_impl(context, output, text): + index = 1 if output == "stdout" else 2 + real_output = context.last_cmd[index] + assert_that(real_output, contains_string(text)) + + +@when('{call} method from {module} is called with {args} args') +def step_impl(context, call, module, args): + imp = importlib.import_module(module) + method = getattr(imp, call) + args = args.split() + context.last_method_call_retval = method(*args) + + +@then('the return value contains a field "{field}={value}"') +def step_impl(context, field, value): + assert_that(context.last_method_call_retval[field], + equal_to(value)) + + +@when('deps for {srpm} are calculated against {chroot}') +def step_impl(context, srpm, chroot): + url = context.test_storage + srpm + context.mock.calculate_deps(url, chroot) + + +@when('a local repository is created from lockfile') +def step_impl(context): + mock_run = context.mock_runs["calculate-build-deps"][-1] + lockfile = mock_run["lockfile"] + + context.local_repo = tempfile.mkdtemp(prefix="mock-tests-local-repo-") + cmd = ["mock-hermetic-repo", "--lockfile", lockfile, "--output-repo", + context.local_repo] + assert_that(run(cmd)[0], equal_to(0)) + + +@when('a hermetic build is retriggered with the lockfile and repository') +def step_impl(context): + context.mock.hermetic_build() + + +@then('the produced lockfile is validated properly') +def step_impl(context): + mock_run = context.mock_runs["calculate-build-deps"][-1] + lockfile = mock_run["lockfile"] + with open(lockfile, "r", encoding="utf-8") as fd: + lockfile_data = json.load(fd) + + assert_that(lockfile_data["buildroot"]["rpms"], + has_item(has_entries({"name": "filesystem"}))) + + schemafile = os.path.join(os.path.dirname(__file__), '..', '..', + "mock", "docs", + "buildroot-lock-schema-1.1.0.json") + with open(schemafile, "r", encoding="utf-8") as fd: + schema = json.load(fd) + + jsonschema.validate(lockfile_data, schema) + + +@given('next mock call uses {option} option') +def step_impl(context, option): + context.next_mock_options.append(option) + + +@then("the directory {directory} is empty") +def step_impl(_, directory): + assert_that(os.path.exists(directory), equal_to(True)) + assert_that(not os.listdir(directory), equal_to(True)) + + +@given('chroot_scan is enabled for {regex}') +def step_impl(context, regex): + context.custom_config += f"""\ +config_opts['plugin_conf']['chroot_scan_enable'] = True +config_opts['plugin_conf']['chroot_scan_opts']['regexes'] = ["{regex}"] +config_opts['plugin_conf']['chroot_scan_opts']['only_failed'] = False +""" + + +@then('{file} file is in chroot_scan result dir') +def step_impl(context, file): + resultdir = os.path.join(context.mock.resultdir, 'chroot_scan') + + # Find the expected file + found = False + print("resultdir: ", resultdir) + for _, _, files in os.walk(resultdir): + for f in files: + print(f) + if f == file: + found = True + break + if found: + break + assert_that(found, equal_to(True)) + + +@given('chroot_scan is configured to produce tarball') +def step_impl(context): + context.custom_config += """\ +config_opts['plugin_conf']['chroot_scan_opts']['write_tar'] = True +""" + + +@then('ownership of all chroot_scan files is correct') +def step_impl(context): + resultdir = os.path.join(context.mock.resultdir, 'chroot_scan') + for root, dirs, files in os.walk(resultdir): + for f in files + dirs: + path = Path(root) / f + assert_that(path.group(), equal_to("mock")) + assert_that(path.owner(), equal_to(context.current_user)) + + +@then('chroot_scan tarball has correct perms and provides dnf5.log') +def step_impl(context): + tarball = Path(context.mock.resultdir, 'chroot_scan.tar.gz') + with tarfile.open(tarball, 'r:gz') as tarf: + for file in tarf.getnames(): + if file.endswith("dnf5.log"): + break + assert_that(tarball.group(), equal_to("mock")) + assert_that(tarball.owner(), equal_to(context.current_user)) + + +@when('OCI tarball from {chroot} backed up and will be used') +def step_impl(context, chroot): + resultdir = f"/var/lib/mock/{chroot}-{context.uniqueext}/result" + tarball_base = "buildroot-oci.tar" + tarball = os.path.join(resultdir, tarball_base) + assert os.path.exists(tarball) + shutil.copy(tarball, context.workdir) + context.mock.buildroot_image = os.path.join(context.workdir, tarball_base) + + +@when('the {chroot} chroot is scrubbed') +def step_impl(context, chroot): + context.mock.scrub(chroot) diff --git a/behave/testlib/__init__.py b/behave/testlib/__init__.py new file mode 100644 index 0000000..a54a12c --- /dev/null +++ b/behave/testlib/__init__.py @@ -0,0 +1 @@ +""" Helper library for Mock's BDD """ diff --git a/behave/testlib/commands.py b/behave/testlib/commands.py new file mode 100644 index 0000000..d2e74d7 --- /dev/null +++ b/behave/testlib/commands.py @@ -0,0 +1,62 @@ +""" +Executing commands in Mock's behave test suite. +""" + +from contextlib import contextmanager +import io +import shlex +import subprocess +import sys + + +@contextmanager +def no_output(): + """ + Suppress stdout/stderr when it is not captured by behave + https://github.com/behave/behave/issues/863 + """ + real_out = sys.stdout, sys.stderr + sys.stdout = io.StringIO() + sys.stderr = io.StringIO() + yield + sys.stdout, sys.stderr = real_out + + +def quoted_cmd(cmd): + """ shell quoted cmd array as string """ + return " ".join(shlex.quote(arg) for arg in cmd) + + +def run(cmd): + """ + Return exitcode, stdout, stderr. It's bad there's no such thing in behave + directly. + """ + try: + with subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) as process: + stdout, stderr = process.communicate() + print(f"Exit code: {process.returncode} in: {quoted_cmd(cmd)}") + if stdout: + print("stdout:") + print(stdout) + if stderr: + print("stderr:") + print(stderr) + return process.returncode, stdout, stderr + except (FileNotFoundError, PermissionError) as e: + print(f"Error running command {quoted_cmd(cmd)}: {e}") + return -1, "", str(e) + + +def run_check(cmd): + """ run, but check nonzero exit status """ + retcode, stdout, stderr = run(cmd) + if retcode != 0: + raise RuntimeError(f"Command failed with return code {retcode}: " + f"{quoted_cmd(cmd)}\n{stderr}") + return stdout, stderr diff --git a/behave/testlib/mock.py b/behave/testlib/mock.py new file mode 100644 index 0000000..9e44fd6 --- /dev/null +++ b/behave/testlib/mock.py @@ -0,0 +1,160 @@ +""" +Stateful "Mock" command object. +""" + +from pathlib import Path +import os + +from testlib.commands import run_check + + +class Mock: + """ /bin/mock wrapper """ + def __init__(self, context): + self.context = context + self.common_opts = [] + + # The chroot being used (e.g. fedora-rawhide-x86_64). If None is used, + # it is automatically set to the default.cfg target. + self.chroot = context.chroot + + # The -r/--root option being used. Sometimes it is convenient to use a + # custom config file that includes `fedora-rawhide-x86_64` + # configuration without overriding the `config_opts["root"]" opt. + # None means "no option used". + self.chroot_opt = None + + # Sometimes we use multiple chroots. Clean them all. + self.more_cleanups = [] + + context.mock_runs = { + "init": [], + "rebuild": [], + "scrubs": [], + "calculate-build-deps": [], + } + + self.buildroot_image = None + + @property + def basecmd(self): + """ return the pre-configured mock base command """ + cmd = ["mock"] + if self.chroot_opt: + cmd += ["-r", self.chroot_opt] + if self.context.uniqueext_used: + cmd += ["--uniqueext", self.context.uniqueext] + for repo in self.context.add_repos: + cmd += ["-a", repo] + if self.common_opts: + cmd += self.common_opts + if self.context.next_mock_options: + cmd += self.context.next_mock_options + self.context.next_mock_options = [] + return cmd + + def init(self): + """ initialize chroot """ + out, err = run_check(self.basecmd + ["--init"]) + self.context.mock_runs['init'] += [{ + "status": 0, + "out": out, + "err": err, + }] + return out, err + + def scrub(self, chroot=None): + """ initialize chroot """ + opts = ["--scrub=all"] + if chroot is not None: + opts += ["-r", chroot] + out, err = run_check(self.basecmd + opts) + self.context.mock_runs['scrubs'] += [{ + "status": 0, + "out": out, + "err": err, + }] + return out, err + + def rebuild(self, srpms): + """ Rebuild source RPM(s) """ + + chrootspec = [] + if self.context.custom_config: + config_file = Path(self.context.workdir) / "custom.cfg" + with config_file.open("w") as fd: + fd.write(f"include('{self.chroot}.cfg')\n") + fd.write(self.context.custom_config) + chrootspec = ["-r", str(config_file)] + + opts = [] + if self.buildroot_image: + # use and drop + opts += ["--buildroot-image", self.buildroot_image] + self.buildroot_image = None + opts += ["--rebuild"] + srpms + + out, err = run_check(self.basecmd + chrootspec + opts) + self.context.mock_runs['rebuild'] += [{ + "status": 0, + "out": out, + "err": err, + "srpms": srpms, + }] + + def calculate_deps(self, srpm, chroot): + """ + Call Mock with --calculate-build-dependencies and produce lockfile + """ + call = self.basecmd + ["-r", chroot] + self.more_cleanups += [call] + out, err = run_check(call + ["--calculate-build-dependencies", srpm]) + self.chroot = chroot + self.context.mock_runs["calculate-build-deps"].append({ + "status": 0, + "out": out, + "err": err, + "srpm": srpm, + "chroot": chroot, + "lockfile": os.path.join(self.resultdir, "buildroot_lock.json") + }) + + def hermetic_build(self): + """ + From the previous calculate_deps() run, perform hermetic build + """ + mock_calc = self.context.mock_runs["calculate-build-deps"][-1] + out, err = run_check(self.basecmd + [ + "--hermetic-build", mock_calc["lockfile"], self.context.local_repo, + mock_calc["srpm"] + ]) + self.context.mock_runs["rebuild"].append({ + "status": 0, + "out": out, + "err": err, + }) + # We built into a hermetic-build.cfg! + self.chroot = "hermetic-build" + self.chroot_opt = "hermetic-build" + + def clean(self): + """ Clean chroot, but keep dnf/yum caches """ + args = ["--scrub=bootstrap", "--scrub=root-cache", "--scrub=chroot"] + run_check(self.basecmd + args) + for call in self.more_cleanups: + run_check(call + args) + + @property + def resultdir(self): + """ Where the results are stored """ + resultdir = "/var/lib/mock/" + self.chroot + if self.context.uniqueext_used: + resultdir += "-" + self.context.uniqueext + return resultdir + "/result" + + +def assert_is_subset(set_a, set_b): + """ assert that SET_A is subset of SET_B """ + if set_a.issubset(set_b): + return + raise AssertionError(f"Set {set_a} is not a subset of {set_b}") diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000..e8adde9 --- /dev/null +++ b/config.yaml @@ -0,0 +1,117 @@ +# deb-mock configuration file +# Debian's equivalent to Fedora's Mock build environment manager + +# Global configuration +global: + basedir: "/var/lib/deb-mock" + rootdir: "/var/lib/deb-mock/chroots" + resultdir: "/var/lib/deb-mock/results" + cache_dir: "/var/cache/deb-mock" + log_dir: "/var/log/deb-mock" + +# Default chroot configuration +defaults: + distribution: "bookworm" + architecture: "amd64" + mirror: "http://deb.debian.org/debian" + security_mirror: "http://deb.debian.org/debian-security" + updates_mirror: "http://deb.debian.org/debian" + + # Package installation + install_packages: + - "build-essential" + - "fakeroot" + - "devscripts" + - "debhelper" + - "dh-make" + - "sbuild" + - "schroot" + + # Build dependencies + build_dependencies: + - "build-essential" + - "fakeroot" + - "devscripts" + - "debhelper" + - "dh-make" + +# Chroot profiles +profiles: + bookworm-amd64: + distribution: "bookworm" + architecture: "amd64" + mirror: "http://deb.debian.org/debian" + security_mirror: "http://deb.debian.org/debian-security" + updates_mirror: "http://deb.debian.org/debian" + components: ["main", "contrib", "non-free"] + + bookworm-arm64: + distribution: "bookworm" + architecture: "arm64" + mirror: "http://deb.debian.org/debian" + security_mirror: "http://deb.debian.org/debian-security" + updates_mirror: "http://deb.debian.org/debian" + components: ["main", "contrib", "non-free"] + + sid-amd64: + distribution: "sid" + architecture: "amd64" + mirror: "http://deb.debian.org/debian" + components: ["main", "contrib", "non-free"] + +# Plugin configuration +plugins: + mount: + enabled: true + mount_points: + - source: "/proc" + target: "/proc" + type: "proc" + - source: "/sys" + target: "/sys" + type: "sysfs" + - source: "/dev" + target: "/dev" + type: "bind" + + cache: + enabled: true + root_cache: true + package_cache: true + build_cache: true + + security: + enabled: true + user_isolation: true + network_isolation: true + resource_limits: true + +# Integration settings +integration: + deb_orchestrator_url: "http://localhost:8080" + deb_compose_url: "http://localhost:8080" + + # Build tools + sbuild_path: "/usr/bin/sbuild" + schroot_path: "/usr/bin/schroot" + debootstrap_path: "/usr/sbin/debootstrap" + + # Package managers + apt_path: "/usr/bin/apt" + dpkg_path: "/usr/bin/dpkg" + +# Logging configuration +logging: + level: "INFO" + format: "text" + file: "/var/log/deb-mock/deb-mock.log" + max_size: "100MB" + max_files: 5 + +# Performance settings +performance: + parallel_downloads: 4 + max_retries: 3 + timeout: 3600 + memory_limit: "2G" + disk_limit: "10G" diff --git a/docs/FAQ.md b/docs/FAQ.md index 4b7cf3b..1425866 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -1,327 +1,37 @@ -# Deb-Mock FAQ +--- +layout: default +title: FAQ +--- -## Frequently Asked Questions +## FAQ -This FAQ addresses common issues and questions about **Deb-Mock**, a Debian-focused alternative to Fedora's Mock. +### How to preserve environment variable in chroot -### Environment Variables in Chroot +Q: I put -**Q: I set environment variables in my configuration, but they're not preserved in the chroot environment.** + config_opts['environment']['VAR'] = os.environ['VAR'] -**A:** This is a common issue with chroot environments. **Deb-Mock** provides several solutions: +into config, but the variable is not preserved. -#### Solution 1: Use the `--preserve-env` flag -```bash -deb-mock shell --preserve-env -``` +A: Environment is sanitized by consolehelper when elevating UID. You need to alter `/etc/security/console.apps/mock` too. -#### Solution 2: Configure specific environment variables -```yaml -# deb-mock.yaml -preserve_environment: - - CC - - CXX - - CFLAGS - - CXXFLAGS - - DEB_BUILD_OPTIONS - - CCACHE_DIR -``` +### I cannot build Fedora or RHEL8 beta package on RHEL/CentOS 7 -#### Solution 3: Disable environment sanitization -```yaml -# deb-mock.yaml -environment_sanitization: false -``` +Q: I am on RHEL 7 and when I run `mock -r fedora-28-x86_64 --init` (similarly for rhelbeta-8-x86_64) I get: -#### Solution 4: Use the `--env-var` option -```bash -deb-mock shell --env-var CC=gcc --env-var CFLAGS="-O2" -``` + .... + ---> Package patch.x86_64 0:2.7.6-4.fc28 will be installed + ---> Package redhat-rpm-config.noarch 0:108-1.fc28 will be installed + Error: Invalid version flag: if -**Why this happens:** Chroot environments sanitize environment variables for security reasons. **Deb-Mock** provides controlled ways to preserve necessary variables while maintaining security. +A: This is not Mock error. This is because redhat-rpm-config in Fedora 28 (& RHEL 8 Beta) contains rich dependency: `Requires: (annobin if gcc)`. This is a new rpm's feature and is not recognized by RHEL7's rpm. When you are installing the fedora-28 chroot, mock is using host's rpm. And RHEL7 rpm cannot install this package, because of the new feature, which does not recognize. -### Cross-Distribution Package Building +The solution is to use mock's [bootstrap feature](https://rpm-software-management.github.io/mock/Release-Notes-1.4.1#bootstrap-chroot). It is not enabled by default, because there are still some [unresolved issues](https://github.com/rpm-software-management/mock/labels/bootstrap), but generally it works. Try: -**Q: I'm on Debian Stable but need to build packages for Debian Sid. How can I do this?** + mock -r fedora-28-x86_64 --init --bootstrap-chroot -**A:** Use **Deb-Mock**'s bootstrap chroot feature, similar to Mock's `--bootstrap-chroot`: +### When I can expect next release -#### Solution: Use bootstrap chroot -```bash -# Create a Sid chroot using bootstrap -deb-mock init-chroot debian-sid-amd64 --suite sid --bootstrap +Q: A developer merged my pull-request. When I can expect the next release with my fix? -# Build packages in the Sid chroot -deb-mock -r debian-sid-amd64 build package.dsc -``` - -#### Configuration-based approach -```yaml -# deb-mock.yaml -use_bootstrap_chroot: true -bootstrap_chroot_name: "debian-stable-bootstrap" -suite: "sid" -architecture: "amd64" -``` - -**Why this is needed:** Building packages for newer distributions on older systems can fail due to package manager version incompatibilities. Bootstrap chroots create a minimal environment with the target distribution's tools to build the final chroot. - -### Build Dependencies Not Found - -**Q: My build fails with "Package not found" errors for build dependencies.** - -**A:** This usually indicates repository or dependency resolution issues: - -#### Solution 1: Update the chroot -```bash -deb-mock update-chroot debian-bookworm-amd64 -``` - -#### Solution 2: Check repository configuration -```yaml -# deb-mock.yaml -mirror: "http://deb.debian.org/debian/" -security_mirror: "http://security.debian.org/debian-security/" -backports_mirror: "http://deb.debian.org/debian/" -``` - -#### Solution 3: Install missing dependencies manually -```bash -deb-mock shell debian-bookworm-amd64 -# Inside chroot: -apt-get update -apt-get install build-essential devscripts -``` - -#### Solution 4: Use verbose output for debugging -```bash -deb-mock --verbose build package.dsc -``` - -### Chroot Creation Fails - -**Q: I get permission errors when creating chroots.** - -**A:** Chroot creation requires root privileges: - -#### Solution 1: Use sudo -```bash -sudo deb-mock init-chroot debian-bookworm-amd64 -``` - -#### Solution 2: Add user to appropriate groups -```bash -sudo usermod -a -G sbuild $USER -# Log out and back in -``` - -#### Solution 3: Check disk space -```bash -df -h /var/lib/deb-mock -``` - -#### Solution 4: Verify debootstrap installation -```bash -sudo apt-get install debootstrap schroot -``` - -### Build Performance Issues - -**Q: My builds are slow. How can I speed them up?** - -**A:** **Deb-Mock** provides several performance optimization features: - -#### Solution 1: Enable caching -```yaml -# deb-mock.yaml -use_root_cache: true -use_package_cache: true -use_ccache: true -``` - -#### Solution 2: Use parallel builds -```yaml -# deb-mock.yaml -parallel_jobs: 8 -parallel_compression: true -``` - -#### Solution 3: Use tmpfs for temporary files -```yaml -# deb-mock.yaml -use_tmpfs: true -tmpfs_size: "4G" -``` - -#### Solution 4: Clean up old caches -```bash -deb-mock cleanup-caches -``` - -### Network and Proxy Issues - -**Q: I'm behind a proxy and can't download packages.** - -**A:** Configure proxy settings in your configuration: - -#### Solution: Configure proxy -```yaml -# deb-mock.yaml -http_proxy: "http://proxy.example.com:3128" -https_proxy: "http://proxy.example.com:3128" -no_proxy: "localhost,127.0.0.1" -``` - -### Package Signing Issues - -**Q: How do I sign packages with GPG keys?** - -**A:** **Deb-Mock** supports package signing through configuration: - -#### Solution: Configure signing -```yaml -# deb-mock.yaml -sign_packages: true -gpg_key: "your-gpg-key-id" -gpg_passphrase: "your-passphrase" # Use environment variable in production -``` - -### Debugging Build Failures - -**Q: My build failed. How can I debug it?** - -**A:** **Deb-Mock** provides several debugging tools: - -#### Solution 1: Use verbose output -```bash -deb-mock --verbose build package.dsc -``` - -#### Solution 2: Keep chroot for inspection -```bash -deb-mock build package.dsc --keep-chroot -deb-mock shell # Inspect the failed build environment -``` - -#### Solution 3: Check build logs -```bash -# Build logs are automatically captured -ls -la ./output/ -cat ./output/build.log -``` - -#### Solution 4: Use debug mode -```bash -deb-mock --debug build package.dsc -``` - -### Chain Building Issues - -**Q: My chain build fails because later packages can't find earlier packages.** - -**A:** This is a common issue with chain builds: - -#### Solution 1: Use the chain command -```bash -deb-mock chain package1.dsc package2.dsc package3.dsc -``` - -#### Solution 2: Continue on failure -```bash -deb-mock chain package1.dsc package2.dsc --continue-on-failure -``` - -#### Solution 3: Check package installation -```bash -deb-mock shell -# Inside chroot, check if packages are installed: -dpkg -l | grep package-name -``` - -### Configuration Issues - -**Q: How do I create a custom configuration?** - -**A:** **Deb-Mock** supports custom configurations: - -#### Solution 1: Create a custom config file -```yaml -# my-config.yaml -chroot_name: "custom-debian" -architecture: "amd64" -suite: "bookworm" -mirror: "http://deb.debian.org/debian/" -use_root_cache: true -use_ccache: true -parallel_jobs: 4 -``` - -#### Solution 2: Use the config file -```bash -deb-mock -c my-config.yaml build package.dsc -``` - -#### Solution 3: Use core configurations -```bash -# List available configurations -deb-mock list-configs - -# Use a core configuration -deb-mock -r debian-bookworm-amd64 build package.dsc -``` - -### Common Error Messages - -#### "Chroot does not exist" -```bash -# Create the chroot first -deb-mock init-chroot debian-bookworm-amd64 -``` - -#### "Permission denied" -```bash -# Use sudo for chroot operations -sudo deb-mock init-chroot debian-bookworm-amd64 -``` - -#### "Package not found" -```bash -# Update the chroot -deb-mock update-chroot debian-bookworm-amd64 -``` - -#### "Build dependencies not satisfied" -```bash -# Install build dependencies -deb-mock shell -# Inside chroot: -apt-get install build-essential devscripts -``` - -### Getting Help - -**Q: Where can I get more help?** - -**A:** Several resources are available: - -1. **Documentation**: Check the main README and configuration documentation -2. **Verbose Output**: Use `--verbose` and `--debug` flags for detailed information -3. **Error Messages**: **Deb-Mock** provides detailed error messages with suggestions -4. **Logs**: Check build logs in the output directory -5. **Community**: Report issues on the project's issue tracker - -### Performance Tips - -1. **Use caching**: Enable root cache, package cache, and ccache -2. **Parallel builds**: Set appropriate `parallel_jobs` for your system -3. **Clean up**: Regularly run `deb-mock cleanup-caches` -4. **Monitor resources**: Use `deb-mock cache-stats` to monitor cache usage -5. **Optimize chroots**: Use tmpfs for temporary files if you have sufficient RAM - -### Security Considerations - -1. **Environment sanitization**: Keep environment sanitization enabled unless necessary -2. **Root privileges**: Only use sudo when required for chroot operations -3. **Package verification**: Verify source packages before building -4. **Network security**: Use HTTPS mirrors and configure proxies securely -5. **Cache security**: Regularly clean caches to remove sensitive build artifacts \ No newline at end of file +A: I try to stick to two month cadence. Check the last release date and add two months and you can set your expectation. Of course things like Christmas or summer holidays can add a few weeks. On the other hand the branching event in Fedora can make it shorter as I usually do a mock release a day before Fedora branches, because I had to add new configs there anyway. diff --git a/docs/Feature-bootstrap.md b/docs/Feature-bootstrap.md new file mode 100644 index 0000000..f1e1713 --- /dev/null +++ b/docs/Feature-bootstrap.md @@ -0,0 +1,30 @@ +--- +layout: default +title: Feature bootstrap +--- + +## Bootstrap chroot + +Mock is calling `dnf --installroot` to install packages for target architecture into the target directory. This works. Mostly. The only problem that use host DNF and rpm to install packages. But this can cause a problem when a new RPM feature is introduced. Like Soft dependencies or Rich dependencies. When you have EL6 host and try to install Fedora rawhide package with Rich dependency then rpm will fail and you cannot do anything about it. You can upgrade your build machine to Fedora rawhide, but that is often not possible when it is part of critical infrastructure. + +So we introduced Boostrap chroot. And 'we' actually means Michael Cullen who implements it. And Igor Gnatenko who proposed this idea. Big kudos for both of them. + +Bootstrap chroot means that we first create very minimal chroot for the target platform and we call DNF/YUM from that platform. For example: when you are on RHEL7 and you want to build a package for `fedora-26-x86_64`, mock will first create chroot called `fedora-26-x86_64-bootstrap`, it will install DNF and rpm there (fc26 versions). Then it will call DNF from `fedora-26-x86_64-bootstrap` to install all needed packages to `fedora-26-x86_64` chroot. + +The disadvantage is that you will need more storage in `/var/lib/mock`, the build is a little bit slower. But you will hardly notice that unless you disabled `yum_cache` and `root_cache` plugins for some reasons. + +The advantage is that you can use a stable version of OS to build packages for even most recent OS. And vice versa. + +This feature is enabled by default. If you want to disable it you should set: + +``` + config_opts['use_bootstrap'] = False +``` + +in your configuration. + +This has been added in Mock 1.4.1. + +### Using bootstrap with local repositories + +It is possible to use `file://` local repositories with bootstrap chroot. However, you should not bind mount repositories located in `/tmp`, `/dev`, etc., as they might be over-mounted by systemd-nspawn. diff --git a/docs/Feature-buildroot-image.md b/docs/Feature-buildroot-image.md new file mode 100644 index 0000000..324a239 --- /dev/null +++ b/docs/Feature-buildroot-image.md @@ -0,0 +1,42 @@ +--- +layout: default +title: Feature buildroot image +--- + +Starting from version v6.0, Mock allows users to use an OCI container image for +pre-creating the buildroot (build chroot). It can be either an online container +image hosted in a registry (or cached locally), or a local image in the form of +a tarball. + +Be cautious when using chroot-compatible images (e.g., it is not advisable to +combine EPEL `ppc64le` images with `fedora-rawhide-x86_64` chroot). + +## Example Use-Case + +1. Mock aggressively caches the build root, so clean up your chroot first: + + ```bash + $ mock -r fedora-rawhide-x86_64 --scrub=all + ``` + +2. Perform any normal Mock operation, but select the OCI image on top of that: + + ```bash + $ mock -r fedora-rawhide-x86_64 \ + --buildroot-image registry.fedoraproject.org/fedora:41 \ + --rebuild /your/src.rpm + ``` + +## Using Exported Buildroot Image + +The [export_buildroot_image](Plugin-Export-Buildroot-Image) plugin allows you to +wrap a prepared buildroot as an OCI archive (tarball). If you have this +tarball, you may select it as well: + +```bash +$ mock -r fedora-rawhide-x86_64 \ + --buildroot-image /tmp/buildroot-oci.tar \ + --rebuild /your/src.rpm +``` + +Again, ensure that you do not combine incompatible chroot and image pairs. diff --git a/docs/Feature-container-for-bootstrap.md b/docs/Feature-container-for-bootstrap.md new file mode 100644 index 0000000..7af0a9d --- /dev/null +++ b/docs/Feature-container-for-bootstrap.md @@ -0,0 +1,48 @@ +--- +layout: default +title: Feature container for bootstrap +--- + +## Container image for bootstrap + +In past, we had some incompatibilities between host and build target. They were, in fact, small. Like using a different package manager. Some were big. Like, the introduction of Weak and Rich dependencies. For this reason, we introduced [bootstrap](Feature-bootstrap). But then comes [zstd payload](https://fedoraproject.org/wiki/Changes/Switch_RPMs_to_zstd_compression). This is a new type of payload. And to install packages with this payload, you need an `rpm` binary, which supports this payload. This is true for all current Fedoras. Unfortunately, neither RHEL 8 nor RHEL 7 supports this payload. So even bootstrap will not help you to build Fedora packages on RHEL 8. + +We come up with a nice feature. Mock will not install bootstrap chroot itself. Instead, it will download the container image, extract the image, and use this extracted directory as a bootstrap chroot. And from this bootstrapped chroot install the final one. + +Using this feature, **any** incompatible feature in either RPM or DNF can be used in the target chroot. Now or in future. And you will be able to install the final chroot. You do not even need to have `rpm` on a host. So this should work on any system, even Debian-based ones. The only requirement for this feature is [Podman](https://podman.io/). Do not forget to install the `podman` package. + +This feature is available since 1.4.20 and enabled by default from 5.0. You can disable it using: + + config_opts['use_bootstrap_image'] = False + +It can be enabled or disabled on the command line using `--use-bootstrap-image` or `--no-bootstrap-image` options. + +Note however that also this is prerequisite: + + config_opts['use_bootstrap_container'] = True # or --bootstrap-chroot option + +To specify which image should be used for bootstrap container you can put in config: + + config_opts['bootstrap_image'] = 'registry.fedoraproject.org/fedora:latest' + +This is a general config. Each config has specified its own image specified. E.g. CentOS 7 has `config_opts['bootstrap_image'] = 'centos:7'` in config. So unless you use your own config, you can enable this feature, and the right image will be used. + +The image contents are typically suboptimal for Mock's use-case. In particular, +Mock needs to have a correct package manager (as specified in the +`package_manager` configuration option) installed inside the image, along with +the `builddep` functionality (typically provided by `python3-dnf-plugins-core`). +This is why Mock still has to 'update the downloaded bootstrap' somehow. If you +happen to have an image with `builddep` pre-installed, you can set +`bootstrap_image_ready` to 'True': + + config_opts['bootstrap_image_ready'] = True + +This option will significantly reduce the bootstrap preparation time, as no +package management actions need to be performed for the bootstrap (no need to +download and initialize package manager caches). + +There is one known issue: + + * Neither Mageia 6 nor 7 works correctly now with this feature. + +Technically, you can use any container, as long as there is the required package manager (DNF or YUM). The rest of the needed packages will be installed by mock. diff --git a/docs/Feature-external-deps.md b/docs/Feature-external-deps.md new file mode 100644 index 0000000..474bd48 --- /dev/null +++ b/docs/Feature-external-deps.md @@ -0,0 +1,43 @@ +--- +layout: default +title: Feature external dependencies +--- +## External dependencies + +It can happen that you need some library that is not packaged. PyPI has ten times more modules than Fedora has packages. The same for Rubygems.org, Crates.io... + +External dependencies allow you to install a package using the native package manager. I.e. not dnf or rpm, but rather using `pip`, `gem`, etc. + +Right now it is possible to do that only for `BuildRequires`. Run-time `Requires` will need more co-operation with DNF and rpm. + +This feature is disabled by default. It can be enabled using: + +``` +config_opts['external_buildrequires'] = True +``` + +## Modules + +### PyPI + +`BuildRequires: external:pypi:foo` - this will run `pip3 --install foo` + +### Crate + +`BuildRequires: external:crate:foo` - this will run `cargo install foo` + +### Others + +Do you miss other languages here? [File an issue](https://github.com/rpm-software-management/mock/issues) and let us know which language you want to add. There are two requirements. The native package manager needs to be available in Fedora. And the manager has to have `--root` or something similar which let you allow to install the files in the different root path. That is because we run this tool in bootstrap chroot. + +## How it works + +When we find the `external::` prefix in BuildRequires then Mock install the native package manager in bootstrap buildroot. E.g., for `external:pypi:foo` mock will install `pip3` and run `pip3 install --root MOCK_CHROOT foo`. + +To satisfy rpm dependencies Mock calls `create-fake-rpm` and creates a fake rpm package that provides `external:pypi:foo` and installs it in chroot. + +All this is logged in `root.log` and usually start with the line `Installing dependencies to satisfy external:*` + +As of now, this feature requires [bootstrap chroot](Feature-bootstrap) enabled and requires `create-fake-rpm` to be present (package) in the target chroot. + +Available since 2.7 diff --git a/docs/Feature-forcearch.md b/docs/Feature-forcearch.md new file mode 100644 index 0000000..75f35c9 --- /dev/null +++ b/docs/Feature-forcearch.md @@ -0,0 +1,35 @@ +--- +layout: default +title: Feature forcearch +--- +## Forcearch + +Previously you were able to only build for compatible architectures. I.e., you can build `i386` package on `x86_64` architecture. When you tried to build for incompatible architecture, you get this error: + +``` +$ mock -r fedora-28-ppc64le shell +ERROR: Cannot build target ppc64le on arch x86_64, because it is not listed in legal_host_arches ('ppc64le',) +``` + +Now, you can build for any architecture using a new option --force-arch ARCH. [GH#120](https://github.com/rpm-software-management/mock/issues/120) You have to have installed package `qemu-user-static`, which is a new soft dependence. Try this: + +``` +$ sudo dnf install qemu-user-static +$ mock -r fedora-28-ppc64le --forcearch ppc64le shell +``` + +and you get the prompt in PPC64LE Fedora. You can do this for any architecture supported by QEMU. +You got just `INFO` in the log stating: + +``` +INFO: Unable to build arch ppc64le natively on arch x86_64. Setting forcearch to use software emulation. +``` + + +Note: Do not confuse `--forcearch` and `--arch` which are very different options. + +:warning: `qemu-user-static` emulates **user** space, but cannot emulate **kernel** space. If your package need some architecture specific kernel calls or e.g., is parsing output of `lscpu` then this feature is not for you. :( + +This has been added to Mock 1.4.11. + +Since version 2.0 you do not need to use `--forcearch` as Mock will detect that you want to use different than your native architecture and use qemu-user-static automatically. diff --git a/docs/Feature-modularity.md b/docs/Feature-modularity.md new file mode 100644 index 0000000..c84ad44 --- /dev/null +++ b/docs/Feature-modularity.md @@ -0,0 +1,41 @@ +--- +layout: default +title: Feature modularity support +--- +## Modularity support + +There is support for Fedora and RHEL Modularity. This requires `dnf`, not merely +`yum`. It is available for RHEL >= 8 and its clones, and built into +all supported releases of Fedora. + +The new modularity format was added with release 2.4 and uses +`module_setup_commands`. Each command can be specified multiple times, +and mock respects the order of the commands when executing them. + +* Artificial example: + * Disable any potentially enabled postgresql module stream. + * Enable _specific_ postgresql and ruby module streams. + * Install the development nodejs profile and (4) disable it immediately. + +``` +config_opts['module_setup_commands'] = [ + ('disable', 'postgresql'), + ('enable', 'postgresql:12, ruby:2.6'), + ('install', 'nodejs:13/development'), + ('disable', 'nodejs'), + ] +``` + +The obsolete, less flexible, but still available modularity syntax was added in Mock 1.4.2. + +``` +config_opts['module_enable'] = ['list', 'of', 'modules'] +config_opts['module_install'] = ['module1/profile', 'module2/profile'] +``` + +This would call these steps during the init phase. +* `dnf module enable list of modules` +* `dnf module install module1/profile module2/profile` + +You can find more about this obsolete format in this comprehensive blogpost. +* [Modularity Features in Mock](http://frostyx.cz/posts/modularity-features-in-mock). diff --git a/docs/Feature-nosync.md b/docs/Feature-nosync.md new file mode 100644 index 0000000..e19e8df --- /dev/null +++ b/docs/Feature-nosync.md @@ -0,0 +1,20 @@ +--- +layout: default +title: Feature nosync +--- +# Nosync + +One of the reasons why mock has always been quite slow is because installing a lot of packages generates a heavy IO load. But the main bottleneck regarding IO is not unpacking files from packages to disk but writing Yum DB entries. Yum DB access (used by both yum and dnf) generates a lot of `fsync`(2) calls. Those don't really make sense in mock because people generally don't try to recover mock buildroots after hardware failure. We discovered that getting rid of `fsync` improves the package installation speed by almost a factor of 4. Mikolaj Izdebski developed a small C library, `nosync`, that is `LD_PRELOAD`ed and replaces the `fsync` family of calls with (almost) empty implementations. I added support for it in mock. +How to activate it? +You need to install the `nosync` package and for multilib systems (`x86_64`), you need versions for both architectures. Then it can be enabled in mock by setting +``` +# dnf install nosync +config_opts['nosync'] = True +``` +If you cannot install both architectures of nosync (for example on RHEL) and still want mock to use it, you can force its usage. Then expect there can be harmless error messages from ld.so when a 32bit program is executed, but it should otherwise work fine for 64bit binaries. Forcing is done using +``` +config_opts['nosync_force'] = True +``` + +It does require those extra steps to set up but it really pays off quickly. + diff --git a/docs/Feature-package-managers.md b/docs/Feature-package-managers.md new file mode 100644 index 0000000..a7fa84e --- /dev/null +++ b/docs/Feature-package-managers.md @@ -0,0 +1,39 @@ +--- +layout: default +title: Feature DNF +--- +## DNF + +This is default package manager for mock. You can enforce it (e.g. in older Mock) using> + +``` +config_opts['package_manager'] = 'dnf' +``` + +## YUM + +Yum is still used in RHEL 7 and olders. You can enable it using + +``` +config_opts['package_manager'] = 'yum' +``` + +## MicroDNF + +MicroDNF is written in C and does not need Python. It is present in minimal OCI containers. It can be enabled using: + +``` +config_opts['package_manager'] = 'microdnf' +``` + +However, MicroDNF still [does not have `buildep` command](https://github.com/rpm-software-management/microdnf/issues/82). The current implementation still installs DNF (using microdnf) and use DNF to query the build deps. + +You can use following options in the config: +```python +config_opts['microdnf_command'] = '/usr/bin/microdnf' +# "dnf-install" is special keyword which tells mock to use install but with DNF +config_opts['microdnf_install_command'] = 'dnf-install microdnf dnf dnf-plugins-core distribution-gpg-keys' +config_opts['microdnf_builddep_command'] = '/usr/bin/dnf' +config_opts['microdnf_builddep_opts'] = [] +config_opts['microdnf_common_opts'] = [] +``` diff --git a/docs/Feature-rhelchroots.md b/docs/Feature-rhelchroots.md new file mode 100644 index 0000000..3c5c1e3 --- /dev/null +++ b/docs/Feature-rhelchroots.md @@ -0,0 +1,44 @@ +--- +layout: default +title: Feature RHEL chroots +--- +## Build package for RHEL + +Previously, when you had to build a package for RHEL you had to use `epel-7-x86_64` chroot (or similar). This chroot is made of CentOS plus EPEL. This causes a problem when you want to use real RHEL for some reason. E.g., when new RHEL is out, but CentOS not yet. + +To build for RHEL you have to [Red Hat subscription](https://www.redhat.com/en/store/linux-platforms). You can use your existing subscription or you can use [free of charge subscription](https://developers.redhat.com/blog/2016/03/31/no-cost-rhel-developer-subscription-now-available/). + +### Mock RHEL configs + +Mock provides `rhel--` configs which use pure RHEL. +There are also `rhel+epel--` configs which use RHEL plus EPEL. + +### Subscription configuration with Simple Content Access + +If you have [Simple Content Access](https://access.redhat.com/articles/simple-content-access#how-do-i-enable-simple-content-access-for-red-hat-subscription-management-2) enabled, +all you need to do is register the machine you are running mock on. +The register command will prompt you for your username and password. + +``` +$ sudo subscription-manager register +``` + +After this the RHEL mock configs should work without further action. + +``` +$ mock -r rhel-9-x86_64 --shell +``` + +Optionally, you can disable the subscription-manager dnf plugin if you do not need subscription repos directly on your machine. + +```sh +$ sudo subscription-manager config --rhsm.auto_enable_yum_plugins 0 +$ sudo sed -e '/^enabled=/ s/1/0/' -i /etc/dnf/plugins/subscription-manager.conf +``` + +### Multiple client keys + +If there are multiple client keys, +mock takes the first one in `glob("/etc/pki/entitlement/-key.pem")` output. +But users still generate configure `config_opts['redhat_subscription_key_id']` in mock configuration, +or on command line `--config-opts=redhat_subscription_key_id=`. diff --git a/docs/Gemfile b/docs/Gemfile new file mode 100644 index 0000000..9ab202a --- /dev/null +++ b/docs/Gemfile @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +source "https://rubygems.org" + +gem 'jekyll' +gem 'jekyll-theme-slate' +gem 'kramdown-parser-gfm' +gem 'webrick' +gem 'jemoji' diff --git a/docs/LICENSE b/docs/LICENSE new file mode 100644 index 0000000..670154e --- /dev/null +++ b/docs/LICENSE @@ -0,0 +1,116 @@ +CC0 1.0 Universal + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator and +subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for the +purpose of contributing to a commons of creative, cultural and scientific +works ("Commons") that the public can reliably and without fear of later +claims of infringement build upon, modify, incorporate in other works, reuse +and redistribute as freely as possible in any form whatsoever and for any +purposes, including without limitation commercial purposes. These owners may +contribute to the Commons to promote the ideal of a free culture and the +further production of creative, cultural and scientific works, or to gain +reputation or greater distribution for their Work in part through the use and +efforts of others. + +For these and/or other purposes and motivations, and without any expectation +of additional consideration or compensation, the person associating CC0 with a +Work (the "Affirmer"), to the extent that he or she is an owner of Copyright +and Related Rights in the Work, voluntarily elects to apply CC0 to the Work +and publicly distribute the Work under its terms, with knowledge of his or her +Copyright and Related Rights in the Work and the meaning and intended legal +effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not limited +to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, communicate, + and translate a Work; + + ii. moral rights retained by the original author(s) and/or performer(s); + + iii. publicity and privacy rights pertaining to a person's image or likeness + depicted in a Work; + + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + + v. rights protecting the extraction, dissemination, use and reuse of data in + a Work; + + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation thereof, + including any amended or successor version of such directive); and + + vii. other similar, equivalent or corresponding rights throughout the world + based on applicable law or treaty, and any national implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention of, +applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and +unconditionally waives, abandons, and surrenders all of Affirmer's Copyright +and Related Rights and associated claims and causes of action, whether now +known or unknown (including existing as well as future claims and causes of +action), in the Work (i) in all territories worldwide, (ii) for the maximum +duration provided by applicable law or treaty (including future time +extensions), (iii) in any current or future medium and for any number of +copies, and (iv) for any purpose whatsoever, including without limitation +commercial, advertising or promotional purposes (the "Waiver"). Affirmer makes +the Waiver for the benefit of each member of the public at large and to the +detriment of Affirmer's heirs and successors, fully intending that such Waiver +shall not be subject to revocation, rescission, cancellation, termination, or +any other legal or equitable action to disrupt the quiet enjoyment of the Work +by the public as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason be +judged legally invalid or ineffective under applicable law, then the Waiver +shall be preserved to the maximum extent permitted taking into account +Affirmer's express Statement of Purpose. In addition, to the extent the Waiver +is so judged Affirmer hereby grants to each affected person a royalty-free, +non transferable, non sublicensable, non exclusive, irrevocable and +unconditional license to exercise Affirmer's Copyright and Related Rights in +the Work (i) in all territories worldwide, (ii) for the maximum duration +provided by applicable law or treaty (including future time extensions), (iii) +in any current or future medium and for any number of copies, and (iv) for any +purpose whatsoever, including without limitation commercial, advertising or +promotional purposes (the "License"). The License shall be deemed effective as +of the date CC0 was applied by Affirmer to the Work. Should any part of the +License for any reason be judged legally invalid or ineffective under +applicable law, such partial invalidity or ineffectiveness shall not +invalidate the remainder of the License, and in such case Affirmer hereby +affirms that he or she will not (i) exercise any of his or her remaining +Copyright and Related Rights in the Work or (ii) assert any associated claims +and causes of action with respect to the Work, in either case contrary to +Affirmer's express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + + b. Affirmer offers the Work as-is and makes no representations or warranties + of any kind concerning the Work, express, implied, statutory or otherwise, + including without limitation warranties of title, merchantability, fitness + for a particular purpose, non infringement, or the absence of latent or + other defects, accuracy, or the present or absence of errors, whether or not + discoverable, all to the greatest extent permissible under applicable law. + + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without limitation + any person's Copyright and Related Rights in the Work. Further, Affirmer + disclaims responsibility for obtaining any necessary consents, permissions + or other rights required for any use of the Work. + + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to this + CC0 or use of the Work. + +For more information, please see + diff --git a/docs/Mock-Core-Configs.md b/docs/Mock-Core-Configs.md new file mode 100644 index 0000000..8fdf952 --- /dev/null +++ b/docs/Mock-Core-Configs.md @@ -0,0 +1,70 @@ +--- +layout: default +title: Mock Core Configs +--- + +# Mock core configs + +Mock project provides the `mock-core-configs` package which installs the default +[configuration files](configuration) for various RPM-based Linux distributions. +This packages is typically installed with Mock by default (runtime dependency). + +Other projects can provide their own configuration files in other packages, we +know of: + +* [mock-core-configs](https://github.com/rpm-software-management/mock/tree/main/mock-core-configs) (this project) +* [mock-centos-sig-configs](https://pagure.io/centos-sig-hyperscale/mock-centos-sig-configs) +* [RPM Fusion Mock conifgs](https://github.com/rpmfusion-infra/mock-rpmfusion) + + +## Maintenance + +The configuration in this package maintained by the community. +When encountering an issue please use your best judgement to decide +whether a Mock config is broken, or the distribution is broken. + + +#### Mock config issues + +If a Mock config is broken (e.g. [#756][mock-756]), please +[create a ticket for this repository][mock-issues] +and tag the responsible maintainer from the table below. + + +#### Distribution or repository issues + +If a distribution or repository is broken (e.g. [#889][mock-889]), +please report the issue to the appropriate issue tracker for the +distribution. + + +#### Table + +| Distribution | Chroots | Maintainer | Distribution or repository issue tracker | +| ------------------------------------------------------------------------------ | ----------------- | --------------------------------------------------------------------- | ------------- | +| [AlmaLinux](https://almalinux.org/) | `almalinux-*` | [@Conan-Kudo](https://github.com/Conan-Kudo), [@javihernandez](https://github.com/javihernandez) | [Issues](https://bugs.almalinux.org/) | +| [Amazon Linux 2](https://aws.amazon.com/amazon-linux-2/) | `amazonlinux-2-*` | [@stewartsmith](https://github.com/stewartsmith) | NA | +| [Amazon Linux](https://aws.amazon.com/linux/amazon-linux-2023/) | `amazonlinux-*` | [@amazonlinux](https://github.com/amazonlinux) | [Issues](https://github.com/amazonlinux/amazon-linux-2023/issues) | +| [Anolis](https://openanolis.cn/) | `anolis-*` | [@geliwei-ali](https://github.com/geliwei-ali) | [Issues](https://bugzilla.openanolis.cn/) | +| [Azure Linux](https://github.com/microsoft/azurelinux) | `azure-linux-*` | [@scaronni](https://github.com/scaronni) | [Issues](https://github.com/microsoft/azurelinux/issues) | +| [CentOS Stream](https://www.centos.org/centos-stream/) | `centos-stream*` | [Mock team][] | [Issues](https://issues.redhat.com/projects/CS) | +| [CentOS Linux](https://www.centos.org/centos-linux/) | `centos*` | End-of-life. | End-of-life. | +| [Circle Linux](https://cclinux.org/) | `circlelinux-*` | [@bella485](https://github.com/bella485) | [Issues](https://bugzilla.cclinux.org/) | +| [EuroLinux](https://en.euro-linux.com/) | `eurolinux-*` | End-of-life. | End-of-life. | +| [Fedora ELN](https://docs.fedoraproject.org/en-US/eln/) | `fedora-eln-*` | [@fedora-eln](https://github.com/fedora-eln) | [Issues](https://github.com/fedora-eln/eln/issues) | +| [Fedora](https://fedoraproject.org/) | `fedora-*` | [Mock team][] | [Issues](https://github.com/rpm-software-management/mock/issues) | +| [Kylin](https://kylinos.cn/) | `kylin-*` | [@scaronni](https://github.com/scaronni) | NA | +| [Mageia](https://www.mageia.org/en/) | `mageia-*` | [@Conan-Kudo](https://github.com/Conan-Kudo) | [Issues](https://bugs.mageia.org/) | +| [Navy Linux](https://navylinux.org/) | `navy-*` | [@unixlabs](https://github.com/unixlabs) | [issues](https://github.com/navy-linux/issue-tracker/issues) | +| [openEuler](https://www.openeuler.org/en/) | `openeuler-*` | [@Yikun](https://github.com/Yikun) | NA | +| [OpenMandriva](https://www.openmandriva.org/) | `openmandriva-*` | [berolinux](https://github.com/berolinux) | [Issues](https://github.com/OpenMandrivaAssociation/distribution/issues) | +| [openSUSE](https://www.opensuse.org/) | `opensuse-*` | [@Conan-Kudo](https://github.com/Conan-Kudo), [@lkocman](https://github.com/lkocman) | [Issues](https://bugzilla.opensuse.org/) | +| [Oracle Linux](https://www.oracle.com/linux/) | `oraclelinux-*` | [@Mno-hime](https://github.com/Mno-hime), [sharewax](https://github.com/sharewax), [Djelibeybi](https://github.com/Djelibeybi) | [issues](https://github.com/oracle/oracle-linux/issues) | +| [RHEL](https://www.redhat.com/en/technologies/linux-platforms/enterprise-linux)| `rhel-*` | [Mock team][] | [Issues](https://issues.redhat.com/projects/RHEL) | +| [Rocky Linux](https://rockylinux.org/) | `rocky-*` | [@nazunalika](https://github.com/nazunalika) | [Issues](https://bugs.rockylinux.org/) | + + +[Mock team]: https://github.com/orgs/rpm-software-management/teams/mock-team +[mock-issues]: https://github.com/rpm-software-management/mock/issues +[mock-756]: https://github.com/rpm-software-management/mock/issues/756 +[mock-889]: https://github.com/rpm-software-management/mock/issues/889 diff --git a/docs/Plugin-BindMount.md b/docs/Plugin-BindMount.md new file mode 100644 index 0000000..e48c7e8 --- /dev/null +++ b/docs/Plugin-BindMount.md @@ -0,0 +1,55 @@ +--- +layout: default +title: Plugin BindMount +--- + +This plugin enables setting up bind mountpoints inside the chroot. It is enabled by default but has no paths setup for bind mounts. + + +## Configuration + +In your config file insert the following lines: + + config_opts['plugin_conf']['bind_mount_enable'] = True + config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path', '/bind/mount/path/in/chroot/' )) + +The `/host/path` is the path to a directory on the host that will be the source of a bind-mount, while the `/bind/mount/path/in/chroot` is the path where it will be mounted inside the chroot. + +Starting from version 1.4.10 it will work for files too. Just be aware that files are still put in 'dirs' directive. E.g., + + config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path/file.txt', '/bind/mount/path/in/chroot/file.txt' )) + +If you want the bind mounts to be available to all configurations, edit [the configuration file](Home#generate-custom-config-file). + +### `/builddir/` cleanup + +**WARNING!** The build user's homedir (`/builddir`) is partially cleaned up even when `--no-clean` is +specified in order to prevent garbage from previous builds from altering +successive builds. Mock can be configured to exclude certain files/directories +from this. Default is `SOURCES` directory to support nosrc rpms: + + config_opts['exclude_from_homedir_cleanup'] = ['build/SOURCES'] + +Paths are relative to build user's homedir. + +So if you do something like this: + + config_opts['plugin_conf']['bind_mount_opts']['dirs'].append((os.path.expanduser('~/MyProject'), '/builddir/MyProject' )) + +Then you SHOULD do: + + config_opts['exclude_from_homedir_cleanup'] = ['build/SOURCES', 'MyProject'] + +otherwise your `~/MyProject` will be wiped out! +Other option is to not mount it under `/builddir`, but somewhere else (`/opt`, `/mnt`...). + +*** + +Set options from command line +``` +mock '--plugin-option=bind_mount:dirs=[("/host/dir", "/mount/path/in/chroot/")]' --init +``` + +:warning: Note that command line arguments override configs (not append them). + +:notebook: Since version mock-1.4.10 and newer you can bind mount even single files. diff --git a/docs/Plugin-BuildrootLock.md b/docs/Plugin-BuildrootLock.md new file mode 100644 index 0000000..eb02c94 --- /dev/null +++ b/docs/Plugin-BuildrootLock.md @@ -0,0 +1,48 @@ +--- +layout: default +title: Plugin buildroot_lock +--- + +buildroot_lock Plugin +===================== + +This plugin generates an additional build artifact—the buildroot *lockfile* +(`buildroot_lock.json` file in the result directory). + +The *lockfile* describes both the list of buildroot sources (e.g., a list of +installed RPMs, bootstrap image info, etc.) and a set of Mock configuration +options. Using this information, Mock can later reproduce the buildroot +preparation (see the [Hermetic Builds feature page](feature-hermetic-builds)). + +This plugin is **disabled** by default but is automatically enabled with the +`--calculate-build-dependencies` option. You can enable it (for all builds) by +this configuration snippet: + +```python +config_opts['plugin_conf']['buildroot_lock_enable'] = True +``` + +**Note:** This plugin does not work with the `--offline` option. + + +Format of the *buildroot_lock.json* file +---------------------------------------- + +The file `buildroot_lock.json` is a JSON file. List of JSON Schema files is +installed together with the Mock RPM package: + + rpm -ql mock | grep schema + /usr/share/doc/mock/buildroot-lock-schema-1.0.0.json + /usr/share/doc/mock/buildroot-lock-schema-1.1.0.json + +Currently, we do not provide a compatibility promise. Only the exact same +version of Mock that produced the file is guaranteed to read and process it. +For more information, see [Hermetic Builds](feature-hermetic-builds). + +Also, in the future we plan to switch to a standardized tooling so we operate +with a standardized format, too. For more info see the [DNF5 feature +request][discussion], [rpm-lockfile-prototype][] and [libpkgmanifest][]. + +[discussion]: https://github.com/rpm-software-management/dnf5/issues/833 +[rpm-lockfile-prototype]: https://github.com/konflux-ci/rpm-lockfile-prototype +[libpkgmanifest]: https://github.com/rpm-software-management/libpkgmanifest diff --git a/docs/Plugin-CCache.md b/docs/Plugin-CCache.md new file mode 100644 index 0000000..758b769 --- /dev/null +++ b/docs/Plugin-CCache.md @@ -0,0 +1,46 @@ +--- +layout: default +title: Plugin CCache +--- + +The ccache plugin is a compiler cache plugin. It is disabled by default and has an upper limit of 4GB of ccache data. + +Note: this plugin was enabled by default in mock-1.2.14 and older. + +## Configuration + +The ccache plugin is disabled by default and has the following values built-in: +```python +config_opts['plugin_conf']['ccache_enable'] = False +config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '4G' +config_opts['plugin_conf']['ccache_opts']['compress'] = None +config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/u%(chrootuid)s/" +config_opts['plugin_conf']['ccache_opts']['hashdir'] = True +config_opts['plugin_conf']['ccache_opts']['debug'] = False +config_opts['plugin_conf']['ccache_opts']['show_stats'] = False +``` + +To turn on ccache compression, use the following in a config file: + +```python +config_opts['plugin_conf']['ccache_opts']['compress'] = 'on' +``` + +The value specified is not important, this just triggers the setting of the CCACHE_COMPRESS environment variable, which is what the ccache program uses to determine if compression of cache elements is desired. + +Setting `hashdir` to `False` excludes the build working directory from the hash used to distinguish two +compilations when generating debuginfo. While this allows the compiler cache +to be shared across different package NEVRs, it might cause the debuginfo to be +incorrect. +The option can be used for issue bisecting if running the debugger is +unnecessary. ([issue 1395][]https://github.com/rpm-software-management/mock/issues/1395) +See [ccache documentation](https://ccache.dev/manual/4.10.html#config_hash_dir). +This option is available since Mock 5.7. + +Setting `debug` to `True` creates per-object debug files that are helpful when debugging unexpected cache misses. +See [ccache documentation](https://ccache.dev/manual/4.10.html#config_debug). +This option is available since Mock 5.7. + +If `show_stats` is set to True, Mock calls `ccache --zero-stats` first (before +doing the build), and then calls `ccache --show-stats`. +This option is available since Mock v5.7+. diff --git a/docs/Plugin-ChrootScan.md b/docs/Plugin-ChrootScan.md new file mode 100644 index 0000000..bd51f7e --- /dev/null +++ b/docs/Plugin-ChrootScan.md @@ -0,0 +1,27 @@ +--- +layout: default +title: Plugin Chroot Scan +--- + +The chroot_scan plugin is used to grab files of interest after a build attempt and copy them to the 'result directory' before the chroot is cleaned and data lost. + +## Configuration + +The chroot_scan plugin is disabled by default. To enable it and to add files to the detection logic, add this code to configure file: +```python +config_opts['plugin_conf']['chroot_scan_enable'] = True +config_opts['plugin_conf']['chroot_scan_opts']['regexes'] = [ + "core(\.\d+)?", + "\.log$", +] +config_opts['plugin_conf']['chroot_scan_opts']['only_failed'] = True +config_opts['plugin_conf']['chroot_scan_opts']['write_tar'] = False +``` + +The above logic turns on the chroot_scan plugin and adds corefiles and log files to the scan plugin. When the 'postbuild' hook is run by mock, the chroot_scan will look through the chroot for files that match the regular expressions in it's list and any matching file will be copied to the mock result directory for the config file. Again if you want this to be enabled across all configs, edit the `/etc/mock/site-defaults.cfg` file. + +When `only_failed` is set to False, then those files are always copied. When it is set to True (default when plugin enabled), then those files are only copied when build failed. + +When `write_tar` is set to True, then instead of `chroot_scan` directory, `chroot_scan.tar.gz` is created with the directory archive. + +The `only_failed` option is available since v1.2.8, `write_tar` since v5.5. diff --git a/docs/Plugin-CompressLogs.md b/docs/Plugin-CompressLogs.md new file mode 100644 index 0000000..4384f05 --- /dev/null +++ b/docs/Plugin-CompressLogs.md @@ -0,0 +1,20 @@ +--- +layout: default +title: Plugin Compress Logs +--- + +This plugin compresses logs created by Mock in the result directory (build.log, +hw_info.log, installed_pkgs.log, root.log and state.log). + +This plugin is **disabled** by default. + + +## Configuration + +To compress your logs with XZ, you can put this into the +[configuration](configuration): +```python +config_opts['plugin_conf']['compress_logs_enable'] = True +config_opts['plugin_conf']['compress_logs_opts']['command'] = "/usr/bin/xz -9" +``` +This plugin is available since mock-1.2.1. diff --git a/docs/Plugin-Export-Buildroot-Image.md b/docs/Plugin-Export-Buildroot-Image.md new file mode 100644 index 0000000..50764c7 --- /dev/null +++ b/docs/Plugin-Export-Buildroot-Image.md @@ -0,0 +1,64 @@ +--- +layout: default +title: Plugin export_buildroot_image +--- + +This plugin allows you to (on demand) export the Mock chroot as an OCI image in +local archive format (tarball). This tarball can provide additional convenience +for local build reproducibility. See the example below for details. + +By default, this plugin is **disabled**. You can enable it using the +`--enable-plugin export_buildroot_image` option in `--rebuild` mode. + +This plugin has been added in Mock v6.0. + +## Example use-case + +First, let's start a standard Mock build, but enable the OCI archive generator: + + $ mock -r fedora-rawhide-x86_64 --enable-plugin export_buildroot_image \ + /tmp/quick-package/dummy-pkg-20241212_1114-1.src.rpm + ... mock installs all build-deps, and does other chroot tweaks ... + Start: producing buildroot as OCI image + ... mock performs the rpmbuild ... + INFO: Results and/or logs in: /var/lib/mock/fedora-rawhide-x86_64/result + Finish: run + +The archive has been saved in the result directory: + + $ ls /var/lib/mock/fedora-rawhide-x86_64/result/*.tar + /var/lib/mock/fedora-rawhide-x86_64/result/buildroot-oci.tar + +You may use this tarball together with [the `--buildroot-image` option +then](Feature-buildroot-image). But also, you can try re-running the +build without Mock, like this: + + $ chmod a+r /tmp/quick-package/dummy-pkg-20241212_1114-1.src.rpm + $ podman run --rm -ti \ + -v /tmp/quick-package/dummy-pkg-20241212_1114-1.src.rpm:/dummy-pkg.src.rpm:z \ + oci-archive:/var/lib/mock/fedora-rawhide-x86_64/result/buildroot-oci.tar \ + rpmbuild --rebuild /dummy-pkg.src.rpm + + Installing /dummy-pkg.src.rpm + setting SOURCE_DATE_EPOCH=1401926400 + Executing(%mkbuilddir): /bin/sh -e /var/tmp/rpm-tmp.XIm441 + ... + Executing(%prep): /bin/sh -e /var/tmp/rpm-tmp.pqJ9hu + ... + Executing(%build): /bin/sh -e /var/tmp/rpm-tmp.iaeMZG + ... + Executing(%install): /bin/sh -e /var/tmp/rpm-tmp.SHktaE + ... + Processing files: dummy-pkg-20241212_1114-1.fc42.x86_64 + ... + Executing(%clean): /bin/sh -e /var/tmp/rpm-tmp.E71FWH + ... + + exit 0 + +**Warning:** This method of reproducing a Mock build is not recommended for +production use. During a normal/full Mock rebuild, Mock ensures the buildroot +is fully up-to-date. Using just plain `rpmbuild` within Podman may result in +outdated files, different structure in the kernel-driven filesystems like +`/proc`, `/dev`, and `/sys`, different SELinux assumptions, permissions, etc. +Proceed with caution, and be prepared to encounter some differences (and perhaps +different build failures). diff --git a/docs/Plugin-Hooks.md b/docs/Plugin-Hooks.md new file mode 100644 index 0000000..ffff145 --- /dev/null +++ b/docs/Plugin-Hooks.md @@ -0,0 +1,37 @@ +--- +layout: default +title: Plugin Hooks +--- + +When you develop a new plugin, you can register your hook with: + + plugins.add_hook("HOOK_NAME", self.your_method) + +Where `HOOK_NAME` is one of: + +* clean +* earlyprebuild - This is called during the very early stage of building RPM or SRPM. It is called even before SRPM is rebuilt or build dependencies have been installed. +* initfailed +* list_snapshots - when `--list-snapshots` or `-l` option is passed on command line, this hook is being called and then Mock exits. +* make_snapshot +* mount_root - This hook is intended for plugins which mount the chroot directory. This is called just before 'preinit' when only chroot directory exists. Result directory does not exist yet. +* postbuild - This hook is called just after RPM or SRPM have been build (with or without success). +* postchroot - This hook is called just after exiting from command in `mock chroot` command. This action is currently being used by `root_cache` plugin. +* postclean - This hook is called when the content of the chroot has been deleted. It is called after `umount_root`. E.g., `lvm_root` use this to delete the LVM volume. +* postdeps - Called when the build dependencies (both static and dynamic) are installed, but the build has not started, yet. +* postinit - Chroot have been initialized and it is ready for installing SRPM dependencies. E.g., root_cache plugin now packs the chroot and put it in a cache. This action is currently being used by `bind_mount`, `lvm_root`, `mount`, `root_cache` plugins. +* postshell - This hook is called just after exiting from the shell in `mock shell` command. This action is currently being used by `root_cache` plugin. +* postupdate - Mock attempts to cache buildroot contents, but tries to automatically update the packages inside the buildroot in subsequent executions (so up2date packages are used during build). This hook is called anytime such update is successful and any package is updated inside the buildroot. Any plugin taking care of the buildroot caches can then react and update the cache to avoid re-updating in the subsequent mock runs. +* postumount - This hook is called when everything in chroot finished or has been killed. All inner mounts have been umounted. E.g., tmpfs plugin uses this to umount tmpfs chroot. This action is currently being used by `lvm_root`, `tmpfs` plugins. +* postyum - This hook is called after any packager manager action is executed. This is e.g., used by the `package_state` plugin to list all available packages. This action is currently being used by `package_state`, `root_cache`, `selinux`, `yum_cache` plugins. +* prebuild - This hook is called after `BuildRequires` have been installed, but before RPM builds. This is not called for SRPM rebuilds. +* prechroot - This hook is called just before executing a command when `mock chroot` command has been used. This action is currently being used by `root_cache` plugin. +* preinit - Only chroot and result_dir directories exist. There may be no other content. No binary. Not even base directories. Root_cache plugin uses this hook to populate the content of chroot_from cache. HW_info gather information about hardware and store it in result_dir. This action is currently being used by `ccache`, `hw_info`, `root_cache`, `yum_cache` plugins. +* preshell - This hook is called just before giving a prompt to a user when `mock shell` command has been used. This action is currently being used by `pm_request`, `root_cache` plugins. +* preyum - This hook is called before any packager manager action is executed. This is, e.g., used by `root_cache` plugin when `--cache-alterations` option is used. This action is currently being used by `root_cache`, `selinux`, `yum_cache`. +* process_logs - Called once the build log files (root.log, build.log, ...) are completed so they can be processed (e.g. compressed by the `compress_logs` plugin). +* remove_snapshot +* rollback_to +* scrub + +You can get inspired by existing [plugins](https://github.com/rpm-software-management/mock/tree/master/mock/py/mockbuild/plugins). diff --git a/docs/Plugin-HwInfo.md b/docs/Plugin-HwInfo.md new file mode 100644 index 0000000..377aa4a --- /dev/null +++ b/docs/Plugin-HwInfo.md @@ -0,0 +1,20 @@ +--- +layout: default +title: Plugin HwInfo +--- + +This plugin allows prints your basic hardware information, which may help identify problems or reproduced the build. + +It print information about CPU, memory (including swap) and storage (only of volume where chroot will be stored). +The information is stored in file hw_info.log in results directory. + +## Configuration + +You can enable the plugin using this settings: +```python +config_opts['plugin_conf']['hw_info_enable'] = True +config_opts['plugin_conf']['hw_info_opts'] = {} +``` +Available since version 1.3.4. + +This plugin is enabled by default. diff --git a/docs/Plugin-LvmRoot.md b/docs/Plugin-LvmRoot.md new file mode 100644 index 0000000..3859faf --- /dev/null +++ b/docs/Plugin-LvmRoot.md @@ -0,0 +1,101 @@ +--- +layout: default +title: Plugin LVM Root +--- + +Mock can use LVM as a backend for caching buildroots which is a bit faster than using root_cache and enables efficient snapshotting (copy-on-write). This feature is intended to be used by people who maintain a lot packages and find themselves waiting for mock to install the same set of `BuildRequires` over and over again. +Mock uses LVM thin provisioning which means that one logical volume (called +thinpool) can hold all thin logical volumes and snapshots used by all +buildroots (you have to set it like that in the config) without each of them +having fixed size. Thinpool is created by mock when it's starts initializing +and after the buildroot is initialized, it creates a postinit snapshot which +will be used as default. Default snapshot means that when you execute clean or +start a new build without `--no-clean` option, mock will rollback to the state in default snapshot. As you install more packages you can create your own snapshots (usually for dependency chains that are common to many of your packages). + +## Expected workflow + +You have multiple packages that all depend on something with big dependency chain, for example Java packages depend on maven-local, therefore it may be useful to create snapshot that would contain maven-local. With the next steps, you install the package and create the snapshot + + mock --install maven-local + mock --snapshot mvn + +Now there should be two snapshots - the initial one and the one you just created. You can verify it with the list-snapshots command (Note it also has short option `-l`) + + mock --list-snapshots + + Snapshots for fedora-20-x86_64: + + postinit + * mvn + +The new one is marked with an asterisk, which means it will be used as the default snapshot to which `--clean` will rollback whenever you build another package. When you want to rebuild a package that doesn't use maven-local, you can use + + mock --rollback-to postinit + +and the initial snapshot will be used for following builds. The mvn snapshot will still exist, so you can get back to it later using + + mock --rollback-to mvn + +To get delete a snapshot completely, use + + mock --remove-snapshot mvn + +Mock will leave the buildroot volume mounted by default (you can override it in the config), so you can easily access the build directory. When you need to umount it, use + + mock --umount + +To getting rid of LVM volumes completely, use + + mock --scrub lvm + +This will delete all volumes belonging to the current config and also the thinpool if and only if there are no other volumes left by other configurations. + +## Setup + +The plugin is distributed as separate subpackage `mock-lvm` because it pulls in additional dependencies which are not available on RHEL6. + +You need to specify a volume group which mock will use to create it's thinpool. Therefore you need to have some unoccupied space in your volume group, so you'll probably need to shrink some partition a bit. Mock won't touch anything else in the VG, so don't be afraid to use the VG you have for your system. It won't touch any other logical volumes in the VG that don't belong to it. + +## Configuration + +```python + config_opts['plugin_conf']['root_cache_enable'] = False + config_opts['plugin_conf']['lvm_root_enable'] = True + config_opts['plugin_conf']['lvm_root_opts'] = { + 'volume_group': 'my-volume-group', + 'size': '8G', + 'pool_name': 'mock', + 'check_size': True, + } +``` +Explanation: You need to disable root_cache - having two caches with the same contents would just slow you down. You need to specify a size for the thinpool. It can be shared across all mock buildroots so make sure it's big enough. Ideally there will be just one thinpool. Then specify name for the thinpool - all configs which have the same pool_name will share the thinpool, thus being more space-efficient. Make sure the name doesn't clash with existing volumes in your system (you can list existing volumes with lvs command). + +Every run, the plugin write usage of data pool and metadata pool. +When utilization is over 75 % then plugin emit warning. +Usage over 90 % is fatal and mock stop with error, but it can be override by + + config_opts['plugin_conf']['lvm_root_opts']['check_size'] = False + +However this override is strongly discouraged as LVM2 have known error when thinpool is full. + +### Other configuration options + +`config_opts['plugin_conf']['lvm_root_opts']['poolmetadatasize']` - specifies separate size of the thinpool metadata. It needs to be big enough, thinpool with overflown metadata will become corrupted. When unspecified, the default value is determined by LVM + +`config_opts['plugin_conf']['lvm_root_opts']['umount_root']` - +boolean option specifying whether the buildroot volume should stay mounted after mock exits. Default is `True` + +`config_opts['plugin_conf']['lvm_root_opts']['filesystem']` - +filesystem name that will be used for the volume. It will use +`mkfs.$filesystem` binary to create it. Default is `ext4` + +`config_opts['plugin_conf']['lvm_root_opts']['mkfs_command']` - the whole command for creating the filesystem that will get the volume path as an argument. When set, overrides above option. + +`config_opts['plugin_conf']['lvm_root_opts']['mkfs_args']` - additional arguments passed to mkfs command. Default is empty. + +`config_opts['plugin_conf']['lvm_root_opts']['mount_opts']` - will +be passed to `-o` option of mount when mounting the volume. Default is empty. + +`config_opts['plugin_conf']['lvm_root_opts']['mount_opts']` - Let user configure mock to wait more patiently for LVM initialization. `lvs' call can be quite expensive, especially when there are hundreds of volumes and multiple parallel mocks waiting for common LVM initialization to complete. (Added in version 1.4.3) + +Note: You can not combine Tmpfs plugin and Lvm_root plugin, because it is not possible to mount Logical Volume as tmpfs. diff --git a/docs/Plugin-Mount.md b/docs/Plugin-Mount.md new file mode 100644 index 0000000..d3549e6 --- /dev/null +++ b/docs/Plugin-Mount.md @@ -0,0 +1,50 @@ +--- +layout: default +title: Plugin Mount +--- + +This plugin allows you to mount directories into chroot. The mount plugin is enabled by default, but has no configured directories to mount. + +## Configuration + +You can disable this plugin by: +```python +config_opts['plugin_conf']['mount_enable'] = False +``` + +You can configure this plugin by: +```python +config_opts['plugin_conf']['mount_enable'] = True +config_opts['plugin_conf']['mount_opts']['dirs'].append(("/dev/device", "/mount/path/in/chroot/", "vfstype", "mount_options")) +``` + +A real life example: +```python +config_opts['plugin_conf']['mount_opts']['dirs'].append(("server.example.com:/exports/data", "/mnt/data", "nfs", "rw,hard,intr,nosuid,nodev,noatime,tcp")) +``` + +### `/builddir/` cleanup + +**WARNING!** The build user's homedir (`/builddir`) is partially cleaned up even when `--no-clean` is +specified in order to prevent garbage from previous builds from altering +successive builds. Mock can be configured to exclude certain files/directories +from this. Default is `SOURCES` directory to support nosrc rpms: + +```python +config_opts['exclude_from_homedir_cleanup'] = ['build/SOURCES'] +``` +Paths are relative to build user's homedir. + +So if you do something like this: +```python +config_opts['plugin_conf']['bind_mount_opts']['dirs'].append((os.path.expanduser('~/MyProject'), '/builddir/MyProject' )) +``` + +Then you SHOULD do: +```python +config_opts['exclude_from_homedir_cleanup'] = ['build/SOURCES', 'MyProject'] +``` + +otherwise your `~/MyProject` will be wiped out! +Other option is to not mount it under `/builddir`, but somewhere else (`/opt`, `/mnt`...). + diff --git a/docs/Plugin-Overlayfs.md b/docs/Plugin-Overlayfs.md new file mode 100644 index 0000000..e41889a --- /dev/null +++ b/docs/Plugin-Overlayfs.md @@ -0,0 +1,77 @@ +--- +layout: default +title: Plugin OverlayFS +--- + +This plugin implements mock's snapshot functionality using overlayfs. From a user perspective, it works similar to LVM plugin, but unlike LVM plugin, it only needs a directory (not a volume group) for its data (snapshots). Plugin has no additional dependencies, it only requires a kernel with overlayfs support, but this is the case for both current Fedora and RHEL-7. + +## Configuration +You can enable overlayfs plugin by adding this line to your configuration: +```python +config_opts['plugin_conf']['overlayfs_enable'] = True +``` + +It is recommended to disable root_cache plugin when overlayfs plugin is enabled. (Plugin does implicit snapshot named "postinit" after init phase similarly to LVM plugin, which makes root cache pointless) +```python +config_opts['plugin_conf']['root_cache_enable'] = False +``` +Base directory sets directory, where places all its data (snapshots etc.) are placed. Multiple configurations can share base directory (every configuration will have its own directory there). +```python +config_opts['plugin_conf']['overlayfs_opts']['base_dir'] = "/some/directory" +``` + +Enabling touch_rpmd option causes the plugin to implicitly "touch" rpm database files after each mount overcoming issue with rpm/mock, caused by limitations of overlayfs. Option may be useful only when running yum/rpm directly. However, it is not necessary when using package-manager related mock commands (e.g., mock --install). For more details see the section: Limitations of overlayfs (lower). +Default: false +```python +config_opts['plugin_conf']['overlayfs_opts']['touch_rpmdb'] = True +``` + +## Usage +As said earlier, plugins allow you to use mock's snapshot functionality. Snapshots hold the state of (current config's) root fs, which can be recovered later. + +To create snapshot run: + + mock --snapshot [name] + +You can then return to snapshot created earlier by running (It also makes that snapshot current for clean operation): + + mock --rollback-to [name] + +To list snapshots use: + + mock --list-snapshots + +Clean operation discards changes done "on top" of current snapshot. ( basically restores current snapshot ). As noted earlier, plugin implicitly creates a snapshot after init phase. So, if no user snapshots are done, plugin behaves more or less as root cache: + + mock --clean # restores current snapshot + +To remove all plugin's data associated with configuration (and therefore snapshots), use: + + mock --scrub overlayfs + +Alternatively, you can remove everything from current configuration: + + mock --scrub all + +You can also see more examples of snapshot commands usage in [LVM plugin wiki page](https://rpm-software-management.github.io/mock/Plugin-LvmRoot) +( Difference is, that LVM plugin keeps root mounted, while overlayfs not. ) + +## Shortly about overlayfs filesystem +Overlayfs is pseudo-filesystem in a kernel, which allows to place multiple directories on each other (as overlays) and combine them to a single filesystem. Upper directory and lower directory(ies) are supplied to mount command as the options. Files from lower files are visible, but all writes happen in a upper layer. Deleted files are represented by special files. For more details see [filesystem's documentation](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt) This plugin uses overlayfs to implement snapshots. + +## Notes about the implementation of snapshots +To avoid confusion (in some cases), snapshots as displayed by mock (when using this plugin) should be understood more like references/aliases to actual physical snapshots (internally called LAYERS). This means you can have multiple names for a single physical snapshot (LAYER). This also means, you can see multiple snapshots marked as current when listing snapshots (by mock). This can happen because layers are created lazily, so new LAYER is not allocated immediately after creating a snapshot, but prior to mount. So, for example, when 2 snapshots are done, and no mock action, which requires mount, is done in between, they will point to the same LAYER (expected, no changes have been done; same applies for snapshots created after restoring snapshot and after a clean operation). However, a user can still safely delete just one of these. This is possible because LAYERS are reference counted. So you just remove reference (alias), but LAYER (holding actual snapshot's data) is only deleted when no longer referenced. Apart from user-visible references, it may be referenced by other LAYERS (by ones based on it) and by "current state" (special references)). So, it is safe to remove any "snapshot" (as seen by mock), even current one, without having to worry about "breaking" the mock. If you are interested in even more implementation details, see detailed documentation in plugin's [source file](https://github.com/rpm-software-management/mock/blob/devel/mock/py/mockbuild/plugins/overlayfs.py) or maybe even [test file](https://github.com/rpm-software-management/mock/blob/devel/mock/tests/overlayfs_layers_test.py) (where LAYERS are tested). + +## Limitations of overlayfs +Overlayfs has known limitations when it comes to POSIX compatibility. This may cause problems with some programs. The problem happens, when a file from the lower layer (directory) is open for writing (forcing overlayfs copy it to upper layer), while the same file is opened read-only. Open file descriptors then point to different files. Rpm/yum are known to be affected by this issue. See: + +[yum/rpm bug on RH bugzilla](https://bugzilla.redhat.com/show_bug.cgi?id=1213602) + +[docker overlayfs-driver page](https://docs.docker.com/storage/storagedriver/overlayfs-driver/#limitations-on-overlayfs-compatibility) + +So, this is not a bug in the plugin. It is caused by nature/design decisions made in overlafs filesystem (and documented). Problem is work-arounded automatically for package manager related operations done using mock (e.g. mock --install). When running yum/rpm manually, an option can be used to overcome the issue automatically (see higher). If you find another program(s) affected by this issue, you should be able to work-around this simply by "touching" problematic files(s) prior to running that program. + + touch /some/file # using mock --shell or mock --chroot + +## Using inside docker +Not tested, but you may need to add additional mount where to place this plugin's base_dir. This is because docker may itself use overlayfs. ( Overlayfs cannot use directories which are part of another overlayfs mount. ) diff --git a/docs/Plugin-PMRequest.md b/docs/Plugin-PMRequest.md new file mode 100644 index 0000000..7e849c7 --- /dev/null +++ b/docs/Plugin-PMRequest.md @@ -0,0 +1,28 @@ +--- +layout: default +title: Plugin PM Request +--- + +This plugin listens to requests for package management commands from within the buildroot, using a UNIX socket. It can be used by buildsystems, that have support for communicating with this plugin, to automatically install missing packages if they're available. It's not advised to enable it globally as it affects build reproducibility, instead, it's better to enable it per-build using `--enable-plugin pm_request`. If the plugin was used during the build, it emits a warning at the end that summarizes the commands that were executed. + +Currently, automatic installation of build dependencies using this plugin is supported by Java packaging tooling when building with Maven, Gradle or Ivy. + +## Configuration + +To enable it globally, set the following: +```python +config_opts['plugin_conf']['pm_request_enable'] = True +``` + +To enable it for a single build, use: + + mock --rebuild foo-1.0-1.src.rpm --enable-plugin pm_request + +## The protocol + +The plugin creates a Unix socket in /var/run/mock/pm-request inside of the chroot and will read the commands from the socket. It also exports environment variable PM_REQUEST_SOCKET with value of the socket path. +For single request, it reads a single line from the socket (don't forget the newline, otherwise it will wait for more input) that is parsed using shell-like quoting (via shlex) and passed to current package manager. After the command completes, it responds with a line containing either "ok" or "nok" denoting whether the command was successful. The output of the package management command is then written to the socket and the connection is closed. Example of a request: + + install foo + +Available since mock-1.2.9 diff --git a/docs/Plugin-PackageState.md b/docs/Plugin-PackageState.md new file mode 100644 index 0000000..029e7dd --- /dev/null +++ b/docs/Plugin-PackageState.md @@ -0,0 +1,26 @@ +--- +layout: default +title: Plugin PackageState +--- + +This plugin optionally dumps additional metadata files into the result dir: +* A list of all available pkgs + repos + other data. This file is named `available_pkgs.log`. This file is not created if you run mock with `--offline` option. +* A list of all installed pkgs + repos + other data. This file is name `installed_pkgs.log`. + +Format of `installed_pkgs.log` file is: + + %{nevra} %{buildtime} %{size} %{pkgid} installed + +## Configuration + +The Package_state plugin is enabled by default. + + # in version 1.2.18 and older the default was False + config_opts['plugin_conf']['package_state_enable'] = True + +The following sub-options may be turned off/on: +```python +config_opts['plugin_conf']['package_state_opts'] = {} +config_opts['plugin_conf']['package_state_opts']['available_pkgs'] = False +config_opts['plugin_conf']['package_state_opts']['installed_pkgs'] = True +``` diff --git a/docs/Plugin-ProcEnv.md b/docs/Plugin-ProcEnv.md new file mode 100644 index 0000000..d694d93 --- /dev/null +++ b/docs/Plugin-ProcEnv.md @@ -0,0 +1,22 @@ +--- +layout: default +title: Plugin ProcEnv +--- + +This plugin allows prints your build time environment, which may help identify problems or reproduced the build. + +It prints information about the entire software build environment, any capabilities, and much more. +The information is stored in file procenv.log in the results directory. +It calls `procenv` command and stores its output. + +## Configuration + +You can enable the plugin using this settings: +```python +config_opts['plugin_conf']['procenv_enable'] = True +config_opts['plugin_conf']['procenv_opts'] = {} +``` + +Available since version 1.4.18. + +This plugin is DISABLED by default. diff --git a/docs/Plugin-RootCache.md b/docs/Plugin-RootCache.md new file mode 100644 index 0000000..cf77810 --- /dev/null +++ b/docs/Plugin-RootCache.md @@ -0,0 +1,37 @@ +--- +layout: default +title: Plugin RootCache +--- + +This plugin caches your buildroots. It creates archive of your buildroot and puts it in `config_opts['plugin_conf']['root_cache_opts']['dir']`, which is be default `/var/cache/mock/NAME_OF_CHROOT/root_cache/cache.tar.gz`. It is enabled by default. + + +## Configuration + +This plugin is enabled by default and has the following values built-in: +```python +config_opts['plugin_conf']['root_cache_enable'] = True +config_opts['plugin_conf']['root_cache_opts'] = {} +config_opts['plugin_conf']['root_cache_opts']['age_check'] = True +config_opts['plugin_conf']['root_cache_opts']['max_age_days'] = 15 +config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/root_cache/" +config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "pigz" +config_opts['plugin_conf']['root_cache_opts']['extension'] = ".gz" +config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] = ["./proc", \ + "./sys", "./dev", "./tmp/ccache", "./var/cache/yum" ] +``` + +* `age_check` - if set to `True` (which is default), then cache date is checked. See option `max_age_days` bellow. Additionally if some config is newer than cache file, then the cache is invalidated as well. +* `max_age_days` - if `age_check` is `True` and cache is older than this value, the cache is invalidated. +* `dir` - where to put cached files. +* `compress_program` - which compress program to use. By default `pigz` is used. If not present, then `gzip` is used. +* `extension` - the cache file is always named as `cache.tar$extension`. When you use different compress program e.g. `bzip2`, you should set different extension e.g. `".bz2"`. +* `exclude_dirs` - list of directories, which should not be archived. + +**WARNING:** You should disable `root_cache` plugin when using `lvm_root` plugin - having two caches with the same contents would just slow you down. + +**NOTE:** If you have enough disk storage you can speed-up it a bit by disabling archiving of cache: +```python +config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "" +config_opts['plugin_conf']['root_cache_opts']['extension'] = "" +``` diff --git a/docs/Plugin-SELinux.md b/docs/Plugin-SELinux.md new file mode 100644 index 0000000..5b18f8a --- /dev/null +++ b/docs/Plugin-SELinux.md @@ -0,0 +1,13 @@ +--- +layout: default +title: Plugin SELinux +--- + +On SELinux enabled box, this plugin will pretend, that SELinux is disabled in build environment. + +* fake /proc/filesystems is mounted into build environment, excluding selinuxfs +* option `--setopt=tsflags=nocontext` is appended to each 'yum' command + +This plugin is enabled by default and there is actually no way to disable it. + +This plugin is not used with NSPAWN chroot (`--new-chroot` option) and will be removed when NSPAWN chroot will be only one option. diff --git a/docs/Plugin-Scm.md b/docs/Plugin-Scm.md new file mode 100644 index 0000000..2b65764 --- /dev/null +++ b/docs/Plugin-Scm.md @@ -0,0 +1,97 @@ +--- +layout: default +title: Plugin SCM +--- + +This plugin provides integration to Scm systems (Git, Svn...). + +This module does not use the plugin infrastructure of Mock, it is provided as a standalone package instead, mock-scm, so we dare to call it plugin. + +## Configuration + +In your config file insert the following lines: + +```python +config_opts['scm'] = True +config_opts['scm_opts']['method'] = 'git' +config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN SCM_PKG' +config_opts['scm_opts']['git_get'] = 'git clone --depth 1 SCM_BRN git://localhost/SCM_PKG.git SCM_PKG' +config_opts['scm_opts']['svn_get'] = 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG' +config_opts['scm_opts']['distgit_get'] = 'rpkg clone -a --branch SCM_BRN SCM_PKG SCM_PKG' +config_opts['scm_opts']['distgit_src_get'] = 'rpkg sources' +config_opts['scm_opts']['spec'] = 'SCM_PKG.spec' +config_opts['scm_opts']['int_src_dir'] = None +config_opts['scm_opts']['ext_src_dir'] = '/dev/null' +config_opts['scm_opts']['write_tar'] = True +config_opts['scm_opts']['git_timestamps'] = True +config_opts['scm_opts']['exclude_vcs'] = True +config_opts['scm_opts']['package'] = 'mypkg' +config_opts['scm_opts']['branch'] = 'master' +``` + +While you can specify this in configuration file, this is less flexible and you may rather use command line options. E.g. `config_opts['scm_opts']['method'] = 'git'` is the same as `--scm-option method=git` or `config_opts['scm_opts']['branch'] = 'master'` is the same as `--scm-option branch=master`. + +## Tar file + +When either `write_tar` is set to `True` or `/var/lib/mock//root/builddir/build/SOURCES/` contains `.write_tar`. Mock will create tar file from whole SVN repo. This is what you probably want to. Otherwise you have to manually create the tar file and put it there yourself before you run mock command. + +Extension and compression method is chosen automatically according your Source line in spec file. Therefore if there is: + + Source: http://foo.com/%{name}-%{version}.tar.xz + +then mock will create tar file with .tar.xz extension and compressed by xz. Similarly if you choose .tar.gz or .tar.bz2 or any other known extension. + +### git_timestamps + +When `git_timestamps` is set to True, then modification time of each file in GIT is altered to datetime of last commit relevant to each file. +This option is available only to Git method and not for others. + +### exclude-vcs + +When `exclude-vcs` is set to True, then `--exclude-vcs` option is passed to tar command. + +### int_src_dir + +When `int_src_dir` is specified, Mock will use that directory in the repository as the SOURCES instead of the repository root. + +### ext_src_dir + +When your source (or patch) file does not exist in `/var/lib/mock//root/builddir/build/SOURCES/` directory then it is looked up in `ext_src_dir` and copy there. + +### dist-git + +Since version 1.3.4, there is support for [dist-git](https://github.com/release-engineering/dist-git). In fact it can support any DistSVN or DistCVS method. You just specify which command clone the repository (`distgit_get`) and which command retrieve sources (`distgit_src_get`). Mock-scm will then construct SRPM from those spec file and sources. Do not forget to specify `config_opts['scm_opts']['method'] = 'distgit'` + +## Example + +In this example, mock will clone `master` branch of `github.com/xsuchy/rpmconf.git` and use `./rpmconf.spec` in that directory to build rpm package: +```sh + mock -r fedora-22-x86_64 \ + --scm-enable \ + --scm-option method=git \ + --scm-option package=rpmconf \ + --scm-option spec=rpmconf.spec \ + --scm-option branch=master \ + --scm-option write_tar=True \ + --scm-option git_get='git clone https://github.com/xsuchy/rpmconf.git' +``` + +Or you can: + + cp /etc/mock/fedora-22-x86_64.cfg ./my-config.cfg + vi ./my-config.cfg + +put there those lines: +```python +config_opts['scm'] = True +config_opts['scm_opts']['method'] = 'git' +config_opts['scm_opts']['git_get'] = 'git clone https://github.com/xsuchy/rpmconf.git' +config_opts['scm_opts']['spec'] = 'rpmconf.spec' +config_opts['scm_opts']['write_tar'] = True +config_opts['scm_opts']['package'] = 'rpmconf' +config_opts['scm_opts']['branch'] = 'master' +``` + +and then just call + + mock -r ./my-config.cfg diff --git a/docs/Plugin-Showrc.md b/docs/Plugin-Showrc.md new file mode 100644 index 0000000..36861a7 --- /dev/null +++ b/docs/Plugin-Showrc.md @@ -0,0 +1,19 @@ +--- +layout: default +title: Plugin Showrc +--- + +This plugin dumps all the build time RPM macros (via `rpm --showrc`) into result directory as `showrc.log` file, which may help identify problems of (or reproduce) the build. + +It prints information about all defined RPM macros. + +## Configuration + +You can enable the plugin using this settings: +```python +config_opts['plugin_conf']['showrc_enable'] = True +``` + +Available since version 2.5. + +This plugin is DISABLED by default. diff --git a/docs/Plugin-Sign.md b/docs/Plugin-Sign.md new file mode 100644 index 0000000..9547e2d --- /dev/null +++ b/docs/Plugin-Sign.md @@ -0,0 +1,21 @@ +--- +layout: default +title: Plugin Sign +--- + +The Sign plugin can call command (from the host) on the produced rpms. + +It was primary created for signing packages, but can call anything you want to. + +## Configuration + +The Sign plugin is disabled by default. To enable it, add this code to configure file: +```python +config_opts['plugin_conf']['sign_enable'] = True +config_opts['plugin_conf']['sign_opts'] = {} +config_opts['plugin_conf']['sign_opts']['cmd'] = 'rpmsign' +config_opts['plugin_conf']['sign_opts']['opts'] = '--addsign %(rpms)s' +``` +The variable %(rpms)s will be expanded to package file name. This command will run as unprivileged and will get all your environment variables and especially it will run in your $HOME. So `~/.rpmmacros` will be interpreted. + +Since mock-1.2.14 there is also available variable `%(resultdir)`, which will be expanded to name of directory where are final rpm packages. diff --git a/docs/Plugin-Tmpfs.md b/docs/Plugin-Tmpfs.md new file mode 100644 index 0000000..84a8fc2 --- /dev/null +++ b/docs/Plugin-Tmpfs.md @@ -0,0 +1,25 @@ +--- +layout: default +title: Plugin TmpFS +--- + +The TmpFS plugin allows you to mount a tmpfs on the chroot dir. This plugin is disabled by default. + +## Configuration + +You can enable the plugin using this settings: +```python +config_opts['plugin_conf']['tmpfs_enable'] = True +config_opts['plugin_conf']['tmpfs_opts'] = {} +config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 +config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '768m' +config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' +config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = False +``` + +* `required_ram_mb` - If system has less memory than this value, the TmpFS plugin will be disabled and a warning is omitted, but `mock` will continue. +* `max_fs_size` - this is passed to `mount.tmpfs` as `-o size=X` +* `mode` - this is passed to to `mount.tmpfs` as `-o mode=X` +* `keep_mounted` - when set to `True`, the `buildroot` is not unmounted when mock exits (which will destroy it's content). Additionally when mock is starting and it detects the tmpfs from a previous run, it will reuse it. + +:warning: You can not combine **Tmpfs plugin** and **Lvm_root plugin**, because it is not possible to mount Logical Volume as tmpfs. diff --git a/docs/Plugin-YumCache.md b/docs/Plugin-YumCache.md new file mode 100644 index 0000000..ac637e8 --- /dev/null +++ b/docs/Plugin-YumCache.md @@ -0,0 +1,25 @@ +--- +layout: default +title: Plugin YumCache +--- + +This plugin pre-mounts `/var/cache/{yum,dnf}` directories inside chroot, so the package manager's metadata don't have to be re-downloaded between subsequent mock commands (the caches survive `mock --clean` for example). This plugin is needed because dnf (or yum) `--installroot DIRECTORY` commands store caches below the `DIRECTORY`. + +It mounts directories `/var/cache/mock//{dnf,yum}_cache` as `/var/cache/{dnf,yum}` in chroot. + +You can explicitly clean the package manager caches by `--scrub=dnf-cache` option. + +## Configuration + +This plugin is **enabled by default** and has the following values built-in: +```python +config_opts['plugin_conf']['yum_cache_enable'] = True +config_opts['plugin_conf']['yum_cache_opts'] = {} +config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] = 30 +config_opts['plugin_conf']['yum_cache_opts']['max_metadata_age_days'] = 30 +config_opts['plugin_conf']['yum_cache_opts']['online'] = True +``` + +* `max_age_days` - when files in cache directory is older than this number of days, then such files are removed +* `max_metadata_age_days` - when metadata (everything with suffix: ".sqlite", ".xml", ".bz2", ".gz") in cache directory is older than this number of days, then such files are removed. +* `online` - when `False`, mock doesn't apply policies for `max_*_age_days` options (complements `--offline` option) diff --git a/docs/Plugin-rpkg-preprocessor.md b/docs/Plugin-rpkg-preprocessor.md new file mode 100644 index 0000000..6726890 --- /dev/null +++ b/docs/Plugin-rpkg-preprocessor.md @@ -0,0 +1,51 @@ +--- +layout: default +title: Plugin rpkg preprocessor +--- + +This plugin allows you to run preprocessing on an input spec file just before srpm build is started. + +Preprocessing is implemented by a simple `preproc` language which allows you to place: `{% raw %}{{{ bash_code }}}{% endraw %}` tags into any text file. When you run such text file through `preproc` command-line utility, a "rendered" text file is output where all the `{% raw %}{{{ bash_code }}}{% endraw %}` tags are now replaced by standard output of the executed `bash_code` that was inside the `{% raw %}{{{ }}}{% endraw %}` tags. + +`preproc` also allows you to load a certain library of macros (by `-s` switch on its command-line) which are essentially just bash functions that you can afterward use from any `{% raw %}{{{ }}}{% endraw %}` tag in the input text file. + +One such library is called `rpkg-macros` and its macros are documented [here](https://docs.pagure.org/rpkg-util/v3/macro_reference.html). These macros are specialized to render rpm spec file's dynamically based on surrounding git metadata. They need the spec file you are building srpm from to be placed in a git repository. + +`preproc` with the `rpkg-macros` library loaded is exactly what is used to perform preprocessing on spec file by the `rpkg_preprocessor` plugin. Instead of calling `preproc` directly with the `-s` switch to load `rpkg-macros`, we use a tiny `preproc-rpmspec` wrapper that does this for us but it could be done either way. + +Note that just enabling the plugin is not enough to get spec file preprocessed. You additionally need `rpkg.conf` file placed next to the spec file with the following content: +```ini +[rpkg] +preprocess_spec = True +``` +That's because for some packages you might want to have preprocessing disabled even though the plugin is globally enabled. Note that even if you have rpkg preprocessing globally enabled (e.g. in `/etc/mock/site-defaults.cfg`) and the `rpkg.conf` file is present with the required content to enable preprocessing, you still have an option to disable preprocessing on command-line by using mock's option `--disable-plugin=rpkg_preprocessor`. There is also `--enable-plugin=rpkg_preprocessor` to do the opposite. + +There is currently only one mock command that does invoke the `rpkg_preprocessor` plugin (if all the conditions above are satisfied): `mock --buildsrpm`. This plugin is not employed for rebuilding existing srpms with `mock --rebuild` command or any other mock operation currently. + +An example usage with the `mock --buildsrpm` command is: + + $ cd + $ mock --buildsrpm --spec ./my_package.spec.rpkg --sources . + +This will run preprocessing on `my_package.spec.rpkg` and the output spec file will be then handed over to `rpmbuild` to build srpm from it. + +`.rpkg` extension for rpm spec files containing rpkg macros is optional but recommended. + +## Configuration + +The plugin supports the following configuration options (mentioned together with the default values): +```python +config_opts['plugin_conf']['rpkg_preprocessor_enable'] = False +config_opts['plugin_conf']['rpkg_preprocessor_opts']['requires'] = ['preproc-rpmspec'] +config_opts['plugin_conf']['rpkg_preprocessor_opts']['cmd'] = '/usr/bin/preproc-rpmspec %(source_spec)s --output %(target_spec)s' +``` + +* `rpkg_preprocessor_enable` option switches the plugin on and off +* `requires` option specifies requirements to perform the preprocessing operation. The operation is done in a chroot where srpm is built afterwards so the tool(s) to perform preprocessing need to be installed there first +* `cmd` option defines the actual command to run the preprocessing operation. + +`requires` and `cmd` options are there for possible future updates in the used tooling. You don't need to modify these options to get the default preprocessing functionality. The only line needed in `/etc/mock/site-defaults.cfg` to enable this plugin is: +```python +config_opts['plugin_conf']['rpkg_preprocessor_enable'] = True +``` +This plugin is available since mock-x.x.x. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..3a088ca --- /dev/null +++ b/docs/README.md @@ -0,0 +1,121 @@ +--- +layout: default +title: README +--- + +# The Slate theme + +[![.github/workflows/ci.yaml](https://github.com/pages-themes/slate/actions/workflows/ci.yaml/badge.svg)](https://github.com/pages-themes/slate/actions/workflows/ci.yaml) [![Gem Version](https://badge.fury.io/rb/jekyll-theme-slate.svg)](https://badge.fury.io/rb/jekyll-theme-slate) + +*Slate is a Jekyll theme for GitHub Pages. You can [preview the theme to see what it looks like](http://pages-themes.github.io/slate), or even [use it today](#usage).* + +![Thumbnail of Slate](thumbnail.png) + +## Usage + +To use the Slate theme: + +1. Add the following to your site's `_config.yml`: + + ```yml + remote_theme: pages-themes/slate@v0.2.0 + plugins: + - jekyll-remote-theme # add this line to the plugins list if you already have one + ``` + +2. Optionally, if you'd like to preview your site on your computer, add the following to your site's `Gemfile`: + + ```ruby + gem "github-pages", group: :jekyll_plugins + ``` + +## Customizing + +### Configuration variables + +Slate will respect the following variables, if set in your site's `_config.yml`: + +```yml +title: [The title of your site] +description: [A short description of your site's purpose] +``` + +Additionally, you may choose to set the following optional variables: + +```yml +show_downloads: ["true" or "false" (unquoted) to indicate whether to provide a download URL] +google_analytics: [Your Google Analytics tracking ID] +``` + +### Stylesheet + +If you'd like to add your own custom styles: + +1. Create a file called `/assets/css/style.scss` in your site +2. Add the following content to the top of the file, exactly as shown: + ```scss + --- + --- + + @import "{{ site.theme }}"; + ``` +3. Add any custom CSS (or Sass, including imports) you'd like immediately after the `@import` line + +*Note: If you'd like to change the theme's Sass variables, you must set new values before the `@import` line in your stylesheet.* + +### Layouts + +If you'd like to change the theme's HTML layout: + +1. For some changes such as a custom `favicon`, you can add custom files in your local `_includes` folder. The files [provided with the theme](https://github.com/pages-themes/slate/tree/master/_includes) provide a starting point and are included by the [original layout template](https://github.com/pages-themes/slate/blob/master/_layouts/default.html). +2. For more extensive changes, [copy the original template](https://github.com/pages-themes/slate/blob/master/_layouts/default.html) from the theme's repository
(*Pro-tip: click "raw" to make copying easier*) +3. Create a file called `/_layouts/default.html` in your site +4. Paste the default layout content copied in the first step +5. Customize the layout as you'd like + +### Customizing Google Analytics code + +Google has released several iterations to their Google Analytics code over the years since this theme was first created. If you would like to take advantage of the latest code, paste it into `_includes/head-custom-google-analytics.html` in your Jekyll site. + +### Overriding GitHub-generated URLs + +Templates often rely on URLs supplied by GitHub such as links to your repository or links to download your project. If you'd like to override one or more default URLs: + +1. Look at [the template source](https://github.com/pages-themes/slate/blob/master/_layouts/default.html) to determine the name of the variable. It will be in the form of `{{ site.github.zip_url }}`. +2. Specify the URL that you'd like the template to use in your site's `_config.yml`. For example, if the variable was `site.github.url`, you'd add the following: + ```yml + github: + zip_url: http://example.com/download.zip + another_url: another value + ``` +3. When your site is built, Jekyll will use the URL you specified, rather than the default one provided by GitHub. + +*Note: You must remove the `site.` prefix, and each variable name (after the `github.`) should be indent with two space below `github:`.* + +For more information, see [the Jekyll variables documentation](https://jekyllrb.com/docs/variables/). + +## Roadmap + +See the [open issues](https://github.com/pages-themes/slate/issues) for a list of proposed features (and known issues). + +## Project philosophy + +The Slate theme is intended to make it quick and easy for GitHub Pages users to create their first (or 100th) website. The theme should meet the vast majority of users' needs out of the box, erring on the side of simplicity rather than flexibility, and provide users the opportunity to opt-in to additional complexity if they have specific needs or wish to further customize their experience (such as adding custom CSS or modifying the default layout). It should also look great, but that goes without saying. + +## Contributing + +Interested in contributing to Slate? We'd love your help. Slate is an open source project, built one contribution at a time by users like you. See [the CONTRIBUTING file](docs/CONTRIBUTING.md) for instructions on how to contribute. + +### Previewing the theme locally + +If you'd like to preview the theme locally (for example, in the process of proposing a change): + +1. Clone down the theme's repository (`git clone https://github.com/pages-themes/slate`) +2. `cd` into the theme's directory +3. Run `script/bootstrap` to install the necessary dependencies +4. Run `bundle exec jekyll serve` to start the preview server +5. Visit [`localhost:4000`](http://localhost:4000) in your browser to preview the theme + +### Running tests + +The theme contains a minimal test suite, to ensure a site with the theme would build successfully. To run the tests, simply run `script/cibuild`. You'll need to run `script/bootstrap` once before the test script will work. diff --git a/docs/Release-Notes-1.2.13.md b/docs/Release-Notes-1.2.13.md new file mode 100644 index 0000000..4060dec --- /dev/null +++ b/docs/Release-Notes-1.2.13.md @@ -0,0 +1,17 @@ +--- +layout: default +title: Release Notes 1.2.13 +--- + +mock-1.2.13 is bugfix release, but some bugfix may be interesting for you: + +* Fedora 23 configs are reverted back to use yum again. To be on pair +with Koji +* Lot of fixes for --new-chroot option +* Mockchain can download SRPM from Dropbox +* DNF does not install weak dependencies by default +* When cleaning up chroots, mock now exclude mountpoints +* When you build using DNF (rawhide) on systems, which does not have DNF (EL6, 7), mock will print warning, wait for confirmation, tell you how to suppress this warning next time. Nevertheless this warning is not fatal and Mock can continue using YUM. +* Previously package_state plugin always used YUM, now it use DNF when chroot is configured to use DNF. +* When file `/usr/bin/yum-deprecated` is present on your machine, then variable `config_opts['yum_command']` is set to this value by default. +* Several others bugfixes diff --git a/docs/Release-Notes-1.2.14.md b/docs/Release-Notes-1.2.14.md new file mode 100644 index 0000000..b4bc82f --- /dev/null +++ b/docs/Release-Notes-1.2.14.md @@ -0,0 +1,12 @@ +--- +layout: default +title: Release Notes 1.2.14 +--- + +mock-1.2.14 is bugfix release, but some bugfix may be interesting for you: + +* --new-chroot now works on rhel7 +* create tmpfs with unlimited inodes [RHBZ#1266453](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1266453) - otherwise architectures with large pages (e.g. PPC64) can use only fraction of the tmpfs capacity. +* Add %(resultdir) placeholder for sign plugin. [RHBZ#1272123](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1272123) +* use --setopt=deltarpm=false as default value for dnf_common_opts [RHBZ#1281355](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1281355) +* fixed issue with /home mount on nfs [RHBZ#1281369](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1281369) diff --git a/docs/Release-Notes-1.2.15.md b/docs/Release-Notes-1.2.15.md new file mode 100644 index 0000000..94c530f --- /dev/null +++ b/docs/Release-Notes-1.2.15.md @@ -0,0 +1,14 @@ +--- +layout: default +title: Release Notes 1.2.15 +--- + +mock-1.2.15 introduce these changes: + +* Fedora 24 chroot configs (and Fedora 21 was removed). +* ccache plugin is by default off - to copy behaviour of Koji and Copr. +* ~/.config/mock.cfg is parsed too (beside ~/.mock/user.cfg). + +And several bugfixes: +* buildroot is removed as root [RHBZ#1294979](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1294979) +* "local" dnf plugin is disable [RHBZ#1264215](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1264215) diff --git a/docs/Release-Notes-1.2.16.md b/docs/Release-Notes-1.2.16.md new file mode 100644 index 0000000..067f0af --- /dev/null +++ b/docs/Release-Notes-1.2.16.md @@ -0,0 +1,19 @@ +--- +layout: default +title: Release Notes 1.2.16 +--- + +mock-1.2.16 has been released to get rid of annoying errors due selinux [RHBZ#1312820](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1312820). + +Additionally mock-1.2.16 introduces these changes: + +* sparc configs has been removed +* instead of systemd-nspawn mock now requires systemd-container in F24+ + +And several bugfixes: +* do not call /bin/su and rather utilize --user of systemd-nspawn [RHBZ#1301953](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1301953) +* tell nspawn which variables it should set [RHBZ#1311796](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1311796) + +Known issue: + +* When you run mockchain with several SRPMs, then it may fails due deluser bug [RHBZ#1315864](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1315864) diff --git a/docs/Release-Notes-1.2.17.md b/docs/Release-Notes-1.2.17.md new file mode 100644 index 0000000..9347ae4 --- /dev/null +++ b/docs/Release-Notes-1.2.17.md @@ -0,0 +1,6 @@ +--- +layout: default +title: Release Notes 1.2.17 +--- + +mock-1.2.17 is just quick release, which is fixing major issue in 1.2.16 and which was not catch during release testing. diff --git a/docs/Release-Notes-1.2.18.md b/docs/Release-Notes-1.2.18.md new file mode 100644 index 0000000..0471a9a --- /dev/null +++ b/docs/Release-Notes-1.2.18.md @@ -0,0 +1,47 @@ +--- +layout: default +title: Release Notes 1.2.18 +--- + +mock-1.2.18 has several bugfixes: + +* copy just content of SRPM not the attributes ([RHBZ#1301985](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1301985)) +* do not fail when we cannot link default.cfg ([RHBZ#1305367](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1305367)) +* Build always fails when using --nocheck ([RHBZ#1327594](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1327594)) +* keep machine-id in /etc/machine-id ([RHBZ#1344305](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1344305)) + +And several changes: +* Unconditionally setup resolver config +* use DNF for F24 chroot +* requires rpm-python +* Escape the escape sequences in PROMPT_COMMAND, improve prompt +* Use root name instead config name for backups dir +* Add MIPS personalities +* scm plugin handle better submodules + +And there are two new groups of configs. There are new [Mageia](https://www.mageia.org) configs: +* mageia-cauldron-armv5tl +* mageia-cauldron-armv7hl +* mageia-cauldron-i586 +* mageia-cauldron-x86_64 +* mageia-6-armv5tl +* mageia-6-armv7hl +* mageia-6-i586 +* mageia-6-x86_64 + +And there are new custom configs: +* custom-1-aarch64 +* custom-1-armhfp +* custom-1-i386 +* custom-1-ppc64 +* custom-1-ppc64le +* custom-1-s390 +* custom-1-s390x +* custom-1-x86_64 + +Those configs does not have any repository configured and base is empty. I.e: + + config_opts['chroot_setup_cmd'] = "" + + +This is useful if you want to prepare the chroot yourself. Or when you use it with mockchain with `--addrepo=REPOS` option. This was added on request of [Koschei](https://fedoraproject.org/wiki/Koschei), which will use it. diff --git a/docs/Release-Notes-1.2.19.md b/docs/Release-Notes-1.2.19.md new file mode 100644 index 0000000..713ab70 --- /dev/null +++ b/docs/Release-Notes-1.2.19.md @@ -0,0 +1,19 @@ +--- +layout: default +title: Release Notes 1.2.19 +--- + +Mock version 1.2.19 has those changes: + +* plugin [PackageState](Plugin/PackageState) is now enabled by default and it has new options. By default this plugin now generate list of installed packages. List of available packages is disabled by default. +* Fedora 25 configs has been added +* GPG keys in configs are now used from package `distribution-gpg-keys`. Keys in `/etc/pki/mock` will be still shipped for some time, so we do not break old user config. But new one will not be added and users are encouraged to migrate their paths to GPG keys. +* you can include some other config using: + ``include('/path/to/config/to/be/included/include.cfg')`` +* there is new option available which will install additional package to minimal chroot. This is extension of already existing option `chroot_setup_cmd`. It was added to easy automated changed of minimal buildroot in Copr.: + ``config_opts['chroot_additional_packages'] = 'some_package other_package'`` +* And it resolves those bugs: RHBZ#1272381, RHBZ#1358397, RHBZ#1362478, RHBZ#1277187, RHBZ#1298220, RHBZ#1264508. + +And few notes about future release: +* in next release will be very likely resolved [RHBZ#1246810](https://bugzilla.redhat.com/show_bug.cgi?id=1246810). I.e. /usr/sbin/mock will be moved to /usr/libexec/mock/mock. Since this is very big change, next release will likely be 1.3.0 +* Development and documentation will likely move to Github or Pagure. Please follow buildsys mailing list. diff --git a/docs/Release-Notes-1.2.20.md b/docs/Release-Notes-1.2.20.md new file mode 100644 index 0000000..e70a819 --- /dev/null +++ b/docs/Release-Notes-1.2.20.md @@ -0,0 +1,6 @@ +--- +layout: default +title: Release Notes 1.2.20 +--- + +Mock 1.2.20 is just bugfix release, which use correct gpg keys for epel in epel* configs. diff --git a/docs/Release-Notes-1.2.21.md b/docs/Release-Notes-1.2.21.md new file mode 100644 index 0000000..ef5b721 --- /dev/null +++ b/docs/Release-Notes-1.2.21.md @@ -0,0 +1,13 @@ +--- +layout: default +title: Release Notes 1.2.21 +--- + +Mock version 1.2.21 is security release. It fixes: + +* CVE-2016-6299 - privilige escalation via mock-scm [RHBZ#1375493](https://bugzilla.redhat.com/show_bug.cgi?id=1375493) + +Additionally it has those changes: +- root_cache: Mention _root_ cache being created in state updates (log messages) +- Rename mageia pubkey to RPM-GPG-KEY-Mageia +- require generic system-release rather than fedora-release [RHBZ#1367746](https://bugzilla.redhat.com/show_bug.cgi?id=1367746) diff --git a/docs/Release-Notes-1.3.2.md b/docs/Release-Notes-1.3.2.md new file mode 100644 index 0000000..f16fd45 --- /dev/null +++ b/docs/Release-Notes-1.3.2.md @@ -0,0 +1,33 @@ +--- +layout: default +title: Release Notes 1.3.2 +--- + +This is a release with big changes. Prior to this release there have been version 1.3.1, but it was never actually released. It was just tagged commit and was intended just for developers for testing. This is first public release after big internal changes. + +There are those changes: + +* move `/usr/sbin/mock` to `/usr/libexec/mock/mock` [RHBZ#1246810](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1246810) + Previously there were /usr/bin/mock and /usr/sbin/mock. It caused some confusion. Script `/usr/sbin/mock` should not be run directly, but some people (and containers) had /usr/sbin first in their path. So this script is now in `/usr/libexec/`, which are not in `$PATH`. In other word: you can still type "mock" and mock will start. If you ever seen this error +`ERROR: The most common cause for this error is trying to run `/usr/sbin/mock` as an unprivileged user.` then you should not see it anymore. +* F22 configs has been removed +* Just for developers: I removed the automake and it will use Tito now. If you are new to tito, then just: +```bash + sudo dnf install tito + tito build --rpm --test -i # install code from last commit + tito build --rpm # build latest tagged version +``` + The benefits are that release process is now *much* easier. And there are no more build artefacts directly in git repo. +* `--nocheck` is working again [GH#2](https://github.com/rpm-software-management/mock/issues/2) +* You can now run mock inside of Docker [RHBZ#1336750](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1336750) - however you need to run docker with `-cap-add=SYS_ADMIN`. +* When building for Fedora 25+ target in container, then buildhost is set to name of host and not to name of container. [RHBZ#1302040](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1302040) + These are the new defaults: +``` +## When RPM is build in container then build hostname is set to name of +## container. This sets the build hostname to name of container's host. +## Works only in F25+ chroots +# config_opts['use_container_host_hostname'] = True +## This works in F25+ chroots. This overrides 'use_container_host_hostname' option +# config_opts['macros']['%_buildhost'] = 'my.own.hostname' +``` +* There was a lot of flake8/pep8/pycodestyle clean up of code. diff --git a/docs/Release-Notes-1.3.3.md b/docs/Release-Notes-1.3.3.md new file mode 100644 index 0000000..ae2a946 --- /dev/null +++ b/docs/Release-Notes-1.3.3.md @@ -0,0 +1,32 @@ +--- +layout: default +title: Release Notes 1.3.3 +--- + +There are new features: + +* All chroot (but rawhide) configs now contains `best=1`. This way DNF will always try to install latest package. + If its dependence cannot be satisfied DNF will report an error. Without this DNF may silently install some older + version which does not have broken deps. + This is fine for regular user, but not for buildsystems, where maintainers usually want to + use latest version. + Note that this change may result in sudden build failure, which previously silently succedded. + In this case, please check your BuildRequires and ask maintainers of those build deps to resolve broken dependency. + This option was not added to rawhide chroots as there are broken dependencied very often. + Additionaly option `best=1` is used for repos passed to mockchain using `-a` option. +* new config for epel-7-aarch64 chroot +* You can use new variable `hostname`: `config_opts['hostname'] = 'my.own.hostname'` + This unconditionally calls `sethostname()`, however + variable `use_container_host_hostname` or `%_buildhost` macro can override this (on F25+). +* Use DNF on RHEL, when it is installed and configured [RHBZ#1405783](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1405783) +* Temporary directories now use `tmp.mock.` prefix. +* Temporary directories are now removed even when buildroot is not cleaned. +* Add bash completion for .cfg files outside /etc/mock [#20](https://github.com/rpm-software-management/mock/pull/20) + +There are some bugfixes: + +* Handle working directories which contains spaces [RHBZ#1389663](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1389663) +* Error: is not iterable [RHBZ#1387895](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1387895) +* Delay mounting of user-defined mountpoints [RHBZ#1386544](http://bugzilla.redhat.com/bugzilla/show_bug.cgi?id=1386544) +* Added example how to use `more_buildreqs` when you need more packages +* Added example how to use `--plugin-option` diff --git a/docs/Release-Notes-1.3.4.md b/docs/Release-Notes-1.3.4.md new file mode 100644 index 0000000..ba31068 --- /dev/null +++ b/docs/Release-Notes-1.3.4.md @@ -0,0 +1,29 @@ +--- +layout: default +title: Release Notes 1.3.4 +--- + +There are new features: + +* [scm plugin](Plugin-Scm) now supports [DistGit](https://github.com/release-engineering/dist-git/) (or DistSVN etc.) . +* log files of [package_state plugin](Plugin-PackageState) got `.log` extension. I.e. `available_pkgs.log` and `installed_pkgs.log`. +* Configuration files for Fedora 26 has been added. +* Configuration files for Fedora 23 has been removed. +* Even rawhide configs now use best=1. +* when Mock is run directly without consolehelper it now return exit code 2. +* You can pass additional arguments to systemd-nspawn using this config option: `config_opts['nspawn_args'] = []`. +* kojipkgs urls now use https instead of http. +* new plugin [hw_info](Plugin-HwInfo) which prints HW information of builder. This plugin is enabled by default. + +There are some bugfixes: + +* reflect change of "Rawhide" to "rawhide" in /etc/os-release [RHBZ#1409735](https://bugzilla.redhat.com/show_bug.cgi?id=1409735) +* in site-defaults.cfg is more examples of how to set up PS1 [RHBZ#1183733](https://bugzilla.redhat.com/show_bug.cgi?id=1183733) +* preserve mode of files when doing chroot_scan [RHBZ#1297430](https://bugzilla.redhat.com/show_bug.cgi?id=1297430) +* shell in systemd-nspawn is run as PID 2 [RHBZ#1372234](https://bugzilla.redhat.com/show_bug.cgi?id=1372234) - this is not done in EL7 version of systemd-nspawn does not support it +* debuginfo repos has been renamed so `mock --dnf-cmd debuginfo-install PACKAGE` works now [RHBZ#1409734](https://bugzilla.redhat.com/show_bug.cgi?id=1409734) + +Notes: + +* next version will have systemd-nspawn as default. This can break your scripts built on top of Mock. You can try the new behaviour using `--new-chroot` option. +* next version will not be released for EL6. You are advised to upgrade to EL7. diff --git a/docs/Release-Notes-1.3.5.md b/docs/Release-Notes-1.3.5.md new file mode 100644 index 0000000..ac2bf69 --- /dev/null +++ b/docs/Release-Notes-1.3.5.md @@ -0,0 +1,10 @@ +--- +layout: default +title: Release Notes 1.3.5 +--- + +This is bugfix release only for EL6: + +* Change path to "df" in hw-info plugin [RHBZ#1428301](https://bugzilla.redhat.com/show_bug.cgi?id=1428301) + +EL7 and Fedoras are not affected, therefore I did not release this version there. diff --git a/docs/Release-Notes-1.4.1.md b/docs/Release-Notes-1.4.1.md new file mode 100644 index 0000000..453ef32 --- /dev/null +++ b/docs/Release-Notes-1.4.1.md @@ -0,0 +1,47 @@ +--- +layout: default +title: Release Notes 1.4.1 +--- + +There are new features: + +* Mock previously used chroot technology. Few past releases Mock offered systemd-nspawn which is modern container technology for better isolation. This release use systemd-nspawn as default. If you want to preserve previous behaviour you can use `--old-chroot` option. *NOTE*: network is disabled inside container by default now; take a look into `site-defaults.cfg` for desired options (like `rpmbuild_networking`). +* Mock now uses bootstrap chroot to install target chroot. This is big change and see special paragraph at the bottom of this release notes. +* Chroot now contains `/dev/hwrng` and `/dev/prandom` when they exists in host [[#33](https://github.com/rpm-software-management/mock/issues/33)]. +* We added `%distro_section` macro to Mageia configs. + +There are some bugfixes: + +* Resultdir is now chowned to user who executed mock so they can delete the files. +* `hw_info` plugin chown logs to user who executed mock also. +* Previously we declared that *package state plugin* is enabled by default, but the plugin was in fact disabled. It is now enabled by default (as stated in mock documentation) [[RHBZ#1277187](https://bugzilla.redhat.com/show_bug.cgi?id=1277187)]. +* Creating directories for mount points have been delayed after mount of tmpfs [[#57](https://github.com/rpm-software-management/mock/issues/57)]. +* Exit code of `machinectl` is now ignored as `machinectl` set non-zero code even for non-fatal errors. Errors which are quite often not relevant nor important for mock. +* `hw_info` plugin does not crash when output contains non-ASCII characters [[#68](https://github.com/rpm-software-management/mock/issues/68)]. + + +Notes: +* This version has not been released for EL6. If you are using EL6 and you want to use latest Mock, please upgrade you infrastructure to EL7. +* Configs for `s390` architecture has been removed as it is not supported any more. +* Configs for `aarch64` and `ppc64le` now use different GPG key as those architectures has been moved from Secondary to Primary. +* Epel5 config points now to vault.centos.org. Note that EL5 has been EOLed. We will keep epel-5 config for some time. But any issue with building for epel-5 target will not be fixed. + +## Bootstrap chroot + +Mock is calling `dnf --installroot` to install packages for target architecture into target directory. This works. Mostly. The only problem that use host DNF and rpm to install packages. But this can cause problem when new RPM feature is introduces. Like Soft dependencies or Rich dependencies. When you have EL6 host and try to install Fedora rawhide package with Rich dependency then rpm will fail and you cannot do anything about it. You can upgrade your build machine to Fedora rawhide, but that is often not possible when it is part of critical infrastructure. + +So we introduced Boostrap chroot. And 'we' actually means Michael Cullen who implement it. And Igor Gnatenko who proposed this idea. Big kudos for both of them. + +Bootstrap chroot means that we first create very minimal chroot for target platform and we call DNF/YUM from that platform. For example: when you are on RHEL7 and you want to build package for `fedora-26-x86_64`, mock will first create chroot called `fedora-26-x86_64-bootstrap`, it will install DNF and rpm there (fc26 versions). Then it will call DNF from `fedora-26-x86_64-bootstrap` to install all needed packages to `fedora-26-x86_64` chroot. + +The disadvantage is that you will need more storage in `/var/lib/mock`, the build is little bit slower. But you will hardly notice that unless you disabled `yum_cache` and `root_cache` plugins for some reasons. + +The advantage is that you can use stable version of OS to build packages for even most recent OS. And vice versa. + +If you want to preserve previous behaviour you can use `--no-bootstrap-chroot` command line option or set: + +``` + config_opts['use_bootstrap_container'] = False +``` + +in your configuration. diff --git a/docs/Release-Notes-1.4.10.md b/docs/Release-Notes-1.4.10.md new file mode 100644 index 0000000..f2f7772 --- /dev/null +++ b/docs/Release-Notes-1.4.10.md @@ -0,0 +1,51 @@ +--- +layout: default +title: Release Notes 1.4.10 +--- + +Released on 2018-05-10. + +Features: +- There is a new plugin [overlayfs](Plugin-Overlayfs). This plugin implements mock's snapshot functionality using overlayfs. From a user perspective, it works similar to LVM plugin, but unlike LVM plugin, it only needs a directory (not a volume group) for its data (snapshots). +- Previously a [bind_mount](Plugin-BindMount) plugin allowed to mount just a directory. Now, you can bind mount even a single file. +- Previously a [chroot_scan](Plugin-ChrootScan) allowed retrieving artifacts from chroot where build started. Now, it can extract objects even from chroot which failed to initialize. + +Bugfixes: +- Change sign plugin to sign only built RPMs and not every file in results directory [RHBZ#1217495](https://bugzilla.redhat.com/show_bug.cgi?id=1217495) +- encode content before writing [GH#176](https://github.com/rpm-software-management/mock/issues/176) +- revert workaround introduced in 057c51d6 [RHBZ#1544801](https://bugzilla.redhat.com/show_bug.cgi?id=1544801) + +Note: + +Few weeks ago, I released a new `mock-core-configs` package and there is a new feature. + +It contains: + +``` +$ ls -l /etc/mock +... +lrwxrwxrwx. 1 root mock 26 May 2 09:13 fedora-29-aarch64.cfg -> fedora-rawhide-aarch64.cfg +lrwxrwxrwx. 1 root mock 25 May 2 09:13 fedora-29-armhfp.cfg -> fedora-rawhide-armhfp.cfg +lrwxrwxrwx. 1 root mock 23 May 2 09:13 fedora-29-i386.cfg -> fedora-rawhide-i386.cfg +lrwxrwxrwx. 1 root mock 24 May 2 09:13 fedora-29-ppc64.cfg -> fedora-rawhide-ppc64.cfg +lrwxrwxrwx. 1 root mock 26 May 2 09:13 fedora-29-ppc64le.cfg -> fedora-rawhide-ppc64le.cfg +lrwxrwxrwx. 1 root mock 24 May 2 09:13 fedora-29-s390x.cfg -> fedora-rawhide-s390x.cfg +lrwxrwxrwx. 1 root mock 25 May 2 09:13 fedora-29-x86_64.cfg -> fedora-rawhide-x86_64.cfg +``` + +The plan is that during Fedora branching event I will: + +* remove those symlinks and create regular files pointing to Fedora 29 repos +* create fedora-30-* as symlinks to fedora-rawhide-* + +This will give you a choice to target rawhide using "rawhide" string or using the number. + +Following contributors contributed to this release: + +* Martin Necas +* Michal Novotný +* Neal Gompa +* Todd Zullinger +* Zdenek Zambersky + +Thank you. diff --git a/docs/Release-Notes-1.4.11.md b/docs/Release-Notes-1.4.11.md new file mode 100644 index 0000000..562630b --- /dev/null +++ b/docs/Release-Notes-1.4.11.md @@ -0,0 +1,86 @@ +--- +layout: default +title: Release Notes 1.4.11 +--- + +Released on 2018-06-12. + +## Features: + +- Previously you were able to only build for compatible architectures. I.e., you can build `i386` package on `x86_64` architecture. When you tried to build for incompatible architecture, you get this error: + +``` +$ mock -r fedora-28-ppc64le shell +ERROR: Cannot build target ppc64le on arch x86_64, because it is not listed in legal_host_arches ('ppc64le',) +``` + +Now, you can build for any architecture using new option --force-arch ARCH. [GH#120](https://github.com/rpm-software-management/mock/issues/120) You have to have installed package `qemu-user-static`, which is a new soft dependence. Try this: + +``` +$ sudo dnf install qemu-user-static +$ mock -r fedora-28-ppc64le --forcearch ppc64le shell +``` + +and you get the prompt in PPC64LE Fedora. You can do this for any architecture supported by QEMU. Note: Do not confuse `--forcearch` and `--arch` which are very different options. + +- Mock previously only supported GNU Tar. Now Mock supports BSD Tar as well. [GH#169](https://github.com/rpm-software-management/mock/issues/169) There is a new option in config available: + +``` +# You can configure which tar is used (for root cache and SCM plugin) +# valid options are: "gnutar" or "bsdtar" +# config_opts['tar'] = "gnutar" +``` + +Be aware that if you created a cache using gnutar then you cannot extract it using bsdtar. Therefore when changing this option, you have to scrub all caches. + + +- There is a new config option: + +``` +# name of user that is used when executing commands inside the chroot +# config_opts['chrootuser'] = 'mockbuild' +``` + +This can be changed to different value. E.g., 'root'. However, be aware that any other value than 'mockbuild' is not tested by upstream. + +- There is initial support for MicroDnf [GH#76](https://github.com/rpm-software-management/mock/issues/76) Be aware the due MicroDNF is missing `--installroot` option, the buildroot and build dependencies are still installed by DNF. These are new options related to MicroDNF: + +``` +# config_opts['microdnf_command'] = '/usr/bin/microdnf' +## "dnf-install" is special keyword which tells mock to use install but with DNF +# config_opts['microdnf_install_command'] = 'dnf-install microdnf dnf dnf-plugins-core distribution-gpg-keys' +# config_opts['microdnf_builddep_command'] = '/usr/bin/dnf' +# config_opts['microdnf_builddep_opts'] = [] +# config_opts['microdnf_common_opts'] = [] +# config_opts['microdnf_command'] = '/usr/bin/microdnf' +config_opts['package_manager'] = 'microdnf' +``` + +Right now, Mock does not ship any config which use this package manager. + +- There is a new option `--spec`, which you can use to build an SRPM. + +``` +# dnf download --source foo +# rpm2cpio foo.src.rpm | cpio -dimv +(Add/modify compilation flags in .spec file) +# mock --spec foo.spec foo.src.rpm --postinstall +(or) +# mock --spec foo.spec --sources ./ +``` + + +## Bugfixes: + +- The file `/etc/resolv.conf` is now empty when networking is disabled. [RHBZ#1514028](https://bugzilla.redhat.com/show_bug.cgi?id=1514028) This reduces timeout when code tries to connect to a network. Note that option `config_opts['use_host_resolv']` changed from True to False as networking is disabled by default too. Note that `--enable-network` automatically set `config_opts['use_host_resolv']` to True now. + +Following contributors contributed to this release: + +* ArrayMy +* Ken Dreyer +* Neal Gompa +* Neil Horman +* Sam Fowler +* Todd Zullinger + +Thank you. diff --git a/docs/Release-Notes-1.4.13.md b/docs/Release-Notes-1.4.13.md new file mode 100644 index 0000000..33abc66 --- /dev/null +++ b/docs/Release-Notes-1.4.13.md @@ -0,0 +1,52 @@ +--- +layout: default +title: Release Notes 1.4.13 +--- + +Released on 2018-08-13. + +## Features: + +- Starting with mock-core-configs version 29.1 the gpg keys for rawhide are checked now. + +- There is a new config option `print_main_output`, which allows you to override default behavior: + +``` + # By default, mock only prints the build log to stderr if it is a tty; you can + # force it on here (for CI builds where there is no tty, for example) by + # setting this to True, or force it off by setting it to False. + # config_opts['print_main_output'] = None +``` + +- Following new environment variables are passed to mock from user environment: `http_proxy`, `ftp_proxy`, `https_proxy`, `no_proxy`. + +- bash completion has been reworked and is now much simple and hopefully better + +- There are new configs for Fedora 30. + +## Bugfixes: + +- Mockchain will again stop after the first failure if -c or --recurse is not used. + +- Commands started by mock will be using `C.UTF-8` locale instead of `en_US.UTF-8`, which does not need to be available. + +- There is new default for `nspawn_args`: `config_opts['nspawn_args'] = ['--capability=cap_ipc_lock']`. This will enable cap_ipc_lock in nspawn container, which will allow to use `mlock()` [RHBZ#1580435](https://bugzilla.redhat.com/show_bug.cgi?id=1580435). + +- Do not get spec from the command line when using scm [GH#203](https://github.com/rpm-software-management/mock/issues/203) + +- use host's resolv.conf when --enable-network is set on cml [RHBZ#1593212](https://bugzilla.redhat.com/show_bug.cgi?id=1593212) + +Following contributors contributed to this release: + +* Bruno Vernay +* Chuck Wilson +* Jaroslav Škarvada +* Neal Gompa +* Owen W. Taylor +* Seth Wright +* Todd Zullinger +* Tomasz Torcz + +Thank you. + +Note: version 1.4.12 has been skipped due error discovered during releasing. diff --git a/docs/Release-Notes-1.4.14.md b/docs/Release-Notes-1.4.14.md new file mode 100644 index 0000000..540b7db --- /dev/null +++ b/docs/Release-Notes-1.4.14.md @@ -0,0 +1,79 @@ +--- +layout: default +title: Release Notes 1.4.14 +--- + +Released on 2019-02-19. + +Release together with `mock-core-configs-30.1` which has these changes: + +- Added repositories for Fedora 30 (and Fedora 31 repos now points to rawhide). + +- distribution-gpg-keys for rhel8beta is being installed directly from Koji, because EPEL8 does not exist yet. + +- Fedora 27 config has been moved to `eol` directory. + +- `gpgcheck` is enabled for testing and debuginfo now. + +- Fedoras 29+ have included modular repos now. Additionally, there is now `module_platform_id` defined in these configs, which allows you to install modules without errors. + +## Mock new features: + +- All mock configs are parsed and evaluated by [Jinja2](http://jinja.pocoo.org/). Here is small example how it can be used: + +``` +# define your own config variable +config_opts['fedora_number'] = '30' +config_opts['root'] = 'fedora-{{ fedora_number }}-x86_64' +config_opts['dist'] = 'fc{{ fedora_number }}' +``` + +Another - more general - example from `site-defaults.cfg`: + +``` +# You can use jinja templates, e.g.: +# config_opts['foobar'] = '{{ foo }} bar' +# which will result in 'bar bar' (using value defined few lines above) +# more complicated example: +# config_opts['foo'] = "{{ plugin_conf['package_state_enable'] }}" +# which will result in "True" +``` + +This feature can simplify mock's configs in the future. I intentionally did not use it now, because it is too fresh. Please experiment with this feature on your own and report any error or issues. If there would be none, then I will start using it in main configs. + +- Use 32-bit personality for armv7*/armv8* builds. + +- You can now specify decompress program for root_cache. This is new default in `site-defaults.cfg` [GH#230](https://github.com/rpm-software-management/mock/issues/230): + +``` +## decompress_program is needed only for bsdtar, otherwise `compress_program` with `-d` is used +## for bsdtar use "unpigz" or "gunzip" +# config_opts['plugin_conf']['root_cache_opts']['decompress_program'] = "pigz" +``` + + +## Bugfixes: + +- Added Scientific Linux on the list of RHEL clones [GH#228](https://github.com/rpm-software-management/mock/issues/228) + +- Fixed exclude pattern for BSDTar [GH#219](https://github.com/rpm-software-management/mock/issues/219) + +- There used to be living part of `site-defaults.cfg`: + +``` +config_opts['bootstrap_chroot_additional_packages'] = [] +config_opts['bootstrap_module_enable'] = [] +config_opts['bootstrap_module_install'] = [] +``` + +This is now commented out by default, and the defaults are set in mock code. You can still override it in `site-defaults.cfg`. + +Following contributors contributed to this release: + +* Bernhard Rosenkränzer +* František Zatloukal +* Pavel Raiskup +* Petr Junák +* Sam Fowler + +Thank you. diff --git a/docs/Release-Notes-1.4.15.md b/docs/Release-Notes-1.4.15.md new file mode 100644 index 0000000..5e9cae4 --- /dev/null +++ b/docs/Release-Notes-1.4.15.md @@ -0,0 +1,50 @@ +--- +layout: default +title: Release-Notes 1.4.15 +--- + +Released on 2019-04-22. + +## Mock new features: + +- Mock supports [Dynamic Build Requires](https://fedoraproject.org/wiki/Changes/DynamicBuildRequires). There is still ongoing work in `rpmbuild`; therefore you cannot use it yet. Once the new rpmbuild lands in Fedora you can immediately use it with Mock. [[GH#245](https://github.com/rpm-software-management/mock/issues/245)] + +- I have seen people who do not know about [setup](https://rpm-software-management.github.io/mock/#setup). Now, when you are not in the `mock` group, and Mock asks you via `consolehelper` for root password, it prints this banner: `You are not in the `mock` group. See https://github.com/rpm-software-management/mock/wiki#setup` [[GH#244](https://github.com/rpm-software-management/mock/issues/228)] + +- Previously when Mock executed DNF, then Mock disabled DNF plugin `local`. Now the list of plugins which will be disabled can be configured via: + +``` +config_opts['dnf_disable_plugins'] = ['local', 'spacewalk'] +``` + +The above is the new default, i.e., the plugin `spacewalk` is now disabled as well. [[GH#210](https://github.com/rpm-software-management/mock/issues/210)] + +This change simplified `dnf_common_opts` default, which is now: + +``` +config_opts['dnf_common_opts'] = ['--setopt=deltarpm=False'] +``` + +## Bugfixes: + +- In Flatpak, the method `distro.version()` returns float, which produced fatal error in Mock. This is now fixed [[RHBZ#1690374](https://bugzilla.redhat.com/show_bug.cgi?id=1690374)] + +- new rpm library now returns strings instead of bytes. Mock has been altered that it can accept both types [[RHBZ#1693759](https://bugzilla.redhat.com/show_bug.cgi?id=1693759)] + +- Mock used FileNotFoundError class for a error handling. This class is not defined in Python 2 and caused a traceback during an error handling [[RHBZ#1696234](https://bugzilla.redhat.com/show_bug.cgi?id=1696234)] + +## Known issues: + +- On Fedora 30+, the createrepo_c prints its output to STDERR, which is fatal to mockchain. For the time being, I changed the mockchain behavior and creterepo_c errors are not fatal. However, mockchain print them as an error even there is no error at all. [GH#249](https://github.com/rpm-software-management/mock/issues/249) + +Following contributors contributed to this release: + +* Igor Gnatenko +* Jeroen van Meeuwen (Kolab Systems) +* Jo Shields +* Martin Kutlák +* Neal Gompa +* Pat Riehecky +* Toshio Kuratomi + +Thank you. diff --git a/docs/Release-Notes-1.4.16.md b/docs/Release-Notes-1.4.16.md new file mode 100644 index 0000000..a378936 --- /dev/null +++ b/docs/Release-Notes-1.4.16.md @@ -0,0 +1,20 @@ +--- +layout: default +title: Release Notes 1.4.16 +--- + +Released on 2019-05-22. + +## Mock new features and bugfixes: + +- switch to python3 on el7 +- respect use_host_resolv config even with use_nspawn (praiskup@redhat.com) +- Fix crash on non-ascii dnf log messages (bkorren@redhat.com) + +Following contributors contributed to this release: + +* Barak Korren +* Miro Hrončok +* Pavel Raiskup + +Thank you. diff --git a/docs/Release-Notes-1.4.17.md b/docs/Release-Notes-1.4.17.md new file mode 100644 index 0000000..368c526 --- /dev/null +++ b/docs/Release-Notes-1.4.17.md @@ -0,0 +1,72 @@ +--- +layout: default +title: Release Notes 1.4.17 +--- + +Released on 2019-08-08. + +## Mock-core-configs new features: + + * Added updates-modular to Fedora 29 and Fedora 30, but with `enabled=0` for now due + [bug in DNF](https://bugzilla.redhat.com/show_bug.cgi?id=1737469). + * Removed info about metadata expire. + * Replace groupadd using sysusers.d. + * epel-7 profiles to use mirrorlists. + * EOLed Fedora 28. + * Do not protect packages in chroot [[GH#286]](https://github.com/rpm-software-management/mock/pull/286). + * Fix value for dist for OpenMandriva 4.0 configs. + * Add initial OpenMandriva distribution targets. + +## Mock new features and bugfixes: + + * Mockchain has been replaced by `mock --chain`. This new command inherited most + mockchain command-line options. The [return codes](https://github.com/rpm-software-management/mock/blob/master/mock/py/mockbuild/exception.py#L26) are little different. + This has been done to remove duality - mockchain parsed configs differently than mock. + Now, the behavior should be unified. Mockchain has been marked obsolete - it even prints warning + when you execute, and you are encouraged to migrate to `mock --chain`. I will try to preserve `mockchain` for next + 12 months, but mockchain will not be receiving any new functionality. + * Mock is now able to run in [Fedora Toolbox](https://docs.fedoraproject.org/en-US/fedora-silverblue/toolbox/). + * Added support for [Cheat](https://github.com/cheat/cheat) - try running `cheat mock`. + * There is a new tool `mock-parse-buildlog --path FILE` which tries to parse build.log file and give you nice + human friendly description, why the build failed. Right now, it support just two use cases. Feel free to + send pull request to enhance it. + * Secondary groups are now loaded [[RHBZ#1264005]](https://bugzilla.redhat.com/show_bug.cgi?id=1264005). + * When installing dependencies, Mock pass --allowerasing to DNF now. [[GH#251]](https://github.com/rpm-software-management/mock/pull/251). + * make include() functional for --chain [[GH#263]](https://github.com/rpm-software-management/mock/pull/263). + * Removing BUILDSTDERR from log - it is now configurable via `config_opts['_mock_stderr_line_prefix`]', which is by default empty string. + * Use rpm -qa --root instead of running rpm -qa in chroot. + * Run more that one loop for DynamicBuildrequires if it is neeed. + * Number of loop devices is now configurable using `config_opts['dev_loop_count'] = 12` and the new default has been raised from 4 to 12. This change only affects `--old-chroot`. We are working on making it functional in nspawn chroot as well. + * Return back to call binaries using /bin for split-usr setups. + * Repeat [dynamic requires](https://fedoraproject.org/wiki/Changes/DynamicBuildRequires) loop if needed [[GH#276]](https://github.com/rpm-software-management/mock/pull/276) + * Fix compatibility with pre-4.15 RPM versions with DynamicBuildRequires. + * Enable [Dynamic BuildRequires](https://fedoraproject.org/wiki/Changes/DynamicBuildRequires) by default. + * Independent network configuration [[GH269]](https://github.com/rpm-software-management/mock/pull/269) + * Now, when you execute `mock -r FOO`, mock will check if `~/.config/mock/FOO.cfg` exists and use this config. If it does not exists, it will use the `/etc/mock/FOO.cfg`. This is useful if you want to localy override default configs. + * respect use_host_resolv config even with use_nspawn. + * Fix crash on non-ascii dnf log messages. + * switch to python3 on el7 (msuchy@redhat.com) + +## Future + +Note that in upcoming versions, I would like to: + + * drop python2 support as even EL7 version is running on python3 now. + * drop EL7 support (likely spring 2020). I mean to stop building Mock for EL7. Building packages for EL7 using Mock will be still supported. + * make DNF default package manager. E.g., you will have to state in config that you want to use yum explicitly. + * Pavel Raiskup is preparing support for building for RHEL 8 targets. So besides traditional CentOS targets, you will be able to build for RHEL, if you have Red Hat subscription. This will allows you to not wait for CentOS release when RHEL has already been released. + +Following contributors contributed to this release: + + * Barak Korren + * Bernhard Rosenkränzer + * Igor Gnatenko + * khoitd1997 + * Martin Necas + * Miro Hrončok + * Neal Gompa + * Owen W. Taylor + * Pavel Raiskup + * Silvie Chlupova + +Thank you. diff --git a/docs/Release-Notes-1.4.18.md b/docs/Release-Notes-1.4.18.md new file mode 100644 index 0000000..fd1db89 --- /dev/null +++ b/docs/Release-Notes-1.4.18.md @@ -0,0 +1,41 @@ +--- +layout: default +title: Release Notes 1.4.18 +--- + +Released on 2019-08-27. + +## Mock-core-configs new features: + + * New configs for RHEL. See [separate page](Feature-rhelchroots) for more info. + * Fedora 31 has been added + * Add local-source repo definition to Fedora Rawhide. + * revert sysusers setting [RHBZ#1740545](https://bugzilla.redhat.com/show_bug.cgi?id=1737469) + +## Mock new features and bugfixes: + + * When a foreign architect is detected, Mock will automatically enable `--forcearch`. + * Support for subscription-manager has been added. See [RHEL chroots](Feature-rhelchroots) page. + * bootstrap-chroot always explicitly install shadow-utils + * Add [procenv plugin](Plugin-ProcEnv.md) for more detailed build time information. This plugin is disabled by default. + * Resolved issues with SELinux from the previous release. You may still experience some warnings, but none of them should be fatal. + * SIGTERM, SIGPIPE, and SIGHUP signals are now propagated to chroot. + +## Future + +Note that in upcoming versions, I would like to: + + * drop python2 support as even EL7 version is running on python3 now. + * drop EL7 support (likely spring 2020). I mean to stop building Mock for EL7. Building packages for EL7 using Mock will be still supported. + * make DNF default package manager. E.g., you will have to state in the config that you want to use yum explicitly. + +Following contributors contributed to this release: + +* Dominik Turecek +* Jan Buchmaier +* Jiri Konecny +* Miro Hrončok +* Pat Riehecky +* Pavel Raiskup + +Thank you. diff --git a/docs/Release-Notes-1.4.19.md b/docs/Release-Notes-1.4.19.md new file mode 100644 index 0000000..201f470 --- /dev/null +++ b/docs/Release-Notes-1.4.19.md @@ -0,0 +1,17 @@ +--- +layout: default +title: Release Notes 1.4.19 +--- + +Released on 2019-09-10. + +## Mock bugfixes: + + * Biggest reason for this release have been regression that results are owned by root user instead of unpriv user [[GH#322]](https://github.com/rpm-software-management/mock/issues/322) + * Previously resultdir variable in config has not been documented. This is now fixed. + +Following contributors contributed to this release: + +* Silvie Chlupová + +Thank you. diff --git a/docs/Release-Notes-1.4.2.md b/docs/Release-Notes-1.4.2.md new file mode 100644 index 0000000..4a814bc --- /dev/null +++ b/docs/Release-Notes-1.4.2.md @@ -0,0 +1,28 @@ +--- +layout: default +title: Release Notes 1.4.2 +--- + +There are new features: + +* The bootstrap feature is now disabled by default. There were too many issues with it. You can enable it locally with `--bootstrap-chroot`, but first see knows [bugs](https://bugzilla.redhat.com/buglist.cgi?bug_status=NEW&bug_status=ASSIGNED&component=mock&known_name=mock-all&list_id=7491839&product=Fedora&product=Fedora%20EPEL&query_based_on=mock-all&query_format=advanced) and [issues](https://github.com/rpm-software-management/mock/issues). +* There is initial support for Fedora Modularity. You can add to config: + +``` +config_opts['module_enable'] = ['list', 'of', 'modules'] +config_opts['module_install'] = ['module1/profile', 'module2/profile'] +``` + +This will call `dnf module enable list of modules` and `dnf module install module1/profile module2/profile` during the init phase. EDIT: If you want to use this feature you have to have experimental DNF, it can be obtained from this [Copr project](https://copr.fedorainfracloud.org/coprs/mhatina/DNF-Modules/). + +There are some bugfixes: + +* NSpawn chroot is switched off for EL6 targets [[RHBZ#1456421](https://bugzilla.redhat.com/show_bug.cgi?id=1456421)]. +* LVM root is not umounted when `umount_root` is set to false [[RHBZ#1447658](https://bugzilla.redhat.com/show_bug.cgi?id=1447658)] +* Shell in NSpawn container is now called with `--login` so `profile.d` scripts are executed [[RHBZ#1450516](https://bugzilla.redhat.com/show_bug.cgi?id=1450516)] [[RHBZ#1462373](https://bugzilla.redhat.com/show_bug.cgi?id=1462373)] +* yum rather then yum-deprecated is used when using bootstrap chroot [[RHBZ#1446294](https://bugzilla.redhat.com/show_bug.cgi?id=1446294)] +* Custom chroot does not use bootstrap [[RHBZ#1448321](https://bugzilla.redhat.com/show_bug.cgi?id=1448321)] +* Mock now use `dnf repoquery` instead of repoquery for chroots which uses DNF. +* LVM's scrub hook for bootstrap chroot is called [[RHBZ#1446297](https://bugzilla.redhat.com/show_bug.cgi?id=1446297)] +* `--mount` will mount LVM volumes [[RHBZ#1448017](https://bugzilla.redhat.com/show_bug.cgi?id=1448017)] + diff --git a/docs/Release-Notes-1.4.20.md b/docs/Release-Notes-1.4.20.md new file mode 100644 index 0000000..0960bbd --- /dev/null +++ b/docs/Release-Notes-1.4.20.md @@ -0,0 +1,90 @@ +--- +layout: default +title: Release Notes 1.4.20 +--- + +Released on 2019-10-04. + +## Mock new features: + +### Container image for bootstrap + +Previously we have some incompatibilities between host and build target. They were, in fact, small. Like using a different package manager. Some were big. Like, the introduction of Weak and Rich dependencies. For this reason, we introduced [bootstrap](Feature-bootstrap). But then comes [zstd payload](https://fedoraproject.org/wiki/Changes/Switch_RPMs_to_zstd_compression). This is a new type of payload. And to install packages with this payload, you need rpm binary, which supports this payload. This is true for all current Fedoras. Unfortunately, neither RHEL 8 nor RHEL 7 supports this payload. So even bootstrap will not help you to build Fedora packages on RHEL 8. + +We come up with a nice feature. Mock will not install bootstrap chroot itself. Instead, it will download the container image, extract the image, and use this extracted directory as a bootstrap chroot. And from this bootstrapped chroot install the final one. + +Using this feature, **any** incompatible feature in either RPM or DNF can be used in the target chroot. Now or in future. And you will be able to install the final chroot. You do not even need to have RPM on a host. So this should work on any system. Even Debian based. The only requirement for this feature is [Podman](https://podman.io/). + +This feature is now disabled by default. You can enable it using: + + config_opts['use_bootstrap_image'] = True + +It can be enabled or disabled on the command line using `--use-bootstrap-image` or `--no-bootstrap-image` options. + +Note however that also this is prerequisite: + + config_opts['use_bootstrap_container'] = True # or --bootstrap-chroot option + +To specify which image should be used for bootstrap container you can put in config: + + config_opts['bootstrap_image'] = 'fedora:latest' + +This is a general config. Each config has specified its own image specified. E.g. CentOS 7 has `config_opts['bootstrap_image'] = 'centos:7'` in config. So unless you use your own config, you can enable this feature, and the right image will be used. + +There is one known issue: + + * Neither Mageia 6 nor 7 works correctly now with this feature. + +Technically, you can use any container, as long as there is the required package manager (DNF or YUM). The rest of the needed packages will be installed by mock. + +### Mockchain removed + +Mockchain has been removed. I wanted to keep it longer, but because of [[RHBZ#1757388](https://bugzilla.redhat.com/show_bug.cgi?id=1757388)] I decided to remove it now. You should use `mock --chain` instead of `mockchain`. There is present simple wrapper `/usr/bin/mockchain` which calls `mock --chain`. Most of the mockchain parameters are still preserved for `mock --chain`. + +### New config option `package_manager_max_attempts` + +When your infrastructure is not reliable and you see failing builds because of network issues, you can increase number of attemps to execute package manager's action. This can be now tuned using: + + config_opts['package_manager_max_attempts'] = 1 + config_opts['package_manager_attempt_delay'] = 10 + +### Bind mount local repos to bootstrap chroot + +Previously when you have in your config something like: + + config_opts['yum.conf'] = """ + ... + [myrepo] + baseurl=file:///srv/myrepo + +then the path `/srv/myrepo` was not available inside of bootstrap container. The package manager was then unable to fetch those repositories. + +This is now fixed and those directories are now automatically bind-mounted to bootstrap chroot. + +This was actually the last known issue with bootstrap chroots. You may expect that in a future version of Mock, the bootstrap chroot will be enabled by default. + +## Mock-core-config bugfixes + + * Fix baseurl typo in centos-stream config + + * Disabled modular repo for f29 - this was accidentally enabled during transition to templates. + +## Mock bugfixes + + * Several files - mainly logs - are created as unprivileged user now. This will fix several issues when you use NFS. [[#341](https://github.com/rpm-software-management/mock/issues/341)], [[#322](https://github.com/rpm-software-management/mock/issues/322)], [[RHBZ#1751944](https://bugzilla.redhat.com/show_bug.cgi?id=1751944)] + + * `/var/log` is now ignored when creating root cache. + + * `mock --chain` now creates local repositories using `skip_if_unavailable=0` + +Following contributors contributed to this release: + + * Daniel Mach + * Denis Ollier + * Chuanhao jin + * Jakub Kadlcik + * Jiri 'Ghormoon' Novak + * Pavel Raiskup + * Silvie Chlupova + +Thank you. diff --git a/docs/Release-Notes-1.4.21.md b/docs/Release-Notes-1.4.21.md new file mode 100644 index 0000000..d06f0fc --- /dev/null +++ b/docs/Release-Notes-1.4.21.md @@ -0,0 +1,31 @@ +--- +layout: default +title: Release Notes 1.4.21 +--- + +Released on 2019-11-01. + +## Mock-core-configs 31.7 + + * Added configs for epel8-playground + * Added 3 base packages to epel-playground and epel buildroot [RHBZ#1764445](https://bugzilla.redhat.com/show_bug.cgi?id=1764445) + +## Mock 1.4.21 bugfixes: + +This is a bugfix-only release. There is already ongoing work on 1.5.0 version. I cherry-picked some commits, which resolves some painfull bugs: + +There were some issue with initialization of "Container image for bootstrap" feature. [GH#380](https://github.com/rpm-software-management/mock/issues/380). This is now fixed. As side effect there are two changes. Download of container image has been moved from `root_cache` plugin to main Mock code. As result you do not need to have root cache enabled to use this feature. Second, distribution-gpg-keys are always copied to bootstrap chroot if you use bootstrap container feature. + +Commands `--install` and `--installdeps` now works with bootstrap [RHBZ#1447627](https://bugzilla.redhat.com/show_bug.cgi?id=1447627) + +There was an ugly bug, which involved systemd, CGroups v2 and SELinux and can lead to complete freeze of a system. This has been now resolved. [RHBZ#1756972](https://bugzilla.redhat.com/show_bug.cgi?id=1756972) + +Rarely you may hit bug with incorrect rpmbuildstate. This is now fixed. [GH#349](https://github.com/rpm-software-management/mock/issues/349). + +Following contributors contributed to this release: + + * Jakub Kadlcik + * Merlin Mathesius + * Pavel Raiskup + +Thank you. diff --git a/docs/Release-Notes-1.4.3.md b/docs/Release-Notes-1.4.3.md new file mode 100644 index 0000000..65f1a4f --- /dev/null +++ b/docs/Release-Notes-1.4.3.md @@ -0,0 +1,24 @@ +--- +layout: default +title: Release Notes 1.4.3 +--- + +This is bug fix release, we fixed following issues: + +* `--nocheck` macro was not properly escaped [[RHBZ#1473359](https://bugzilla.redhat.com/show_bug.cgi?id=1473359)]. +* Use python3 and dnf module on Fedoras to guess architecture in `%post` scriptlet [[RHBZ#1462310](https://bugzilla.redhat.com/show_bug.cgi?id=1462310)]. +* enhanced detection of RHEL [[RHBZ#1470189](https://bugzilla.redhat.com/show_bug.cgi?id=1470189)]. +* scm: define `_sourcedir` to checkout directory [[PR#98](https://github.com/rpm-software-management/mock/pull/98)]. +* Mageia Cauldron `releasever` is now 7 [[PR#95](https://github.com/rpm-software-management/mock/pull/95)] +* Create `/dev` nodes even when using `nspawn` [[RHBZ#1467299](https://bugzilla.redhat.com/show_bug.cgi?id=1467299)]. +* SELinux: do not try to import yum when PM is dnf [[RHBZ#1474513](https://bugzilla.redhat.com/show_bug.cgi?id=1474513)]. +* When you have hundreds of volumes in LVM you can tell mock to wait longer using `config_opts['plugin_conf']['lvm_root_opts']['sleep_time'] = 1`. + +Thanks to following contributors: + +* Igor Gnatenko +* Jonathan Lebon +* Mikolaj Izdebski +* Neal Gompa +* Ville Skyttä +* pixdrift diff --git a/docs/Release-Notes-1.4.4.md b/docs/Release-Notes-1.4.4.md new file mode 100644 index 0000000..a0413cb --- /dev/null +++ b/docs/Release-Notes-1.4.4.md @@ -0,0 +1,11 @@ +--- +layout: default +title: Release Notes 1.4.4 +--- + +This is bug fix release, we fixed following issues: + +* Fedora 27 configs have been added. +* /etc/dnf/dnf.conf is used instead of /etc/dnf.conf +* /etc/dnf/dnf.conf is populated even when yum is used +* Rename group inside of chroot from mockbuild to mock - this will allow you to install mock inside of mock' chroot. Please invalidate your previous caches. diff --git a/docs/Release-Notes-1.4.6.md b/docs/Release-Notes-1.4.6.md new file mode 100644 index 0000000..c0586e5 --- /dev/null +++ b/docs/Release-Notes-1.4.6.md @@ -0,0 +1,33 @@ +--- +layout: default +title: Release Notes 1.4.6 +--- + +Released on 2017-09-15. + +This is mostly a bugfix release, but there are some features too: + +Features: + +* All chroot configs have been moved to new package mock-base-configs. This will allow us to release new chroot configs independently of Mock main code. +* There is a new command --debug-config available. This command print current mock config (including defaults) to standard output and exit. This can be useful when you experience some issue, which you cannot reproduce anywhere else. +* There is a new script, which can add a default route to loopback for a container with private-network. This is an experimental feature, not used automatically, and will very likely change in future. +* There is short option `-N` for `--no-cleanup-after`. + + +Bugfixes: + +* Mock again create /dev/loop nodes. This caused a lot of pain to Lorax users. [RHBZ#1481370](https://bugzilla.redhat.com/show_bug.cgi?id=1481370) +* Comment about nspawn/chroot default in site-defaults.cfg was previously incorrect, this has been fixed now. +* Previously when you used --private-network then the isolated network was used only during rpm build phase. And, e.g., for shell command, it was not used. This caused some confusion. Now the network is always switched when you specify --private-network. +* The bug "The buildroot LVM volume is not kept mounted after build" [RHBZ#1447658](https://bugzilla.redhat.com/show_bug.cgi?id=1447658) has been fixed once again. Hopefully this time correctly. + +Following contributors contributed to this release: + +* Brian C. Lane +* Jan Synacek +* Matej Kudera +* Michael Simacek +* Ville Skyttä + +P.S. I did not skip 1.4.5. I just find a serious bug in requirement just after the release. So I made two releases in one day. :) This is united release notes. diff --git a/docs/Release-Notes-1.4.7.md b/docs/Release-Notes-1.4.7.md new file mode 100644 index 0000000..f5dc5c5 --- /dev/null +++ b/docs/Release-Notes-1.4.7.md @@ -0,0 +1,38 @@ +--- +layout: default +title: Release Notes 1.4.7 +--- + +Released on 2017-10-31. + +Features: + +* There is a new option in config `config_opts['chrootgroup']`, which allows you to change name of group inside of chroot. +* Any key for `config_opts` you specify with 'bootstrap_*' will be copied to bootstrap config e.g., `config_opts['bootstrap_system_yum_command'] = '/usr/bin/yum-deprecated'` will become `config_opts['system_yum_command'] = '/usr/bin/yum-deprecated'` for bootstrap config. +* There are three new default: +``` + config_opts['bootstrap_chroot_additional_packages'] = [] + config_opts['bootstrap_module_enable'] = [] + config_opts['bootstrap_module_install'] = [] +``` + This will not install any additional packages or modules into bootstrap chroot. +* Mock now recognize DeskOS. +* Previously when `config_opts['rpmbuild_networking']` was enabled we passed `--private-network` to systemd-nspawn. However that lead there was no default route. And you cannot bind() UDP socket to all IP addresses and then join multicast group, without having default route. Now we do onot add `--private-network` to systemd-nspawn, instead we setup network namespace ourselves and we also add default route pointing to loopback interface (only interface in the new namespace). This feature introduce new dependency on pyroute2. + +Bugfixes: + +* Delete rootdir as well when calling clean. In case one overrides the rootdir option, and the rootdir is located outside of basedir, it was not cleaned up when calling --clean. Fix this case by checking if the rootdir is outside basedir. If that is the case, run an extra rmtree() on it. +* Choose good symbolic link of default.cfg on Mageia. +* Ccache is now mounted to /var/tmp as /tmp gets over-mounted with tmpfs when system-nspawn is used. +* Output of `--debug-config` is now sorted. +* Use primary key for Fedora 27+ on s390x. + +Following contributors contributed to this release: + +* Andreas Thienemann +* Dan Horák +* Jan Pokorný +* Mark D Horn +* Michal Sekletar +* Neal Gompa +* Ricardo Arguello diff --git a/docs/Release-Notes-1.4.8.md b/docs/Release-Notes-1.4.8.md new file mode 100644 index 0000000..d72eae4 --- /dev/null +++ b/docs/Release-Notes-1.4.8.md @@ -0,0 +1,77 @@ +--- +layout: default +title: Release Notes 1.4.8 +--- + +Released on 2017-12-22. + +Features: +* There is a new option --config-opts [GH#138](https://github.com/rpm-software-management/mock/issues/138) + +You can run: + +``` + mock --config-opts yum_command=/usr/bin/yum-deprecated --enable-network +``` + +which will set: + +``` + config_opts['system_yum_command'] = '/usr/bin/yum' +``` + +or for a list: + +``` + mock --config-opts extra_chroot_dirs=/mnt/b --config-opts extra_chroot_dirs=/mnt/a +``` + +which will set + +``` + config_opts['extra_chroot_dirs'] = ['/mnt/b', '/mnt/a'] +``` + +or list with a single item: + +``` + mock --config-opts extra_chroot_dirs=/mnt/b --config-opts extra_chroot_dirs= +``` + +which will set + +``` + config_opts['extra_chroot_dirs'] = ['/mnt/b'] +``` + +It can detect boolean: + +``` + mock --config-opts nosync=False --debug-config |grep nosync + config_opts['nosync'] = False +``` + +A specialized option has priority. Therefore: + +``` + mock --config-opts rpmbuild_networking=False --enable-network --debug-config |grep rpmbuild_networking + config_opts['rpmbuild_networking'] = True +``` + +It is unable to set complicated variables. Like config_opts['plugin_conf']['package_state_opts'] or anything which has dictionary as value. + +* There is a new option. `--enable-network` which is equivalent to `config_opts['rpmbuild_networking'] = True` + +Bugfixes: +* orphanskill now emits SIGKILL when SIGTERM is not enough [RHBZ#1495214](https://bugzilla.redhat.com/show_bug.cgi?id=1495214) +* when mock tries to force umount, it will try umount recursively +* do not change to directory if nspawn is used [GH#108](https://github.com/rpm-software-management/mock/issues/108) +* when creating yum/dnf.conf, mock now copy timestamp from the host [RHBZ#1293910](https://bugzilla.redhat.com/show_bug.cgi?id=1293910) +* We now mount /proc and /sys in chroot before executing any package manager command (outside of chroot)[RHBZ#1467299](https://bugzilla.redhat.com/show_bug.cgi?id=1467299) +* Dependencies of mock-scm (git, cvs, tar, subversion) are now soft dependencies (Recommends) [RHBZ#1515989](https://bugzilla.redhat.com/show_bug.cgi?id=1515989) +* Previously job control in `mock shell` does not work. [RHBZ#1468837](https://bugzilla.redhat.com/show_bug.cgi?id=1468837). This was a glibc bug and it is resolved in rawhide now. + +Following contributors contributed to this release: + +* Matt Wheeler +* Matthew Stoltenberg diff --git a/docs/Release-Notes-1.4.9.md b/docs/Release-Notes-1.4.9.md new file mode 100644 index 0000000..4014443 --- /dev/null +++ b/docs/Release-Notes-1.4.9.md @@ -0,0 +1,50 @@ +--- +layout: default +title: Release Notes 1.4.9 +--- + +Released on 2018-02-12. + +Note: + +In this release, there are several fixes to bootstrap feature. This is especially important for users who run Mock on EL7. Rich dependencies are now allowed in Fedora and maintainers are starting to use them. So sooner or later, you will be unable to build packages for Fedoras on EL7 host. Unless you start using bootstrap feature (`--bootstrap-chroot`), which is still by default off. + + +Features: +* Stdout and stderr in build.log has been split. All stderr output lines are prefixed by `BUILDSTDERR:` +* There is a new config option `opstimeout`: + +``` +# Set timeout in seconds for common mock operations +# if 0 is set, then no time limit is used +# config_opts['opstimeout'] = 0 +``` + +The default is 0, which means that Mock is waiting until command exit. + +Bugfixes: +* Builds for EL5 are working again - EL5 is sensitive to order of params of adduser [RHBZ#1535328](https://bugzilla.redhat.com/show_bug.cgi?id=1535328) +* Use correct builddep when bootstrap is used. Additionally, ccache is not installed into bootstrap chroot. [RHBZ#1540813](https://bugzilla.redhat.com/show_bug.cgi?id=1540813). +* User defined mounts are not mounted in bootstrap chroot. +* Detect if essential mounts are already mounted - previously, mock assumed that essential mounts (procfs, sysfs) are never mounted when mock starts up. That's not true, as multiple non-destructive mock processes are allowed (`--shell`, `--install`, etc.) to run concurrently. So when you use `mock --shell` and do a `mock --install` in parallel, it breaks your shell, because it unmounts its proc. This improves the situation by first asking whether the mounts aren't there already. +* fix quoting in sign_opts example in site-defaults.cfg [RHBZ#1537797](https://bugzilla.redhat.com/show_bug.cgi?id=1537797). +* Honor the "cwd" flag when nspawn is being used and "chrootPath" is not set. +* Do not produce a warning when we are using different PM for a bootstrap container. +* Default for config_opts['dnf_warning'] in site-defaults.cfg according to docs. + +Additionally, there are several major changes in mock-core-config. This package is independent now, and a new version has been released two weeks ago and will be pushed to Fedora stable next week. I will repeat here changes in that package: +* Fedora 28 configs has been added. +* `failovermethod=priority` has been removed for repos which use DNF. This is the only method which DNF recognize and it cannot be changed. +* Set `skip_if_unavailable=False` for all repos. If a repository is unreachable, then build fails. + + +Following contributors contributed to this release: + +* Barak Korren +* Michael Simacek +* Mikhail Campos Guadamuz +* mprahl +* Pavel Raiskup +* Todd Zullinger + +Thank you. diff --git a/docs/Release-Notes-2.0.md b/docs/Release-Notes-2.0.md new file mode 100644 index 0000000..f57a509 --- /dev/null +++ b/docs/Release-Notes-2.0.md @@ -0,0 +1,150 @@ +--- +layout: default +title: Release Notes 2.0 +--- + +Released on 2020-02-07. + +## Mock 2.0 highlights: + + * The mock versioning policy (or rather style) has changed from three to + two-number pattern. Don't panic, this isn't really special major + release - the change was only done to move from the + `..` pattern to `.`, so practically + we went with *v2.0* instead of previously planned *v1.5.0*. + + * The `--bootstrap-chroot` option is newly enabled by default, this can be + disabled by `--no-bootstrap-chroot`, or by + `config_opts['use_bootstrap'] = False`. The content of bootstrap chroot + is cached by default and never automatically updated, but one + can use the new `--scrub=bootstrap` to remove related caches. The + `--scrub=all` was updated to clean bootstrap as well (but `--clean` + doesn't touch bootstrap chroot at all). + + * The `use_bootstrap_container` configuration option was renamed to + `use_bootstrap` to better describe it's purpose (it never implied usage + of container technology) and to align with `use_bootstrap_image` + option. Please migrate your custom configuration. + + * The output from `--debug-config` option now only shows the differences + from mock's defaults, and the output doesn't have the Jinja templates + expanded. + + * The `config_opts['dnf.conf']` replaced `config_opts['yum.conf']`. Both + still work, but only one of them can exist one config file. + + * The `--old-chroot` and `--new-chroot` options were obsoleted by + `--isolation=chroot|nspawn`, and still default to `--isolation=nspawn`. + Please migrate your tooling. + + * Mock now can now pre-configure DNF variables ([#346](../issues/346)), e.g. + `config_opts['dnf_vars'] = { 'stream': '8-stream' }` + + * The regression in `--use-bootstrap-image` implementation was fixed (did + not work at all in `v1.4.21`), and should work reliably now (`podman` + still needs to be installed manually to make it work). + + * In mock config files we now prefer Jinja templates, instead of + previously used python expansion `"%(variable)" % ..`. It is not + likely, but if you use this in your custom config files, please + migrate. + +## Mock 2.0 other fixes and enhancements: + + * Loop device files are pre-populated even in `--isolation=nspawn` + chroots, similarly to what is done with `--isolation=chroot` + ([#298](../issues/298)). + + * The `include()` statement in mock config now also accepts relative path + names (relative against `config_opts['config_path']` for now). + + * The host local repositories from mock config files + (like `baseurl=file://`) are now correctly bind-mounted to bootstrap + chroot. So installing RPM from such repositories with + `--bootstrap-chroot` now works (related [#381](../issues/381)). + + * Non-interactive commands in chroot are executed through + `systemd-nspawn --console=pipe` (when `--isolation=nspawn`, default) + ([#432](../issues/432)). + + * Better detection of host's package manager (DNF vs. YUM), for both + bootstrap and normal chroot. This should demotivate people from using + `--dnf` and `--yum` options ([#233](../issues/233)). More, on Fedora 31+ there's no + real YUM package manager anymore (there only is `yum.rpm` which actually + provides `/bin/yum` symlink to `/bin/dnf`). This situation is now + properly detected in mock, and the symlink is ignored (we fallback to + DNF). + + * Better re-using of DNF/YUM caches, in both normal and bootstrap chroot. + This is mostly given by previous bullet (YUM vs. DNF detection). To be + 100% sure, we also newly rather bind-mount both DNF and DNF cache + directories into the chroot. + + * Mock expands the config templates (aka `include()`) completely before + executing it by eval(), and the implementation is now much simpler and + clear. + + * Mock doesn't ignore `cleanup_on_success` configuration option after + `--postinstall` action. + + * `mock --chain` file descriptor leak was fixed, so the descriptor usage + is constant with multiple builds. + + * The Jinja templating is now iteratively re-rendered (when Jinja template + expands to another Jinja template), till there is something to expand. Also + we start the Jinja rendering mechanism a bit earlier in the codebase so the + mock configuration isn't really order-dependant (no matter which + configuration option is set first). + + * Fix lvm plugin volume removal feature on modern systems + ([rhbz#1762728](https://bugzilla.redhat.com/1762728)). + + * We don't install `shadow-utils` (we don't need this one) and + `distribution-gpg-keys` (we copy the keys from host instead), so this + makes the initial `dnf_install_command` transaction shorter, and more + reliable across all the variety distributions we support. + + * The `--sources` parameter is not mandatory in `--buildsrpm` mode. + + * Mock now copies `/etc/pkg/ca-trust/extracted` into chroot + ([#397](../issues/397)). + + * The `success` and `fail` files are created under mockbuild user, not root. + + * The `compress_logs`, when turned on, have predefined default `gzip` method. + + * We turned `--forcearch` on long time ago, but mock exited with cryptic + error when `qemu-user-static` wasn't installed. Mock now detects that + `qemu-user-static` is missing and throws instructions instead. + + +## Mock-core-configs 32.0 + + * Added configs for **Fedora 32**, Fedora Rawhide configs moved to F33. The new + package depends on updated **distribution-gpg-keys 1.36** package (avaiable + in Fedora updates at the time of release). + + * Fedora 29 configs EOLed (moved below `eol` subdirectory). + + * All the configuration files were modified to use templates, to de-duplicate + a lot of stuff and many inconsistencies were fixed. + + * On el7, mock/mock-core-configs automatically enable `use_bootstrap_image` + option for **Fedora 31+** chroots (ZSTD compression enabled for RPMs) because without + this option it wouldn't make sense to do anything (neither bootstrap chroot + is installable). + +Both mock and mock-core-configs packages need to be updated together as pair. + +Following contributors contributed to this release: + + * Dominik Tureček + * Jakub Čajka + * Jakub Kadlčík + * Merlin Mathesius + * Scott K Logan + * Sérgio M. Basto + * Silvie Chlupová + * Tomas Hrnciar + +Thank you. diff --git a/docs/Release-Notes-2.1.md b/docs/Release-Notes-2.1.md new file mode 100644 index 0000000..6cf80c3 --- /dev/null +++ b/docs/Release-Notes-2.1.md @@ -0,0 +1,65 @@ +--- +layout: default +title: Release Notes 2.1 +--- + +Released on 2020-03-11. + +## Mock 2.1 bugfixes: + + * Fixed `mock --install ` request when `` is a file or directory + in CWD, or an absolute path on host (#474). + + * We do not emit the warning `WARNING: Not using '/usr/bin/yum', it is symlink + to '/usr/bin/dnf-3'` anymore for installing bootstrap chroot (#477, + rhbz#1802930). + + * The `config_opts['dnf.conf']` option is made equivalent to + `config_opts['yum.conf']` (#486). + + * Allow specifying host-local repositories with `baseurl=/absolute/path`, not + only with `baseurl=file:///absolute/path`. This did not work with bootstrap + mode before (#480). + + * Fixed broken sign plugin (#476, rhbz#1806577). + + * Fixed too deep jinja recursion caused by trailing newlines in `dnf.conf` + config option (rhbz#1806482). + + * The `mock --scrub` with lvm_root plugin enabled did not work (rhbz#1805179). + + * Do not fail when host doesn't provide CA certificates on expected locations + (#492). + + * Traceback fix for `mock --chain` with tmpfs `keep_mounted` enabled (#479). + + * Dnf caches aren't cleaned for consecutive builds with `mock --chain` (#483). + +## Mock 2.1 new features: + + * Mock expects that `rpmbuild -br` (for %generate_buildrequires spec statement, + aka "dynamic BuildRequires") can return both exit status 0 and 11. Currently + released RPM always returns 11, but the plan is to fix that to return 0. + + * New option `ssl_ca_bundle_path`. When specified, the CA certificate bundle + is copied from host to the specified path in chroot (usually it is enough to + keep the default behavior when whole `/etc/pki/ca-trust/extracted` is + copied, but e.g. OpenSUSE has different path to bundle) (#500). + + +## Mock-core-configs 32.4 + + * Specify CA bundle path for OpenSUSE chroots (#500). + + * EOL Mageia 6 configs. + + * Temporarily disable package_state plugin for openmandriva 4.0 and Cooker (#525). + +Following contributors contributed to this release: + + * Jakub Kadlcik + * Miroslav Suchý + * Remi Collet + * Tomas Hrnciar + +Thank you. diff --git a/docs/Release-Notes-2.10.md b/docs/Release-Notes-2.10.md new file mode 100644 index 0000000..b7c8d43 --- /dev/null +++ b/docs/Release-Notes-2.10.md @@ -0,0 +1,49 @@ +--- +layout: default +title: Release Notes 2.10 +--- + +Released on - 2021-04-27. + +## Mock 2.10 bugfixes: + + * The `podman run` command which is used to pre-prepare the base mock bootstrap chroot + is now called just with `-i`, not with `-i -t`. That's because the new Podman + variants dislike `-t` when no tty is on the input. + + * Fixed traceback for copying the Katello configs into the bootstrap chroot, + [PR 678][PR#678]. + + * Mount point handling was fixed; newly we use the correct and expected + mount-point options for recursive mounts (mostly needed for older util-linux + variants), and we correctly umount sub-set of already ḿounted mountpoints + upon some failure (traceback). [PR 712][PR#712] + + +## Mock-core-configs v34.3: + + * Added Oracle Linux 7 and 8 configs. + + * Add openSUSE Leap 15.3 configs. + + * The openSUSE Leap 15.1 config was marked to EOL in configs. + + * Add openSUSE Tumbleweed s390x config + + * AlmaLinux 8 configs added + + * The 'make' package was removed from the minimal ELN buildroot. + + +The following contributors contributed to this release: + + * David Ward + * Miro Hrončok + * Miroslav Suchý + * Neal Gompa + +Thank you! + +[PR#712]: https://github.com/rpm-software-management/mock/pull/712 +[PR#678]: https://github.com/rpm-software-management/mock/pull/678 + diff --git a/docs/Release-Notes-2.11.md b/docs/Release-Notes-2.11.md new file mode 100644 index 0000000..f52816c --- /dev/null +++ b/docs/Release-Notes-2.11.md @@ -0,0 +1,59 @@ +--- +layout: default +title: Release Notes 2.11 +--- + +Released on - 2021-06-09 + + +## Mock 2.11 features: + + * You can use `--cwd` together with `--shell` now. [[PR 732][PR#732]] + + * You can use `mock --install 'external:pypi:hwdata'` now. [[PR 733][PR#733]] + + * Mock now defines macro `%{_platform_multiplier}` which is set to 1 by default. However, when [forcearch][forcearch] is used, then it is set to 10. [[PR 730][PR#730]] + + This can be used to tune timeouts in e.g., `%check` and reflects that emulated platforms can take longer to finish task. Suggested use case can be: + + ``` + %{!?_platform_multiplier:%global _platform_multiplier 1} + timeout $(( 60*%_platform_multiplier )) the-long-running-task + ``` + + This will timeout after 60 seconds but on emulated platforms after 600 seconds. + + If you have slow builder for some architecture, you can put in your config + + ``` + config_opts['macros']['%_platform_multiplier'] = 5 + ``` + + to tune up this macro. + +## Mock 2.11 bugfixes: + + * Plug-in `compress_logs` now compresses log files even in case of DNF + repository failures [[PR 736][PR#736]]. + + * Broken "usage" section in `mock --help` output was fixed [[issue 738][#738]]. + + +## Mock-core-configs v34.4: + + * centos-stream-8 repositories use mirrorlist now. And have additional repositories which are presented in default centos-stream-8 installation [[PR 729][PR#729]] + +The following contributors contributed to this release: + + * Neal Gompa + * Miroslav Suchý + +Thank you! + +[PR#729]: https://github.com/rpm-software-management/mock/pull/729 +[PR#730]: https://github.com/rpm-software-management/mock/pull/730 +[PR#732]: https://github.com/rpm-software-management/mock/pull/732 +[PR#733]: https://github.com/rpm-software-management/mock/pull/733 +[PR#736]: https://github.com/rpm-software-management/mock/pull/736 +[#738]: https://github.com/rpm-software-management/mock/issues/738 +[forcearch]: https://github.com/rpm-software-management/mock/wiki/Feature-forcearch diff --git a/docs/Release-Notes-2.12.md b/docs/Release-Notes-2.12.md new file mode 100644 index 0000000..49752ef --- /dev/null +++ b/docs/Release-Notes-2.12.md @@ -0,0 +1,58 @@ +--- +layout: default +title: Release Notes 2.12 +--- + +Released on - 2021-07-19 + + +## Mock 2.12 bugfixes: + +This is rather a small bugfix release, the most interesting stuff has been done +in mock-core-configs package (see below). + + * We don't set --cwd for --shell mode when a systemd-nspawn without the + `--chdir` option is installed on the sytem (typically el7) + + * An RPM `addMacro()` traceback fixed. The SCM plugin was fixed to explicitly + convert the configured macro macro values (in e.g. + `config_opts['macros']['%_platform_multiplier'] = 10`) to strings. + [[PR 753][PR#753]] + + * Explicitly disabled versionlock DNF plugin by default, as we don't want to + affect the builds. [[PR 747][PR#747]] + + * Mock package requirement on `shadow-utils` was removed from + `mock-core-configs` to proper `mock-filesystem`. [[PR 743][PR#743]] + + +## Mock-core-configs v34.6: + + * CentOS Stream 9 "preview" files added + + * Rocky Linux configs added + + * AlmaLinux 8 AArch64 configs added. + + * Add AlmaLinux Devel repo as an optional repo for AlmaLinux 8. + + * Fixed GPG key path for SLE updates in openSUSE Leap 15.3. + + * Switch CentOS templates to use quay.io images for bootstrap. + + * EPEL Next 8 configs added. + +The following contributors contributed to this release: + + * Carl George + * Igor Raits + * Louis Abel + * Miroslav Suchý + * Neal Gompa + * Scott K Loga + +Thank you! + +[PR#747]: https://github.com/rpm-software-management/mock/pull/743 +[PR#747]: https://github.com/rpm-software-management/mock/pull/747 +[PR#753]: https://github.com/rpm-software-management/mock/pull/753 diff --git a/docs/Release-Notes-2.13.md b/docs/Release-Notes-2.13.md new file mode 100644 index 0000000..1cc5827 --- /dev/null +++ b/docs/Release-Notes-2.13.md @@ -0,0 +1,65 @@ +--- +layout: default +title: Release Notes 2.13 +--- + +Released on - 2021-11-02 + + +## New Mock 2.13 features: + +* A new option `--additional-package` is added. During package + development, this option can be used with `mock --rebuild` mode to specify + an additional set of build requirements (still, properly setting + `BuildRequires:` is a preferred way to achieve this) [[PR 776][PR#776]]. + +* A new option `--debug-config-expanded` is now available. It provides a very + similar mock configuration output to the `--debug-config` option, except that + the `{{ Jinja }}` constructs the configuration are expanded + [[PR 765][PR#765]]. + +## Mock 2.13 bugfixes: + +* The [`external:` dependencies](Feature-external-deps) are now properly + installed into a proper build chroot, not into a bootstrap chroot + [[PR 771][PR#771]]. + +* The option parsing mechanism was migrated from the `optparse` library to + `argparse`. This in particular shouldn't be a user visible change, so please + report changes in mock behavior if you observe any. + +* The repositories generated locally by mock are not automatically signed. But + since Mock did not specify the default `gpgpcheck=` option before, and some of + our config files didn't have `gpgcheck=0` in the `[main]` section, + DNF applied its own `gpgcheck=1` default and it led to `mock --chain` build + failures. Newly we set `gpgcheck=0` by default by Mock and any GPG signed + repository used in mock configuration needs to overwrite this explicitly + [[PR 782][PR#782]]. + +* When re-mounting, we newly don't specify the source of the mountpoint as it is + not needed in our case, and because the other (preferred) `mount --target ...` + variant is more portable (behaves correctly with older `util-linux` + implementations). [[issue 715][issue#715]] + +* The `distro.linux_distribution()` call is now deprecated, we use + `distro.id()` instead. [[PR 767][PR#767]] + +* Fixed LVM error message caused by copy/paste error [[PR 758][PR#758]]. + + +The following contributors contributed to this release: + + * Gustavo Costa + * Kamil Dudka + * Miroslav Suchý + * Sérgio M. Basto + +Thank you! + +[PR#758]: https://github.com/rpm-software-management/mock/pull/758 +[PR#765]: https://github.com/rpm-software-management/mock/pull/765 +[PR#767]: https://github.com/rpm-software-management/mock/pull/767 +[PR#776]: https://github.com/rpm-software-management/mock/pull/776 +[PR#782]: https://github.com/rpm-software-management/mock/pull/782 +[PR#771]: https://github.com/rpm-software-management/mock/pull/771 +[issue#715]: https://github.com/rpm-software-management/mock/issues/715 diff --git a/docs/Release-Notes-2.14.md b/docs/Release-Notes-2.14.md new file mode 100644 index 0000000..1f6107c --- /dev/null +++ b/docs/Release-Notes-2.14.md @@ -0,0 +1,15 @@ +--- +layout: default +title: Release Notes 2.14 +--- + +Released on - 2021-11-04 + + +## Mock 2.14 has just one regression fix: + +* The `--enablerepo` and `--disablerepo` options got broken in v2.13 by the + `optparse => argparse` rewrite. This should be fixed now, and the options + should be working fine. + +Thanks to Aleksei Bavshin for reporting the issue in Fedora Bodhi. diff --git a/docs/Release-Notes-2.15.md b/docs/Release-Notes-2.15.md new file mode 100644 index 0000000..21bf9a0 --- /dev/null +++ b/docs/Release-Notes-2.15.md @@ -0,0 +1,27 @@ +--- +layout: default +title: Release Notes 2.15 +--- + +Released on - 2021-11-18 + + +## Mock 2.15 contains just two bugfixes: + +* Mock v2.13 and v2.14 had a problem with old-style specified `chroot` and + `shell` mode (e.g. `--chroot` specified without leading dashes like `chroot`), + together with commands specified after the `--` separator. If used, Mock + misinterpreted the first part of the command to be executed; concretely, `--` + was considered to be a part of the command to be executed + [[rhbz#2024620][rhbz#2024620]]. + +* Fixed English grammar in `mock.1` [[PR#796][PR#796]]. + + +The following contributors contributed to this release: + + * Adam Williamson + * Cheese1 + +[rhbz#2024620]: https://bugzilla.redhat.com/2024620 +[PR#796]: https://github.com/rpm-software-management/mock/pull/796 diff --git a/docs/Release-Notes-2.16.md b/docs/Release-Notes-2.16.md new file mode 100644 index 0000000..bf47004 --- /dev/null +++ b/docs/Release-Notes-2.16.md @@ -0,0 +1,154 @@ +--- +layout: default +title: Release Notes 2.16 +--- + +Released on 2021-12-16. + +## Mock-core-configs 36.4 + +The biggest change is the removal of `epel-8-*` configs. It has been replaced by several configs: `alma+epel-8-*`, `centos+epel-8-*`, `oraclelinux+epel-8-*`, `rhel+epel-8-*`, `rocky+epel-8-*`. Every config has its pros and cons: + +* `alma+epel-8-*` - This uses Alma Linux 8 + Fedora EPEL. It works right off the bat and it is recommended replacement. The only disadvantage is that Koji actually uses RHEL + EPEL for EPEL builds. +* `centos+epel-8-*` - This uses CentOS 8 + Fedora EPEL. We do **not** recommend using this config, because [CentOS 8 will reach EOL on December 31, 2021](https://www.centos.org/centos-linux-eol/), and will be removed from mirrors on January 31, 2022. +* `oraclelinux+epel-8-*` - This uses Oracle Linux 8 + Fedora EPEL. +* `rhel+epel-8-*` - This uses RHEL 8 + Fedora EPEL. This is the configuration that Koji uses. But it [requires some settings](Feature-rhelchroots) and RHEL subscriptions. As a developer, you can have [16 subscriptions for free](https://developers.redhat.com/blog/2021/02/10/how-to-activate-your-no-cost-red-hat-enterprise-linux-subscription). +* `rocky+epel-8-*` - This uses Rocky Linux 8 + Fedora EPEL. It works right off the bat and it is recommended replacement. The only disadvantage is that Koji actually uses RHEL + EPEL for EPEL builds. + +There were [several options on how to handle EOL of epel-* configs](https://docs.google.com/document/d/1wF7-7_y6Ac_oB-kCFdE6VBWPW8o8zjXd2Z0SGy4VxUA/edit?usp=sharing) and we asked EPEL Steering Committee to decide and they [decided to remove epel-* config](https://pagure.io/epel/issue/133#comment-765381). So it is up to you to decide what to use. + +Mock will ease it and if you try to build for `epel-8-*` config and this config does not exist, you will get this message: + +``` +$ mock -r epel-8-x86_64 --shell +ERROR: Could not find required config file: /etc/mock/epel-8-x86_64.cfg +ERROR: There are those alternatives: +ERROR: +ERROR: [1] alma+epel-8-x86_64 +ERROR: Use instead: mock -r alma+epel-8-x86_64 --shell +ERROR: Builds against AlmaLinux 8 repositories, together with the official EPEL repositories. +ERROR: Project page: https://almalinux.org/ +ERROR: Enable permanently by: +ERROR: $ ln -s /etc/mock/alma+epel-8-x86_64.cfg /home/praiskup/.config/mock/epel-8-x86_64.cfg +ERROR: +ERROR: [2] centos+epel-8-x86_64 +ERROR: Use instead: mock -r centos+epel-8-x86_64 --shell +ERROR: Builds against CentOS Linux 8 repositories, together with the official EPEL repositories. +ERROR: This will reach end-of-life in January 2021. +ERROR: Enable permanently by: +ERROR: $ ln -s /etc/mock/centos+epel-8-x86_64.cfg /home/praiskup/.config/mock/epel-8-x86_64.cfg +ERROR: +ERROR: [3] rhel+epel-8-x86_64 +ERROR: Use instead: mock -r rhel+epel-8-x86_64 --shell +ERROR: Builds against Red Hat Enterprise Linux 8 repositories, together with the official EPEL repositories. +ERROR: This mimics what is done in the official EPEL build system, but you need a Red Hat subscription: +ERROR: https://rpm-software-management.github.io/mock/Feature-rhelchroots +ERROR: Enable permanently by: +ERROR: $ ln -s /etc/mock/rhel+epel-8-x86_64.cfg /home/praiskup/.config/mock/epel-8-x86_64.cfg +ERROR: +ERROR: [4] rocky+epel-8-x86_64 +ERROR: Use instead: mock -r rocky+epel-8-x86_64 --shell +ERROR: Builds against Rocky Linux 8 repositories, together with the official EPEL repositories. +ERROR: Project page: https://rockylinux.org/ +ERROR: Enable permanently by: +ERROR: $ ln -s /etc/mock/rocky+epel-8-x86_64.cfg /home/praiskup/.config/mock/epel-8-x86_64.cfg +``` + +Additional changes are: + + * Fedora 33 configs were moved to eol/ directory + * EOLed EPEL Playground configs, per [EPEL Steering Committee decision](https://pagure.io/epel/issue/136) + * Added configs for CentOS Stream 9 + EPEL Next 9 + * We expanded `dnf_vars` which cause an issue on EL7 hosts [RHBZ#2026571](https://bugzilla.redhat.com/show_bug.cgi?id=2026571) + * Added compatibility symlinks for EPEL 7 to centos+epel-7-* + * Resolved the multiple "local" repo problems + * Dropped rhel+epel-8-ppc64 config + * Added rhel+epel-8-s390x config + * Added navy-8-x86_64 config + * Reduced packages installed in EPEL chroots + +## Mock 2.16 changes: + +- Mock got a new configuration option: + + ``` + config_opts["no-config"]["epel-8"] = {} + config_opts["no-config"]["epel-8"]["alternatives"] = { + "alma+epel-8": { + "description": [ + "Builds against AlmaLinux 8 repositories, " + "together with the official EPEL repositories.", + "Project page: https://almalinux.org/" + ], + } + } + ``` + + When the configuration file for `epel-8-*` does not exist, it will print the text from the `description` field. + There is new file `/etc/mock/chroot-aliases.cfg` which contains defaults, but you can add your own option in your user config. + +- There was one issue with BSD Tar, which has been resolved [[GH#820](https://github.com/rpm-software-management/mock/pull/820)] + +- There is new option `ssl_extra_certs` [[GH#801](https://github.com/rpm-software-management/mock/pull/801)] + + ``` + config_opts['ssl_extra_certs'] = ['/etc/pki/tls/certs/client.crt', '/etc/pki/tls/certs/', + '/etc/pki/tls/private/client_nopass.key.crt', '/etc/pki/tls/private/'] + #config_opts['ssl_extra_certs'] = ['/path/on/host', '/path/in/mock/chroot', + # '/path/on/host2', '/path/in/mock/chroot2', ... ] + ``` + + It copies the host's SSL certificates into a specified location inside the chroot if + mock needs access to repositories requiring client certificate + authentication. Specify the full path to the public certificate on the host + and the destination directory in the chroot. Do the same for the private key. + The private key should not be password-protected if you want Mock to run + unattended. + +- For the "bootstrap_image" feature, we use `podman run` command to install + `dnf` stack into the bootstrap container. Prevously we cleaned-up the + environment for the Podman process which in turn caused DNF installation + problems on EL8 ([issue 831](https://github.com/rpm-software-management/mock/issues/831)) + +- We disabled `seccomp` filtering in `--isolation=nspawn` (the default). This + has been done to avoid build failures on hosts with stricter filters than on + the target chroot ([issue 811](https://github.com/rpm-software-management/mock/issues/831)) + +- Note **this is the last 2.x release** made from the `main` branch. After this + release, we will create a new branch, and future 2.x versions will get only + important bug fixes and important changes to the config. + + A version in `main` will be 3.x and will stop supporting EL7 as build *host*. + This will allow us to get rid of some compatibility code. However, we will + still support building **for** EL7. + + We plan to build 3.x for all supported Fedora versions and EPEL 8+. EPEL 7 + will stay on 2.x version. + + +## Currently known issues: + +- On Fedora 35+, there are [problems with the nosync.so plugin](https://bugzilla.redhat.com/show_bug.cgi?id=2019329). + Please, to avoid the problems, temporarily disable the nosync.so plugin. + +- The `subscription-manager` plugins breaks the DNF stack on Fedora, so when + installed even the normal DNF operations don't work. Updated packages (not + yet in Bodhi updates) help, [see the workaround](https://bugzilla.redhat.com/show_bug.cgi?id=1995465#c6). + + +**Following contributors contributed to this release:** + + * Adil Hussain + * Carl George + * Daniel Berteaud + * Istiak Ferdous + * Justin Vreeland + * Louis Abel + * Miroslav Suchý + * Neal Gompa + * Patrick Laimbock + * Pavel Raiskup + +Thank you. + + diff --git a/docs/Release-Notes-2.2.md b/docs/Release-Notes-2.2.md new file mode 100644 index 0000000..65bddee --- /dev/null +++ b/docs/Release-Notes-2.2.md @@ -0,0 +1,111 @@ +--- +layout: default +title: Release Notes 2.2 +--- + +Released on 2020-04-02. + +## Mock 2.2 new features: + + * `/etc/mock/site-defaults.cfg` was moved from /etc to %doc, and the + config file is now much smaller (and moved to `mock-core-configs`). + Even before the file was meant to be documentation-only (everything + commented-out), but since it was also configuration file - with + frequent updates in RPM - it was very easy to stop following what's new + there ([#555](../pulls/555)). + + * Mock no more strictly depends on `mock-core-configs` package, but depends on + `mock-configs` instead. Even though `mock-core-configs` package still + provides `mock-configs`, but other packages can as well, so users now can + provide alternatives to `mock-core-configs` ([#544](../pulls/544)). + + * New `config_opts['isolation']` option invented (alternative to + `--isolation`) to replace boolean `config_opts['use_nspawn']`. The + possible values are `nspawn`, `simple` and `auto` (default). When + `auto` is specified mock tries to use `nspawn` and if it is not + possible, it falls-back to `simple` chroot. This is useful to make + mock work by default in environments like Fedora Toolbox, Podman and + Docker. The old `use_nspawn` option still works, but `isolation` has + preference ([#337](../pulls/337) and [#550](../pulls/550)). + + * The `LANG` is set to `C.UTF-8` by default (and always) for chrooted + processes. Previously mock inherited this value from host environment, + and defaulted to `C.UTF-8` otherwise. This was done to make mock more + deterministic, users can change the default by + `config_opts['environment']['LANG']` ([#451](../issues/451)). + +## Mock 2.2 bugfixes: + + * Fix for doubled log entries in some situations ([#539](../pulls/539), + [RHBZ#1805631](https://bugzilla.redhat.com/1805631)). + + * Fix to make mock work in *Fedora Toolbox* even with + `--bootstrap-chroot` ([#550](../pulls/550)). + + * Fix for mock in `--privileged` docker container where `os.getlogin()` + did not work ([#551](../pulls/551)). + + * When `--bootstrap-chroot` is enabled, things like `rpm -qa --root ...` are + executed in bootstrap chroot, instead of on host. This is to assure that the + RPM used is compatible with target chroot RPMDB ([#525](../issues/525)). + + * The `mock --chroot -- CMD ARG1 ARG2` command was fixed so it works correctly + for both `--isolation=simple|nspawn` and `use_bootstrap=True|False`, the + caveats in `--shell` and `--chroot` are now better documented in manual + page ([#550](../pulls/550)). + + * Mock `--chain` with `--isolation=simple` was fixed to work with + external URLs ([#542](../pulls/542)). + + * Killing of forgotten chrooted processes was made more robust. We now + kill also "daemons" started on background during chroot initialization + -- when packages are installed to mock chroot and some package + scriptlet mistakenly spawns background process ([#183](../issues/183)). + + * The `--use-bootstrap-image` was fixed to work on EL7 properly + ([#536](../pulls/536)). + + * Stuff below `/tmp` is now passed down to mock chroot even + with `--isolation=nspawn` (default). Previously - everything mock prepared + below that directory was automatically overmounted by `systemd-nspawn`. + So newly, stuff like `--install /tmp/some.rpm` or repositories like + `file:///tmp/test-repo` will be correctly used through `--bootstrap-chroot`. + This fix requires new-enough `systemd-nspawn` which supports + `$SYSTEMD_NSPAWN_TMPFS_TMP` environment variable ([#502](../issues/502)). + + * Mock configuration; the host-local + `baseurl=file:///some/path/$basearch` repositories with dnf variables + inside were fixed for `--bootstrap-chroot` + ([RHBZ#1815703](https://bugzilla.redhat.com/1815703)). + + * Mock configuration; the host-local `metalink=file:///some/host/file` + (and mirrorlist) repositories were are fixed for bootstrap + ([RHBZ#1816696](https://bugzilla.redhat.com/1816696)). + + * With bootstrap, we use configured yum commands instead of hard-wired + `/usr/bin/yum` ([#518](../pulls/518)). + + * The `package_state` plugin was fixed to cleanup RPMDB before executing + `rpm -qa`. This broke builds on targets with incompatible RPMDB + backends before (e.g. OpenMandriva). + +## Mock-core-configs 32.6 + + * The `site-defaults.cfg` config file was moved from mock to + `mock-core-configs`. + + * The `config_opts['isolation']` is now used instead of + `config_opts['use_nspawn']`, when necessary. + + * We declare the minimal version of `mock` by `Requires:` now. At this + point it is version **2.2+**. + + * The default bootstrap image was specified for Amazon Linux conifgs. + +Following contributors contributed to this release: + + * Neal Gompa + * Owen W. Taylor + * Paul Howarth + +Thank you. diff --git a/docs/Release-Notes-2.3.md b/docs/Release-Notes-2.3.md new file mode 100644 index 0000000..20663fe --- /dev/null +++ b/docs/Release-Notes-2.3.md @@ -0,0 +1,38 @@ +--- +layout: default +title: Release Notes 2.3 +--- + +Released on 2020-05-22. + +## Mock 2.3 bugfixes: + + * The `--resultdir RESULTDIR` directory is bind-mounted to bootstrap chroot, so + we can use `--postinstalll` even with `--bootstrap-chroot`, + [#564](../issues/564), + + * easier configuration for `mount` plugin, [#578](../issues/578), + + * mock raises better error message when `%prep` script fails during + dynamic_biuldrequires resolution, [#570](../issues/570), + + * local mirrorlists are correctly bind-mounted to bootstrap chroot, + [RHBZ#1816696](https://bugzilla.redhat.com/1816696), + + * traceback for invalid `getresuid()` call, [#571](../issues/571), + + * use-cases with `--rootdir` and `--bootstrap-chroot` were fixed, [#560](../issues/560), + + * use bootstrap (not host's) `/bin/rpm` when producing list of installed packages, + [PR#568](../pulls/568), (pmatilai@redhat.com), + + * braced dnf variables are now expanded in repo URLs, + [PR#577](../pulls/577), (dmarshall@gmail.com). + +Following contributors contributed to this release: + + * David Marshall + * Neal Gompa + * Panu Matilainen + +Thank you. diff --git a/docs/Release-Notes-2.4.md b/docs/Release-Notes-2.4.md new file mode 100644 index 0000000..937a05b --- /dev/null +++ b/docs/Release-Notes-2.4.md @@ -0,0 +1,56 @@ +--- +layout: default +title: Release Notes 2.4 +--- + +Released on - 2020-07-21. + +## Mock 2.4 features: + + * The file `/dev/btrfs-control` is now available in chroot if host supports it. + This allows to create btrfs-based image builds. [[fedora-infra#9138](https://pagure.io/fedora-infrastructure/issue/9138)]. + + * Copy source CA certificates - + Prior to this change, we would only copy the "extracted" SSL CA + certificates into the chroot. If anything ran "update-ca-trust" inside + the chroot, this would delete our custom SSL certificates from the + "extracted" directory. For example, Fedora and RHEL's main + "ca-certificates" package always does this in %post, and any custom + third-party package could do this as well. + Copy the entire parent directory so that "sources" and "extracted" are + both present in the chroot. With this change, "update-ca-trust" + does not wipe out the CA certificates from the chroot. [[#588](../issues/588)] + + * Add `module_setup_commands` configuration option, The new config option + obsoletes `module_enable` and `module_install` configuration options (but + those are still supported), and allows users to also configure "disable", + "remove" and other commands. + + Each command can be specified multiple times, and mock respects the + order of the commands when executing them. + + Artificial example: (1) Disable any potentially enabled postgresql module + stream, (2) enable _specific_ postgresql and ruby module streams, + (3) install the development nodejs profile and (4) disable it immediately. + +``` + config_opts['module_setup_commands'] = [ + ('disable', 'postgresql'), + ('enable', 'postgresql:12, ruby:2.6'), + ('install', 'nodejs:13/development'), + ('disable', 'nodejs'), + ] +``` + +## Mock 2.4 bugfixes: + + * `.rpmmacros` is now created in "rootdir" instead of "basedir" + [[rhbz#1848201](https://bugzilla.redhat.com/1848201)] + +Following contributors contributed to this release: + + * Ken Dreyer + * Neal Gompa + * Pavel Raiskup + +Thank you. diff --git a/docs/Release-Notes-2.5.md b/docs/Release-Notes-2.5.md new file mode 100644 index 0000000..677e6fe --- /dev/null +++ b/docs/Release-Notes-2.5.md @@ -0,0 +1,48 @@ +--- +layout: default +title: Release Notes 2.5 +--- + +Released on - 2020-09-03. + +## Mock 2.5 features: + + * Since the introduction of `mock-configs` virtual provides, it can + happen that `mock-core-configs` is not actually installed. Previously, + the `mock` group would be missing though on such installation because + it was installed by the `mock-core-configs` package. Newly, both + `mock` and `mock-core-configs` depend on the new `mock-filesystem` + package that is responsible for installing both the `mock` system group + and some basic directory layout. + + * Mock newly configures the DNF so it sets a custom HTTP User Agent + header when downloading packages. This information can be later used + for better download statistics (e.g.normal end-user package downloads + vs. build-system downloads). + + * A new [showrc plugin](https://rpm-software-management.github.io/mock//Plugin-Showrc) was added. It puts the output of the command + `rpm --showrc` into a separate log file in result directory so users may + e.g. use this info during debugging the macro definition peculiarities. + + +## Mock 2.5 bugfixes: + + * Previously, when macro wasn't specified with leading `%` (see the + difference between `config_opts['macros']['foo'] = 'baz'` vs. + `config_opts['macros']['%foo'] = 'baz'`), mock on newer systems + (with new-enough Python 3.8+) failed hard with not really helpful + error. This has been fixed (issue#605). + +## Mock-core-configs v33 changes: + + * New Fedora ELN config files are provided. + + * Some adjustments were done for the new mock-filesystem package. + + +Following contributors contributed to this release: + + * Miroslav Suchý + * Pat Riehecky + +Thank you. diff --git a/docs/Release-Notes-2.6.md b/docs/Release-Notes-2.6.md new file mode 100644 index 0000000..0675ee1 --- /dev/null +++ b/docs/Release-Notes-2.6.md @@ -0,0 +1,48 @@ +--- +layout: default +title: Release Notes 2.6 +--- + +Released on - 2020-09-15. + +## Mock 2.6 new features: + + * The default `--rebuild` mode now supports `-a|--addrepo` option, as + well as the `--chain` did before, + [rhbz#1857918](bugzilla.redhat.com/1857918). + + * The default `--rebuild` mode now also accepts URLs pointing at source + RPMs. In previous versions mock only worked with local source RPMs. + The auto-downloading feature was previously available only in the + `--chain` mode. + +## Mock 2.6 bugfixes: + + * The configuration files inside buildroot are pre-configured + (or re-configured) by mock even if they are pre-installed by packages + as symbolic links, [rhbz#1878924](bugzilla.redhat.com/1878924). + + * Mock previously swallowed the 'rpm -i' error output when installing the + source RPM into chroot and failed. Newly the error output is printed to + stderr. + + * Each particular build failure reason in `--chain` build is now properly + dumped to stderr. + + * The `--chain` mode now fails right after the first build failure, as + it was previously documented in the manual page. To follow to the + other package builds, one has to specify `--continue`. + + * Mock creates `/etc/localtime` as a symlink even with isolation=simple + (per [fedora discussion](https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/thread/BNTFZH6VS43Q7FLRIZYSBOTKDK6KMQZQ/)). + + * When systemd-nspawn supports the `--resolv-conf=`, mock newly always + runs it with `--resolv-conf=off`. This is done to revert back the previous + expected name resolution behavior inside mock chroot (the new default + --resolv-conf=auto has broken it). + +The following contributors contributed to this release: + + * Miroslav Suchý + +Thank you. diff --git a/docs/Release-Notes-2.7.md b/docs/Release-Notes-2.7.md new file mode 100644 index 0000000..0c43010 --- /dev/null +++ b/docs/Release-Notes-2.7.md @@ -0,0 +1,89 @@ +--- +layout: default +title: Release Notes 2.7 +--- + +Released on - 2020-12-01. + +## Mock 2.7 new features: + + * [External (non-RPM) build requires](Feature-external-deps) proof of concept introduced. Initially, + there's only a support for PyPI and Crates packages. Any feedback and + patches (e.g. for other software providers) are welcome! It is disabled by default. It can be enabled using: + +``` +config_opts['external_buildrequires'] = True +``` + + and then you can use in SPEC files: + +``` +BuildRequires: external:pypi:foo +``` + + For more details see [feature page](Feature-external-deps). + + + + * There's a new plugin for pre-processing the input spec files; so the input + spec file "templates" are instantiated right before the source RPM build is + started. See the [plugin documentation](Plugin-rpkg-preprocessor) for more + info. + + * The full mock's NAME-VERSION-RELEASE string is now dumped to the log files, + it is now easier to understand what precise Mock version was used during + particular package build. + + * Added a new `postupdate` plugin hook; newly the Mock plugins can implement + the automatic "snapshoting" of the buildroot after any package update inside + chroot. This was now used by `root_cache` and `lvm_root` plugins and they + now newly udpate the buildroot cache after `dnf update` ([rhbz#1175346]). + + * Mock automatically copies the Katello CA pem file for the local Satellite + server into bootstrap chroot, if such CA is configured on host ([issue#638]). + +## Mock 2.7 bugfixes: + + * The `config_opts['resultdir']` path can contain `%`-sign, previous versions + of Mock failed on processing such configuration ([issue#639]). + + * The `--addrepo ` option newly doesn't fail the mock build when the + `` directory doesn't exist. This unifies the behavior of that + option because other errors/typos in the `--addrepo` option are ignored as + well. + + * Mock doesn't always traceback if the `rpmbuild` process exists with exit code + 11. That exit code only means that there are still some missing "dynamic + Buildrequires" (`%generate_buildrequires`) to be installed by Mock + ([issue#560]). We also enhanced the build.log output a bit so it is more + obvious what Mock installs on demand. + + * The bare `mock --shell` (login shell execution) was fixed so it doesn't call + `setsid()` prior executing the shell itself. This fixes the shell warning + message `Inappropriate ioctl for device`. + + * The `sign` plugin now treats the non-zero exit code from the configured + auto-sign command (usually some `rpmsign` wrapper). Previous versions of + Mock just ignored the failure ([koji#2570]). + + * Strange failure on RHEL 8 s390x issue fixed by removing one (probably + invalid) logging call from `preexec_fn`, but the [PR#653] still needs + proper fix (help is welcome). + +The following contributors contributed to this release: + + * Dominik Turecek + * Jiri Konecny + * Markus Linnala + * Merlin Mathesius + * Michal Novotný + * Miroslav Suchý + +Thank you! + +[rhbz#1175346]: https://bugzilla.redhat.com/1175346 +[issue#560]: https://github.com/rpm-software-management/mock/issues/650 +[issue#639]: https://github.com/rpm-software-management/mock/issues/639 +[PR#653]: https://github.com/rpm-software-management/mock/pull/653 +[issue#638]: https://github.com/rpm-software-management/mock/issues/638 +[koji#2570]: https://pagure.io/koji/issue/2570 diff --git a/docs/Release-Notes-2.8.md b/docs/Release-Notes-2.8.md new file mode 100644 index 0000000..56a901b --- /dev/null +++ b/docs/Release-Notes-2.8.md @@ -0,0 +1,24 @@ +--- +layout: default +title: Release Notes 2.8 +--- + +Released on - 2020-12-15. + +## Mock 2.8 bugfixes: + + * The systemd-nspawn wasn't used for --isolation=nspawn. This is regression + in the release v2.7. See [issue 678][issue#678]. + + * Better error message for certain `rmtree` failures, [PR 677][PR#677]. + +The following contributors contributed to this release: + + * Adam Williamson + * Miroslav Suchý + * Timm Bäde + +Thank you! + +[issue#678]: https://github.com/rpm-software-management/mock/issues/678 +[PR#677]: https://github.com/rpm-software-management/mock/pull/677 diff --git a/docs/Release-Notes-2.9.md b/docs/Release-Notes-2.9.md new file mode 100644 index 0000000..6fa8171 --- /dev/null +++ b/docs/Release-Notes-2.9.md @@ -0,0 +1,37 @@ +--- +layout: default +title: Release Notes 2.9 +--- + +Released on - 2021-01-18. + +## Mock 2.9 new features: + + * The `rpkg_preprocessor` plugin got new `force_enable` option. This option + tells rpkg_preprocessor to ignore rpkg.conf and always preprocess the spec + file. This is useful for testing mass package changes where it's not + practical to add an rpkg.conf to every package. + + * The configuration mechanism was cut-out from mock, and is now newly provided + as new package `python3-templated-dictionary` that mock package depends on. + +## Mock-core-configs v33.5 fixes: + + * EPEL 6 configuration was marked EOL (moved to eol/ subdirectory). + + * Fedora 31 configuration is EOL, too. + + * Fixed bootstrap of Fedora on Enterprise Linux 7 boxes. + + * Bootstrap images defined for OpenSUSE Tumbleweed. + + * RepoIDs renamed for EL8 chroots, according to real repoIDs in normal repo + files. + +The following contributors contributed to this release: + + * Miroslav Suchý + * Neal Gompa + * Tom Stellard + +Thank you! diff --git a/docs/Release-Notes-3.0.md b/docs/Release-Notes-3.0.md new file mode 100644 index 0000000..e1b1658 --- /dev/null +++ b/docs/Release-Notes-3.0.md @@ -0,0 +1,141 @@ +--- +layout: default +title: Release Notes - Mock v3.0 +--- + +Released on 2022-04-07. + +## Mock v3.0 changes: + +- Mock `v2.*` releases of Mock were supported on Enterprise Linux 7+. Since + this version `v3.0`, the prerequisite is Enterprise Linux 8 or newer. Mock for + the Enterprise Linux 7 is still supported in the [mock-2 branch][mock-2] + upstream, but it will only receive bug fixes. + + This only affects the Mock RPM installation, i.e. the **host** EL7 operating + system where Mock is run. **Building** packages for the **target** Enterprise + Linux 7 chroots continues to be supported in Mock v3.X. More info in the + [original issue][issue #755]. + +- The minimal runtime requirement now is **Python 3.6**. + +- Mock has a new command `--list-chroots` which prints the list of available + chroots with short descriptions ([PR#869][pull #869]). It will go through both + system-wide configuration files in `/etc/mock` and users' configuration in + `~/.config/mock/`. The output looks like: + + ``` + $ mock --list-chroots + INFO: mock.py version 2.16 starting (python version = 3.10.2, NVR = mock-2.16-1.git.3339.8f0b45e.fc35)... + Start(bootstrap): init plugins + INFO: selinux enabled + Finish(bootstrap): init plugins + Start: init plugins + INFO: selinux enabled + Finish: init plugins + INFO: Signal handler active + Start: run + config name description + Global configs: + alma+epel-8-aarch64 AlmaLinux 8 + EPEL + alma+epel-8-ppc64le AlmaLinux 8 + EPEL + alma+epel-8-x86_64 AlmaLinux 8 + EPEL + [..snip..] + rhel-8-x86_64 RHEL 8 + rocky+epel-8-aarch64 Rocky Linux 8 + EPEL + rocky+epel-8-x86_64 Rocky Linux 8 + EPEL + rocky-8-aarch64 Rocky Linux 8 + rocky-8-x86_64 Rocky Linux 8 + Custom configs: + + mockbuild.exception.ConfigError: Could not find included config file: /etc/mock/foohkhk + + fedora-50-x86_64 error during parsing the config file + fedora-rawhide-python39 Fedora Rawhide + Finish: run + ``` + In this example, the `fedora-50-x86_64` is a user's configuration file which + has some syntax issue(s). + +- There is a new function `mockbuild.config.simple_load_config(name)` available. + You should use it if you want to parse Mock's configuration files. The use is + as simple as: + + ``` + >>> from mockbuild.config import simple_load_config + >>> config_opts = simple_load_config("fedora-rawhide-x86_64") + >>> config_opts["resultdir"] + '/var/lib/mock/fedora-rawhide-x86_64/result' + ``` + +- the [hw_info plugin](Plugin-HwInfo) now reports utilization of volume with `cachedir` directory. + +- Source CA certificates found in `/usr/share/pki/ca-trust-source` are now + automatically copied from host to the target chroot, together with the + `/etc/pki/ca-trust` ([PR 864][pull #864]). + +- bash completion for `--scrub` and `--short-circuit` has been improved. + +- SECCOMP was disabled for `systemd-nspawn` before, and newly we disable it for + Podman commands by default, too ([PR 885][pull #885]). + + The SECCOMP rules (syscall allow-lists) maintained in those tools are often + different across distributions or even distro versions. Because Mock does + cross-distribution builds, the "host" distro rules are not always applicable + on the "target" distribution. To not complicate things, and because by design + Mock doesn't have to fully isolate, we disable SECCOMP for those + containerization tools by default. But if you want to enable it, you can now + do it using: + + ``` + config_opts["seccomp"] = True + ``` + +- Since the last release of Mock we have done a few mock-core-configs releases, + see below. + +## mock-core-configs-37-1 + +- EOL CentOS/EPEL 8 configs +- Add Fedora 36 +- drop failovermethod=priority from EL8 configs +- Add Extras repo for CentOS Stream 9 + +## mock-core-configs-37.1-1 + +- drop EL7 related hack +- link default.cfg file to the right EL N config +- Add CentOS Stream 8 + EPEL 8 configs + +## mock-core-configs-37.2-1 + +- Update CentOS Stream 9 Extras repo to use correct key +- Add AlmaLinux+EPEL 8 for POWER (ppc64le) +- Add AlmaLinux 8 for POWER (ppc64le) +- Deleted Fedora 37/Rawhide armhfp configs + +## mock-core-configs-37.3-1 + +* Provided 'epel-9' symlinks for 'fedpkg mockbuild' +* allow n-2 gpg key for Fedora ELN +* added new key `description` for `--list-chroots` command + + +**Following contributors contributed to this release:** + + * Derek Schrock + * Didik Supriadi + * Miro Hrončok + * Miroslav Suchý + * Neal Gompa + * Pavel Raiskup + * Tomas Tomecek + +Thank you. + + +[mock-2]: https://github.com/rpm-software-management/mock/tree/mock-2 +[issue #755]: https://github.com/rpm-software-management/mock/issues/755 +[pull #864]: https://github.com/rpm-software-management/mock/pull/864 +[pull #885]: https://github.com/rpm-software-management/mock/pull/885 +[pull #869]: https://github.com/rpm-software-management/mock/pull/869 diff --git a/docs/Release-Notes-3.1.md b/docs/Release-Notes-3.1.md new file mode 100644 index 0000000..c2e0152 --- /dev/null +++ b/docs/Release-Notes-3.1.md @@ -0,0 +1,77 @@ +--- +layout: default +title: Release Notes - Mock v3.1 +--- + +Released on 2022-07-22. + +## Mock v3.1 changes: + +- There's a fix for a new RPM behavior on F37+ where `rpmbuild` automatically + cleans the `%buildroot` directory upon a successful build. This behavior is + not desired when Mock user wants to keep the buildroot contents for further + debugging (`config_opts["cleanup_on_success"] = False` is + configured, or `--no-cleanup-after` option is used). [Original bug + report.][rhbz#2105393] + +- Mock v3.1+ started using `/bin/tar` instead of `/bin/gtar` for normal work + with archives. This default can be changed by a new option + `config_opts["tar_binary"]`. This should help with distributing Mock to + GNU/Linux distributions where `/bin/gtar` symbolic link doesn't exist. + +- Mock v3.1+ *still* expects that that the default system Tar represents + a GNU tar implementation (unless `config_opts["tar"] = 'bsdtar'`). Mock v3.1 + version though got several fixes that make the work with the BSD tar + implementation more convenient. + + +## mock-core-configs-37-4-1 + +* Add AlmaLinux 9 and AlmaLinux 9 + EPEL configs (neal@gompa.dev) +* Update the AlmaLinux 8 GPG key path (neal@gompa.dev) +* Fix description typo on AlmaLinux 8 for x86_64 (neal@gompa.dev) +* Add RHEL9 templates and configs (carl@george.computer) + + +## mock-core-configs-37.5-1 + +* configs: add ELN local Koji repo +* config: sync epel-8 and epel-9 templates +* Add Rocky Linux 9 Configuration and Mod RL8 (label@rockylinux.org) +* Update Fedora ELN repo template (sgallagh@redhat.com) +* EuroLinux 9 chroot configs added (git@istiak.com) +* Fedora 34 is EOL +* circlelinux+epel-8 as epel-8 alternative +* Fix dist value for openSUSE Leap 15.4 (ngompa@opensuse.org) +* Add CircleLinux 8 configs (bella@cclinux.org) +* Add openSUSE Leap 15.4 configs (ngompa@opensuse.org) +* Move openSUSE Leap 15.2 to EOL directory (ngompa@opensuse.org) +* Use MirrorCache for openSUSE repositories instead of MirrorBrain (ngompa@opensuse.org) +* Add Anolis OS 7 and Anolis OS 8 templates and configs (wb-zh951434@alibaba-inc.com) + + +**Following contributors contributed to this release:** + + * babakovalucie + * Bella Zhang + * Carl George + * DominikaMarchovska + * Istiak Ferdous + * JeremiasVavak + * katerin71 + * Louis Abel + * Miroslav Suchý + * naveen + * Neal Gompa + * Nico Kadel-Garcia + * Papapja + * PastelBrush + * SpiderKate + * Stephen Gallagher + * terezakoktava + * Zhao Hang + +Thank you. + + +[rhbz#2105393]: https://bugzilla.redhat.com/show_bug.cgi?id=2105393 diff --git a/docs/Release-Notes-3.2.md b/docs/Release-Notes-3.2.md new file mode 100644 index 0000000..598296f --- /dev/null +++ b/docs/Release-Notes-3.2.md @@ -0,0 +1,61 @@ +--- +layout: default +title: Release Notes - Mock v3.2 +--- + +Released on 2022-10-14. + +## Mock v3.2 changes: + +- The `cleanup_on_success=False` needs to use `rpmbuild --noclean` to avoid + an automatic cleanup of `%{buildroot}` by RPM (we started using `--noclean` in + [v3.1](Release-Notes-3.1). The `--noclean` option is though not available on + old systems (EL6 and older). The new Mock *v3.2* fixes this, and stops using + `--noclean` for old target builds. Related [rhbz#2105393][]. + +- The `mock --list-chroots` now performs much faster; the amount of `fork()` + calls while listing the chroot configuration files was minimized. + +- Files installed into `/var/lib/mock` and `/var/cache/mock` no longer get the + SGID bit (which enforces `mock` group ownership). The bit shouldn't be needed + (by common sense, but also given our testsuite is green), so we dropped the + bit. Should you newly have any inconvenience, please report. + +- The `simple_load_config()` library method (used e.g. by Fedora Review) was + simplified so it doesn't attempt to elevate the `mock` group process + ownership. Processing Mock's configuration is equal to evaluating a Python + code, so for security reasons we artificially fail the `simple_load_config()` + call if executed by root, [rhbz#2080735][]. + +- Error() (Exception) code was rewritten a bit, so the raised exceptions are + easily (de-)serializable by the Python pickle library. We can then e.g. + naturally handle the exceptions raised by Mock's `fork()` sub-processes + (exceptions from child processes are pickle-deserialized). This change could + potentially affect the `mockbuild:get_class_by_code()` users, if any. + +- Mock now better detects the Docker environment run on top of + Control Group V2, [PR#986][]. + +- The `--use-bootstrap-image` now works even if `podman` produces an unexpected + `stderr` output, [PR#954][]. + +- The `mock-scm` package newly runtime-depends on the `rpkg-util` package. This + package is used for building source RPMs from DistGit, fixes [rhbz#2128212][]. + +- Mock starts using the SPDX format of License field in the spec file. + + +**Following contributors contributed to this release:** + + * Achal Velani + * Michael Ho + * Miroslav Suchý + +Thank you. + + +[rhbz#2105393]: https://bugzilla.redhat.com/2105393 +[PR#954]: https://github.com/rpm-software-management/mock/pull/954 +[rhbz#2128212]: https://bugzilla.redhat.com/2128212 +[rhbz#2080735]: https://bugzilla.redhat.com/2080735 +[PR#986]: https://github.com/rpm-software-management/mock/pull/986 diff --git a/docs/Release-Notes-3.3.md b/docs/Release-Notes-3.3.md new file mode 100644 index 0000000..09c65c6 --- /dev/null +++ b/docs/Release-Notes-3.3.md @@ -0,0 +1,15 @@ +--- +layout: default +title: Release Notes - Mock v3.3 +--- + +Released on 2022-10-17. + +## Mock v3.3 bugfixes: + +The fix for `--list-chroots` and `simple_load_config()` from +[v3.2](Release-Notes-3.2) disallowed running Mock under the `root` user. Since +this is still a common practice [issue#990][], we relaxed the rule and we only +raise a warning if `simple_load_config()` is executed by `root`. + +[issue#990]: https://github.com/rpm-software-management/mock/issues/990 diff --git a/docs/Release-Notes-3.4.md b/docs/Release-Notes-3.4.md new file mode 100644 index 0000000..333a9df --- /dev/null +++ b/docs/Release-Notes-3.4.md @@ -0,0 +1,30 @@ +--- +layout: default +title: Release Notes - Mock v3.4 +--- + +Released on 2022-11-15. + +## News in Mock v3.4: + +- For cross-arch builds (see manual page for the `--forcearch` option), Mock + tries to detect if the (potentially missing) `qemu-user-static` package is + installed. Since Fedora 37, the package got split into a set of packages + arch-specific packages (like `qemu-user-static-x86`, `qemu-user-static-ppc`, + etc.). The Mock v3.4 does a better check, and raises more useful error if the + package is missing. + +- Mock newly provides the `/dew/mapper/control` file in chrot, so users may + control the device mapper. This has been requested by + the [Koji team][kojipr#3585] while implementing Kiwi support [PR#1005][]. + + +**Following contributors contributed to this release:** + + * Miroslav Suchý + * Neal Gompa + +Thank you. + +[kojipr#3585]: https://pagure.io/koji/pull-request/3585 +[PR#1005]: https://github.com/rpm-software-management/mock/pull/1005 diff --git a/docs/Release-Notes-3.5.md b/docs/Release-Notes-3.5.md new file mode 100644 index 0000000..dada0d6 --- /dev/null +++ b/docs/Release-Notes-3.5.md @@ -0,0 +1,24 @@ +--- +layout: default +title: Release Notes - Mock v3.5 +--- + +Released on 2022-12-01. + +## News in Mock v3.5: + +- For cross-arch builds (see manual page for the `--forcearch` option), Mock + tries to detect if the (potentially missing) `qemu-user-static` package is + installed. Since Fedora 37, the package got split into a set of packages + arch-specific packages (like `qemu-user-static-x86`, `qemu-user-static-ppc`, + etc.). The Mock v3.4 does a better check, and raises more useful error if the + package is missing. + + In Mock v3.5 we further enhanced the related error message(s). + + We also fixed a bug in the detection mechanism — mock no longer fails-hard for + a missing architecture (not configured in + `config_opts['qemu_user_static_mapping']`). Mock in such situation newly just + tries its best and continues the build, even though failure is likely. + +[PR#1007]: https://github.com/rpm-software-management/mock/pull/1007 diff --git a/docs/Release-Notes-39.2.md b/docs/Release-Notes-39.2.md new file mode 100644 index 0000000..c7aadb3 --- /dev/null +++ b/docs/Release-Notes-39.2.md @@ -0,0 +1,20 @@ +--- +layout: default +title: Release Notes - Mock configs 39.2 +--- + +### Mock Core Configs changes + +- The set of GPG keys used for openSUSE Leap 15.5 was updated to include + the correct key used for the openSUSE Backports repository. +- Previous versions of mock-core-configs referenced `fedora:latest` bootstrap + images for `fedora-eln-ARCH` chroots, which led to preparation errors + caused by [package incompatibilities][issue#1238]. ELN folks though already + provide a better and + ["native" ELN image](https://docs.fedoraproject.org/en-US/eln/deliverables/#_container_image), + so the new Mock configs have been switched to use it. +- The `/etc/mock/default.cfg` link installation [has been fixed][pull#1236] for + Fedora ELN. + +[issue#1238]: https://github.com/rpm-software-management/mock/issues/1238 +[pull#1236]: https://github.com/rpm-software-management/mock/pull/1236 diff --git a/docs/Release-Notes-4.0.md b/docs/Release-Notes-4.0.md new file mode 100644 index 0000000..6ab93ff --- /dev/null +++ b/docs/Release-Notes-4.0.md @@ -0,0 +1,94 @@ +--- +layout: default +title: Release Notes - Mock v4.0 +--- + +Released on 2023-05-22. + +## Mock v4.0 new features: + +- The RPM Software Management team(s) work hard on the [DNF5][] project, which + is planned to be the [default package manager in F39][]. Compared to [DNF4][], + DNF5 is a from-scratch rewritten software, implying that features and + command-line options might be implemented differently. That's why Mock needed + a special logic to support it. While DNF5 is still not the default at this + moment, Mock 4+ supports it and allows users to experiment: + + ``` + $ mock -r fedora-rawhide-x86_64 --config-opts=package_manager=dnf5 --shell + ``` + + When used like this, Mock installs DNF5 package manager into the bootstrap + chroot first (using DNF5 itself, if found on host, or just using DNF4). + Later, using DNF5 from bootstrap, installs the target Rawhide buildroot. + +- The [--use-bootstrap-image](Feature-container-for-bootstrap) feature, + implemented using the containerization [Podman][] command-line tooling, did + not work correctly if Mock itself was run + [in container](index#mock-inside-podman-fedora-toolbox-or-docker-container). + At the time of releasing Mock 4.0, running nested Podman containers still + requires quite a lot of + [configuration done in the image](https://github.com/containers/podman/blob/36510f6/contrib/podmanimage/stable/Containerfile). + So the requirements were relaxed to not run Podman containers, but only + extract the container images using the [podman image mount][PR#1073] feature. + So now, the `--use-bootstrap-image` feature works if Mock is run in Podman. + +- Mock historically called the `useradd` utility with `-n` option to not create + the default `mock` group in the chroot. The `-n` option has been a Red Hat + Enterprise Linux downstream patch, later implemented upstream as `-N`. The + `-N` option is now supported almost everywhere (since RHEL 6+). If you build + for older chroots than Enterprise Linux 6 (EOL nowadays), you might need to + modify the `config_opts["useradd"]` option. + +## Mock v4.0 bugfixes: + +- The "essential" mount-points (`/proc`, `/sys`, ..) were not correctly mounted + to the target buildroot at the time of its installation/initialization (when + package manager from bootstrap chroot is used to install the buildroot + packages). This wasn't very obvious, because, during the later phases of Mock + builds, Mock had those essential mount points mounted. This caused issues + with the installation of packages that relied on their existence, see + [rhbz#2166028]. + +- Before killing the leftover in-chroot processes, older Mock versions first + unmounted (well at least it tried) the mounted filesystems in the chroot. + This has been fixed, and Mock does it vice-versa so both unmounting itself is + less likely to have problems and killing the processes is easier. + +- Mock ignored the `bootstrap_` prefixed `config_opts` options, especially + useful on commandline for debugging (e.g. + `--config-opts=bootstrap_chroot_additional_packages=package-foo`). The + configuration option logic + [was adjusted](https://github.com/rpm-software-management/mock/commit/8bd4adcaa197af4a7b6a915a01484c51d1c1cc5b) + to fix this problem. + +- The manual page of Mock was fixed so users are now instructed to fill issues + against the GitHub upstream [Mock project](https://github.com/rpm-software-management/mock/issues), + not the Red Hat Bugzilla. + +## mock-core-configs-38.5-1 + +- Fedora 35 and 36 are now EOL, so the configuration was moved +- The `includepkgs=devtoolset*` options were + [dropped](https://github.com/rpm-software-management/mock/pull/1042) from the + SCL-related CentOS 7 configuration. This allows the installation of other SCL + packages during package build (specified by `BuildRequires:`). +- The `useradd` override configuration was removed as it is not needed now, + Mock v4.0 now uses `useradd -N` (not `useradd -n`) by default. +- The openSUSE i586 repos have been moved out of the main repos into a port. + +**Following contributors contributed to this release:** + + * @cheese1 + * @lilinjie + * Miroslav Suchý + +Thank you. + + +[Podman]: https://podman.io/ +[DNF5]: https://github.com/rpm-software-management/dnf5 +[DNF4]: https://github.com/rpm-software-management/dnf +[PR#1073]: https://github.com/rpm-software-management/mock/pull/1073 +[default package manager in F39]: https://fedoraproject.org/wiki/Changes/ReplaceDnfWithDnf5 +[rhbz#2166028]: https://bugzilla.redhat.com/show_bug.cgi?id=2166028 diff --git a/docs/Release-Notes-4.1.md b/docs/Release-Notes-4.1.md new file mode 100644 index 0000000..08edac7 --- /dev/null +++ b/docs/Release-Notes-4.1.md @@ -0,0 +1,50 @@ +--- +layout: default +title: Release Notes - Mock v4.1 +--- + +Released on 2023-06-05. + +## Mock v4.1 new features: + +- The `/bin/dnf` path can be either provided by [DNF5][] on newer systems + ([Fedora 39+][default package manager in F39]), or by [DNF4][] on older + systems. The detection of `/bin/dnf` though wasn't ideal. Newly, if [DNF4][] + is requested, Mock searches for the `/bin/dnf-3` script instead. Also, when + installing [DNF4][] into a bootstrap chroot, `python3-dnf` is installed + instead of just `dnf` which might install [DNF5][]. + +- We newly allow installing the bootstrap chroot using `/bin/dnf5` as fallback, + if the requested package manager is not found on host (e.g. if + `package_manager=dnf` is set for particular chroot, but only [DNF5][] is + available on host, i.e. the future systems). Previous version of Mock would + just fail verbosely. + +- The `mock.rpm` runtime dependencies were changed and relaxed. We newly don't + strictly require any of the package managers. Having `dnf5` or `python3-dnf` + installed on host is just a `Suggested` thing, and it is newly up to the user + to install one of them (on Fedora 39+, [DNF5][] will be more commonly the + choice). Strictly speaking, with the `--use-bootstrap-image` feature, no + package manager on host is needed at all. + +- We use the same package manager search logic for bootstrap, non-bootstrap or + bootstrap image use-cases. + +## Mock v4.1 bugfixes: + +- The Mock v4.0 broken chroot configurations with custom SSL certificates and + bootstrap (the certificates were not copied into the bootstrap chroot + correctly). This problem [has been fixed][issue#1094]. + +- The `bind_mount` plug-in newly pre-creates the destination directory in-chroot + for bind-mounted files. See [PR#1093][] for more info. + +- The --dnf-cmd option was fixed for the revamped `package_manager` detection + logic. See [PR#1087][] for more info. + +[default package manager in F39]: https://fedoraproject.org/wiki/Changes/ReplaceDnfWithDnf5 +[PR#1087]: https://github.com/rpm-software-management/mock/pull/1087 +[PR#1093]: https://github.com/rpm-software-management/mock/pull/1093 +[issue#1094]: https://github.com/rpm-software-management/mock/issues/1094 +[DNF4]: https://github.com/rpm-software-management/dnf +[DNF5]: https://github.com/rpm-software-management/dnf5 diff --git a/docs/Release-Notes-5.0.md b/docs/Release-Notes-5.0.md new file mode 100644 index 0000000..b68cbbd --- /dev/null +++ b/docs/Release-Notes-5.0.md @@ -0,0 +1,213 @@ +--- +layout: default +title: Release Notes - Mock v5.0 +--- + +Released on 2023-08-09. + +## Mock v5.0 new features: + +- The `use_bootstrap_image` feature has been turned on as it was stabilized in + this release and it speeds up the bootstrap chroot preparation a lot + (by not installing potentially hundreds of packages into the bootstrap chroot, + but just extracting the `bootstrap_image` contents). For the RHEL based chroots + (where UBI `bootstrap_image` is in use, and where `python3-dnf-plugins-core` + is installed by default) we also turned on the `bootstrap_image_ready=True` + which drastically speeds the bootstrap preparation. See the explanation for + `bootstrap_image_ready` below. See [PR#1101][] for more info. + +- Start using `useradd` and `userdel` utilities from the host to modify + in-chroot `etc/passwd` and `etc/group`. This allowed us to remove + the `shadow-utils` package from the list of "minimal buildroot" packages + ([issue#1102][]). + +- There's a new feature `config_opts['copy_host_users'] = ['pesign', ...]` that + allows users to specify a list of users to pre-create in buildroot, copying + the IDs from the host. Users/groups used to be historically created by + post-scriptlets from the buildroot packages, but the user/group IDs often did + not match the IDs on the host - possibly causing weird and hard to fix issues + similar to the [Pesign one][issue#1091]. + +- Make sure the `/dev/fuse` device is available in Mock buildroot with + `--isolation=nspawn`. Without this device, we could not use FUSE filesystems + with image builds ([PR#1158][] and the [Fedora Infra issue][fuseInfra]). + +- New option `config_opts['bootstrap_image_ready'] = True|False` was invented. + When set to `True`, Mock doesn't try to do any package manager actions inside + bootstrap (normally Mock installs `dnf` tooling there). It simply assumes + that the bootstrap image is ready to use (that the `dnf` tooling is already + installed on the image). This might fix hard-to-debug/hard-to-fix issues like + [issue#1088][] when an unexpected bootstrap image is used. This option is + enabled by default for RHEL 8 and 9 chroots only. + +- The Podman container environment is able to automatically pass down the Red + Hat Subscription credentials anytime a RHEL-based (UBI) container is run. But + the credential files are then on different path than when run natively on + the host, and that used to break Mock. Newly, when Mock is run in a Podman + container, the credentials passed down by Podman are automatically detected + and used ([PR#1130][]). + +- Even though the requested buildroot is cross-arch (must be initialized with + cross-arch packages using the `--forcearch RPM` feature), the bootstrap chroot + is newly always prepared with the native architecture. This change has been + done to optimize the final buildroot installation — using the "arch natively" + compiled DNF stack is much faster than cross-arch emulation using + `qemu-user-static`. As a benefit, we can newly simply use the "native" + Podman-pulled `bootstrap_image` even for cross-arch builds (we have to play + with Glibc's "personality" a bit, but still). See [issue#1110][]. + +- The bootstrap-related logging has been changed so the corresponding log + entries are now appended to the default `root.log` file. This change should + lead to a better understanding what is going on in the bootstrap chroot + ([issue#539][] and [PR#1106][]). + +- An easier way to skip Mock plugin execution in Bootstrap chroot has been + invented. It is now enough to just specify `run_in_bootstrap = False` global + variable in such a plugin, see [PR#1116][] for more info. The plugin + `hw_info` has been newly disabled this way. + +- The bootstrap chroot installation was made smaller; newly only the + `python3-dnf` and `python3-dnf-plugins-core` are installed, instead of `dnf` + and `dnf-plugins-core` (which used to install also unnecessary documentation + files). + +- Automatic file downloads (for example with `mock https://exmaple.com/src.rpm` + use cases) have been changed to automatically retry the download several times + to work around random network failures ([PR#1132][] and [PR#1134][]). + +- The `dnf` and `dnf5` processes are newly always executed with the + `--setopt=allow_vendor_change=yes` option. This is done in belief that we + don't have to protect the Mock builds against `Vendor:` changes, while + overriding the distro-default RPMs is quite a common thing (`mock --chain` + e.g.) while mimicking the distro-default `Vendor` tag would be a painful task. + The thing is that the `allow_vendor_change=no` is going to be set by default + in DNF5 soon and we want to prevent unnecessary surprises. Also, openSUSE has + been reported to use this even with DNF4 ([PR#1160][]). + +- Mock now considers DNF5 to be a valid package manager alternative. For + example considering + + 1. that Bootstrap chroot is generated from a bootstrap image, + 2. expected package manager is DNF4 (`package_manager == 'dnf'`) + 3. but the selected image has only `dnf5` installed by default + + then Mock properly finds the `/bin/dnf5` command in-bootstrap, uses it to + install `/bin/dnf-3` into the bootstrap chroot first and then it uses + `/bin/dnf-3` in bootstrap for the final target buildroot installation. + + +## Mock v5.0 bugfixes: + +- The orphan-process killing feature was enhanced to also properly kill + processes started by `dnf install` post-scriptles (installing from bootstrap + to buildroot), see [issue#1165][]. + +- The `podman pull` logic that Mock does in the background with + `--use-bootstrap-image` (now by default) was fixed to not affect the + `mock --shell` stdout. That was a bug, stdout is often parsed by callers. + +- The fact that we use in-bootstrap package manager for *installation stuff into + the bootstrap chroot itself* revealed that we have to do certain mountpoints + [earlier than we used to][PR#1167]. + +- Make sure we properly unmount all the Mock mount internal points even though + the Mock process was interrupted using `CTRL+C` (`KeyboardInterrupt` + exception). This has fixed a long-term observed bug that kept things mounted + longer than necessary, eventually breaking even subsequent `mock --scrub` + attempts. Also, the internal directory removal method has been fixed to + *try its job harder* under [certain circumstances][PR#1058]. Relates to + [rhbz#2176689][], [rhbz#2176691][], [rhbz#2177680][], [rhbz#2181641][], etc. + +- Python `imp` module was removed from Python 3.12, so the code was migrated to + `importlib`, [issue#1140][]. + +- Automatic SRPM downloads from the web were handling the file names specified + by HTTP headers using the long-time deprecated `cgi` module. The module is + being dropped from Python v3.13 so Mock 5.0 has been fixed to use + `email.message` library which now provides the same functionality + ([PR#1134][]). + +- The default recursion limit for Python scripts is set to 1000 (for non-root + users); this hasn't been enough for Mock in some use-cases so the limit has + been increased in v5.0+. For example, some utilities have directory-tree + stress-tests their test suites, and for such cases a very large directory tree + can cause too deep recursive calls of the `shutil.rmtree()` method. Users + can newly also override the Mock's default via the + `config_opts["recursion_limit"]` option. + +- Mock newly never uses the `--allowerasing` option with the `dnf5 remove` + command (this option has not been implemented in DNF5 and DNF5 simply fails, + contrary to the old DNF4 code where it was implemented a no-op, + [issue#1149][]). + +- The SSL certificate copying has been fixed [once more][PR#1113] to use our own + `update_tree()` logic because the `distutils.copy_tree()` was removed from the + Python stdlib, and the new stdlib alternative `shutil.copytree()` is not + powerful enough for the Mock use-cases ([issue#1107][]). + +- Mock no longer dumps the long output of the `rpmbuild --help` command into + `build.log`, fixes [issue#999][]. + +## mock-core-configs v39 changes: + +- Fedora 39 configuration is branched from Rawhide, as branching date + is [2023-08-08](https://fedorapeople.org/groups/schedule/f-39/f-39-all-tasks.html). + +- The new `mock-core-configs` package is not compatible with older Mock + versions, therefore the requirement was bumped to `v5.0` or newer. + +- Started using `bootstrap_image_ready = True` for RHEL-based configs as UBI + images contain all the necessary DNF tooling by default ([PR#1101][]). + +- Disable `use_bootstrap_image` for the Mageia chroots; Mageia doesn't provide + an officially supported (and working) container images that we could set in + the `bootsrap_image` config option ([issue#1111][]). + +- Dropped the `config_opts['useradd']` option from all the configs; the + `useradd` utility from now on always executed on the host (not in chroot) with + the `--prefix ` options ([issue#1102][]). + +- Using `$releasever` in openEuler metalink URLs. + +- Several configuration files were updated to work correctly with the new + `use_bootstrap_image = True` in Mock 5.0. Relates to [issue#1171][] + + +**Following contributors contributed to this release:** + + * Miro Hrončok + * Miroslav Suchý + * Neal Gompa + * zengwei2000 + * zengchen1024 + +Thank you. + +[PR#1058]: https://github.com/rpm-software-management/mock/pull/1058 +[PR#1101]: https://github.com/rpm-software-management/mock/pull/1101 +[PR#1106]: https://github.com/rpm-software-management/mock/pull/1106 +[PR#1113]: https://github.com/rpm-software-management/mock/pull/1113 +[PR#1116]: https://github.com/rpm-software-management/mock/pull/1116 +[PR#1130]: https://github.com/rpm-software-management/mock/pull/1130 +[PR#1132]: https://github.com/rpm-software-management/mock/pull/1132 +[PR#1134]: https://github.com/rpm-software-management/mock/pull/1134 +[PR#1158]: https://github.com/rpm-software-management/mock/pull/1158 +[PR#1158]: https://github.com/rpm-software-management/mock/pull/1160 +[PR#1167]: https://github.com/rpm-software-management/mock/pull/1167 +[issue#539]: https://github.com/rpm-software-management/mock/issues/539 +[issue#999]: https://github.com/rpm-software-management/mock/issues/999 +[issue#1088]: https://github.com/rpm-software-management/mock/issues/1088 +[issue#1091]: https://github.com/rpm-software-management/mock/issues/1091 +[issue#1102]: https://github.com/rpm-software-management/mock/issues/1102 +[issue#1107]: https://github.com/rpm-software-management/mock/issues/1107 +[issue#1110]: https://github.com/rpm-software-management/mock/issues/1110 +[issue#1111]: https://github.com/rpm-software-management/mock/issues/1111 +[issue#1140]: https://github.com/rpm-software-management/mock/issues/1140 +[issue#1149]: https://github.com/rpm-software-management/mock/issues/1149 +[issue#1165]: https://github.com/rpm-software-management/mock/issues/1165 +[issue#1171]: https://github.com/rpm-software-management/mock/issues/1171 +[rhbz#2176689]: https://bugzilla.redhat.com/2176689 +[rhbz#2176691]: https://bugzilla.redhat.com/2176691 +[rhbz#2177680]: https://bugzilla.redhat.com/2177680 +[rhbz#2181641]: https://bugzilla.redhat.com/2181641 +[fuseInfra]: https://pagure.io/fedora-infrastructure/issue/11420 diff --git a/docs/Release-Notes-5.1.1.md b/docs/Release-Notes-5.1.1.md new file mode 100644 index 0000000..9cede07 --- /dev/null +++ b/docs/Release-Notes-5.1.1.md @@ -0,0 +1,29 @@ +--- +layout: default +title: Release Notes - Mock 5.1.1 +--- + +Released on 2023-09-18. + +## Mock 5.1.1 bugfixes + +- [commit#1e13b56ce3c0efdf81][] caused "basedir" to be created only once per Mock + run, but likewise directory "rootdir" was created only once. + + Since Mock automatically unmounts rootdir **after each build** and then + also **removes the rootdir** directory to finish the cleanup tasks (at + least if tmpfs or other "root" plugin is in use, --resultdir is in + use, ...), subsequent builds failed to re-mount the rootdir with, e.g.: + + ERROR: Command failed: + $ mount -n -t tmpfs -o mode=0755 -o nr_inodes=0 -o size=140g mock_chroot_tmpfs /var/lib/mock/fedora-37-x86_64-1694797505.326095/root + + This caused problems e.g. [in Fedora Copr][copr_issue#2916] where each + Mock build is actually a two-step build done like: + + mock --spec foo.spec --sources . --resultdir ... + + So Mock first builds SRPM, and then builds RPMs (two builds in one run). + +[commit#1e13b56ce3c0efdf81]: https://github.com/rpm-software-management/mock/commit/1e13b56ce3c0efdf81 +[copr_issue#2916]: https://github.com/fedora-copr/copr/issues/2916 diff --git a/docs/Release-Notes-5.1.md b/docs/Release-Notes-5.1.md new file mode 100644 index 0000000..d19a699 --- /dev/null +++ b/docs/Release-Notes-5.1.md @@ -0,0 +1,156 @@ +--- +layout: default +title: Release Notes - Mock v5.1 and mock-core-configs v39.1 +--- + +Released on 2023-09-15. + + +## New 5.1 features + +- We [implemented a convenience fallback][PR#1200] from **bootstrap-from-image** + to the slower **bootstrap-installed-DNF-from-host** for the cases when Podman + can not be used properly (when container image can not be pulled, image can + not be mounted, image architecture mismatch, Podman is not available or not + working - e.g. if run in non-privileged Docker, etc). + + There's also a new ["podman pull" backoff logic][commit#395fc07f796] that + makes Mock to retry Podman pulling for 120s (default). Feel free to adjust this + timeout by `config_opts["bootstrap_image_keep_getting"]` option. +- Mock [newly logs out][PR#1210] the package management toolset versions (e.g. + version of DNF, RPM, etc.) that is used for the buildroot installation. This + is a feature helping users to diagnose problems with buildroot installation + (minimal buildroot, `BuildRequires`, dynamic build requires, etc.). It might + seem like a trivial addition, but sometimes it isn't quite obvious where the + tooling comes from (is that from host? from bootstrap? was it downloaded + "pre-installed" with bootstrap image?). +- There's a [new "INFO" message][commit#8c7aad5680e8f86] raised when running + Podman in Docker, potentially without `docker run --privileged`. This should + decrease the confusion if Mock subsequently falls-back to non-default + `use_bootstrap_image=False`. See [issue#1184][] for more info. +- New exception `BootstrapError` was invented with (if not caught) returns with + exit status 90. This exception covers problems with the bootstrap chroot + preparation. +- The `package_state.py` plugin has been updated to sort the displayed list of + installed and available packages alphabetically (previously the list of packages + was printed in random order). +- Per [PR#1220][] discussion, Mock package newly `Recommends` having DNF5, DNF and + YUM package managers installed on host. These packages are potentially useful, + at least when the (default) bootstrap preparation mechanism (bootstrap image) + fails and the bootstrap needs to be installed with host's package management. + Previously Mock just "suggested" having them installed, which though used to + have almost zero practical effect (as Suggests are not installed by default). +- Mock now, at least on the best effort basis (if used with + `package_manager=dnf`), ["fails" with exit status 30][issue#42] if it isn't able + to process the `--postinstall` request (i.e. installing the built packages into + the target chroot). Previous Mock versions used to ignore (with warning) the + failed package installation attempts. +- The SCM logic [got a new option][PR#1197] + `config_opts['scm_opts']['int_src_dir']` that instructs Mock to search for + sources in a specified sub-directory. + + +### Bugfixes + +- Some container images Mock is using for initializing bootstrap chroot (e.g. + `centos:7`), do not provide all the needed architectures. Podman [though + silently pulls arch-incompatible + image](https://github.com/containers/podman/issues/19717), which caused + [hard-to-debug build failures](https://github.com/fedora-copr/copr/issues/2875). + Mock 5.1 therefore [implements a new assertion][PR#1199] failing the build + early, before mistakenly trying to run emulated "--forcearch" chroot leading to + failure. The assertion is exposed as a Python library call + `mockbuild.podman:podman_check_native_image_architecture()`. +- When bootstrap chroot is initialized from downloaded container image, it + typically contains `/etc/rpm/macros.image-language-conf` file with locale + filtering of some kind (per defaults from the fedora-kickstarts project). + This bootstrap configuration though affects the buildroot installation + (filtering l11n files from buildroot) and this is sometimes unexpected. + Mock now [automatically removes the macro file][PR#1189], see + [issue#1181][] for more info. +- Manual page has been fixed to better describe the `--config-opts=option=value` + semantics. +- Mock uses `tmpfs` mountpoints in some cases just to hide (on host) some rather + complicated mount structures (done in chroot/separate mount namespace). These + "barrier" mount points though used to have the default `mode=0777` potentially + allowing anyone to accidentally write there and cause e.g. unmount failures. + New Mock [uses `mode=0755`][PR#1213] instead. +- The `systemd-nspawn` utility [v253.9 started + failing](https://github.com/systemd/systemd/issues/29174) with pre-mounted + `/proc` directory (used like `systemd-nspawn -D `). + The resulting Mock error, per several reports like [this + one](https://github.com/fedora-copr/copr/issues/2906), was rather cryptic: + + # /usr/bin/systemd-nspawn -q -M 50743cd0fe0a4142b9b2dbb2c5f8eea6 -D /var/lib/mock/fedora-39-x86_64-bootstrap-1694347273.676351/root + Failed to mount /proc/sys (type n/a) on /proc/sys (MS_BIND ""): Invalid argument + Failed to create /user.slice/user-1000.slice/session-12.scope/payload subcgroup: Structure needs cleaning + + Previous versions of `systemd-nspawn` silently over-mounted the `/proc` + filesystem so Mock simply could _always_ pre-mount `/proc` (with + `--isolation=simple` it is still needed). + + To work-around this problem, new Mock now [stopped "pre-mounting"][PR#1214] + `/proc` directory when `--isolation=nspawn` (default) and the package + management downloaded with bootstrap image is used for **installing packages + into the bootstrap chroot**. +- Mock automatically kills "orphan" processes started in buildroot (unwanted + "daemons"). These are typically started by DNF installation that trigger some + buggy scriptlet. The corresponding code has been [moved][PR#1214] to a better + place which assures that such processes are **always** killed before "buildroot + in bootstrap" recursive bind-mount is unmounted. +- Mock properly dumps Podman's standard error output to logs to allow the user + better diagnose related errors. Per [issue#1191][] report. +- Mock 5.0 release contained a bug causing that `postyum` hooks (in Mock plugins) + were not called. In turn, e.g. `root_cache` locking mechanism [was + broken][issue#1186] causing Mock to wait for the lock unnecessary long. +- Previous version of Mock used to bind-mount `/proc` and `/proc/filesystems` in + wrong order, eventually causing that `/proc/filesystems` was not visible (this + could affect some scriptlests from packages installed into such a chroot). This + [has been fixed now][PR#1214]. +- The `--installdeps foo.spec` feature is implemented using the RPM Python API. + Previously we used the method `hdr.dsFromHeader()` to get the list of + `BuildRequires`. This method has been removed from the Python RPM API (rpm + v4.19) in favor of the long-enough existing `rpm.ds(hdr, ...)` method. Mock + [has started using `rpm.ds()` API call][PR#1223] to fix the [`AttributeError: + 'rpm.hdr' object has no attribute 'dsFromHeader'`][issue#1203] traceback on + newer systems (e.g. Fedora 40+). +- The `--shell` standard output is no longer affected by `podman image unmount` + output executed in the background (prints out the image ID). + + +### mock-core-configs v39.1 changes + +- Mageia 9 to branched (released recently) and Cauldron retargeted to Mageia 10. +- openSUSE Leap 15.3 became end-of-life at the end of 2022 and the corresponding + Mock configuration is now [end-of-life][PR#1175], too. +- openSUSE Leap 15.5 configuration [added][PR#1175]. + + +**Following contributors contributed to this release:** + + * Evan Goode + * Miroslav Suchý + * Neal Gompa + * Pavel Raiskup + * Takuya Wakazono + +Thank you. + +[PR#1200]: https://github.com/rpm-software-management/mock/pull/1200 +[issue#1186]: https://github.com/rpm-software-management/mock/issues/1186 +[PR#1220]: https://github.com/rpm-software-management/mock/pull/1220 +[issue#1203]: https://github.com/rpm-software-management/mock/issues/1203 +[issue#42]: https://github.com/rpm-software-management/mock/issues/42 +[PR#1189]: https://github.com/rpm-software-management/mock/pull/1189 +[issue#1184]: https://github.com/rpm-software-management/mock/issues/1184 +[commit#395fc07f796]: https://github.com/rpm-software-management/mock/commit/395fc07f796 +[PR#1210]: https://github.com/rpm-software-management/mock/pull/1210 +[PR#1197]: https://github.com/rpm-software-management/mock/pull/1197 +[issue#1191]: https://github.com/rpm-software-management/mock/issues/1191 +[PR#1199]: https://github.com/rpm-software-management/mock/pull/1199 +[commit#8c7aad5680e8f86]: https://github.com/rpm-software-management/mock/commit/8c7aad5680e8f86 +[PR#1223]: https://github.com/rpm-software-management/mock/pull/1223 +[issue#1181]: https://github.com/rpm-software-management/mock/issues/1181 +[PR#1213]: https://github.com/rpm-software-management/mock/pull/1213 +[PR#1175]: https://github.com/rpm-software-management/mock/pull/1175 +[PR#1214]: https://github.com/rpm-software-management/mock/pull/1214 diff --git a/docs/Release-Notes-5.2.md b/docs/Release-Notes-5.2.md new file mode 100644 index 0000000..b397185 --- /dev/null +++ b/docs/Release-Notes-5.2.md @@ -0,0 +1,30 @@ +--- +layout: default +title: Release Notes - Mock 5.2 +--- + +Released on 2023-09-27. + +### Mock 5.2 new features + +- Mock newly logs out its command-line arguments to better deduct what was + happening at build time. + + +### Bugfixes + +- The fixes introduced in Mock 5.1 included a compatibility issue with Python in + Enterprise Linux 8 due to a dependency on the `capture_output=True` feature in + the `subprocess` module, which was added in Python 3.7. However, EL 8 is + running on Python 3.6. This compatibility issue has been resolved in Mock by + using `stdout=subprocess.PIPE` instead. This update was made based on a [report + from Bodhi update](https://bodhi.fedoraproject.org/updates/FEDORA-EPEL-2023-45ace77fca). +- Previous versions of Mock mistakenly expanded every `~` occurrence + (tilde character) in the specified source path with `--copyout`. So + files `~/foo~bar.txt` were searched on path `/builddir/foo/builddirbar.txt` + instead of just `/builddir/foo~bar.txt`. Fixes [rhbz#2239035][]. +- The Mock state monitoring (creating state.log) was fixed so that Mock, unless + some exception is raised, always checks that we finished all the states we + started. + +[rhbz#2239035]: https://bugzilla.redhat.com/2239035 diff --git a/docs/Release-Notes-5.3.md b/docs/Release-Notes-5.3.md new file mode 100644 index 0000000..3f211d7 --- /dev/null +++ b/docs/Release-Notes-5.3.md @@ -0,0 +1,97 @@ +--- +layout: default +title: Release Notes - Mock v5.3 +--- + +Released on 2023-12-13. + +### Mock 5.3 new features + +- A new plugin to pre-process spec files with rpmautospec [has been + implemented][PR#1253]. + + If this plugin is enabled, mock pre-processes spec files that use rpmautospec + features (for automatic release numbering and changelog generation) before + building a source RPM. + +- Only run the `%prep` section once when running `%generate_buildrequires` + multiple times. + Previously Mock run `%prep` repeatedly before each `%generate_buildrequires` + round except for the last one. This was inconsistent and unnecessary + slow/wasteful. + + When the original support for `%generate_buildrequires` landed into Mock, + the intention was to only call `%prep` once. + However when Mock added support for multiple rounds of + `%generate_buildrequires`, `%prep` ended up only being skipped in the final + `rpmbuild` call. This was an oversight. `%prep` is now only called once, as + originally intended. + + Some RPM packages might be affected by the change, especially if a dirty + working directory after running `%generate_buildrequires` affects the results + of subsequent rounds of `%generate_buildrequires`. However, such behavior was + undefined and quite buggy even previously, due to the lack of the `%prep` + section in the final `rpmbuild` call. + + Packages that need to run commands before every round of + `%generate_buildrequires` should place those commands in the + `%generate_buildrequires` section itself rather than `%prep`. + +- The automatic killing feature for orphan processes within the chroot environment + [was][PR#1255] [improved][PR#1268] to also provide the user with information + about the command-line arguments of the terminated process: + + `WARNING: Leftover process 1331205 is being killed with signal 15: daemon --with-arg` + +- The info about package management tooling used to install the target buildroot + has been updated to provide the info earlier, before the buildroot + installation happens. Mock newly informs also about dnf5 presence. + + +### Bugfixes + +- The Bash completion bug in Mock for options accepting multiple arguments, + tracked in the [long-standing issue][issue#746], has been resolved through [PR#1262]. + +- If DNF 5 sees an "interactive" TTY on stdout, it will try to draw progress bars + and cause the Mock logs to [be garbled](https://github.com/fedora-copr/copr/issues/3040). + This release brings a fix that simply sets the output of DNF5 to a pipe instead + of a PTY. + +- When Mock completes the installation of all the requirements generated + by `%generate_buildrequries`, it calls `rpmbuild -ba` to perform a final build + of the package. + + During the final build, `%generate_buildrequries` runs again in order to + generate a list of `BuildRequires` to be added to the built SRPM metadata. + An arbitrary `%generate_buildrequries` section may generate different + requirements that may not have been installed. + + Previously, the `rpmbuild -ba` call used the `--nodeps` option, + hence it was [possible to successfully build a package with + unsatisfiable BuildRequires in the built SRPM metadata][issue#1246]. + + When a bootstrap chroot is used, the `--nodeps` option is + [no longer used][PR#1249] in the final `rpmbuild -ba` call. + If `%generate_buildrequries` attempts to generate new unsatisfied requirements + during the final build, the build will fail. + When a bootstrap chroot is not used, the `--nodeps` option remains because + Mock cannot know if the RPM in chroot can read the RPM database. + +**Following contributors contributed to this release:** + + * Evan Goode + * Jakub Kadlcik + * Miro Hrončok + * Orion Poplawski + * Stephen Gallagher + +Thank you! + +[issue#746]: https://github.com/rpm-software-management/mock/issues/746 +[PR#1268]: https://github.com/rpm-software-management/mock/pull/1268 +[issue#1246]: https://github.com/rpm-software-management/mock/issues/1246 +[PR#1255]: https://github.com/rpm-software-management/mock/pull/1255 +[PR#1262]: https://github.com/rpm-software-management/mock/pull/1262 +[PR#1249]: https://github.com/rpm-software-management/mock/pull/1249 +[PR#1253]: https://github.com/rpm-software-management/mock/pull/1253 diff --git a/docs/Release-Notes-5.4.md b/docs/Release-Notes-5.4.md new file mode 100644 index 0000000..b56cb03 --- /dev/null +++ b/docs/Release-Notes-5.4.md @@ -0,0 +1,13 @@ +--- +layout: default +title: Release Notes - Mock 5.4 (bugfix release) +--- + +Released on 2024-01-04. + +### Mock 5.4 bugfixes + +- This release fixes how the rpmautospec plugin installs its dependencies into + the build root, see [PR#1275][] for more info. + +[PR#1275]: https://github.com/rpm-software-management/mock/pull/1275 diff --git a/docs/Release-Notes-5.5.md b/docs/Release-Notes-5.5.md new file mode 100644 index 0000000..a0cce78 --- /dev/null +++ b/docs/Release-Notes-5.5.md @@ -0,0 +1,128 @@ +--- +layout: default +title: Release Notes - Mock 5.5 +--- + +Released on 2024-02-14. + + +### Mock 5.5 New Features + +- New `write_tar` option for `chroot_scan` plugin [added][PR#1324]. Without it, + directory structure is created in `resultdir`. If `write_tar` is set to + `True`, `chroot_scan.tar.gz` tarball will be created instead. + +- A new `{% raw %}{{ repo_arch }}{% endraw %}` Jinja2 template (templated-dictionary) is provided + by Mock. This variable is usable for DNF config options denoting URLs like + `baseurl=`, `metalink=`, etc. Namely, it can be used instead of the DNF-native + `$basearch` variable which [doesn't work properly for all the + distributions][issue#1304]. The new `config_opts['repo_arch_map']` option has + been added too, if additional tweaks with `repo_arch` template need to be done. + +- Previously, only the file sizes were reported by the hw_info plugin: + + ~~~ + Filesystem Size Used Avail Use% Mounted on + /dev/mapper/luks-3aa4fbe3-5a19-4025-b70c-1d3038b76bd4 399G 9.1G 373G 3% / + /dev/mapper/luks-3aa4fbe3-5a19-4025-b70c-1d3038b76bd4 399G 9.1G 373G 3% / + ~~~ + + Newly, [also file system type is reported][issue#1263]: + + ~~~ + Filesystem Type Size Used Avail Use% Mounted on + /dev/mapper/luks-3aa4fbe3-5a19-4025-b70c-1d3038b76bd4 btrfs 399G 9.1G 373G 3% / + /dev/mapper/luks-3aa4fbe3-5a19-4025-b70c-1d3038b76bd4 btrfs 399G 9.1G 373G 3% / + ~~~ + + +### Mock 5.5 Bugfixes + +- The Bash completion script has been fixed to properly complete arguments + of [multi-arg options][issue#1279] like `--install` or `--chain`. This + bug-fix is a follow-up for fix related to [issue#746][]. + +- Mock [has been][PR#1322] [fixed][commit#27dde5da] so that it no longer + inadvertently changes the group ownership of the in-chroot $HOME directory + to the wrong group ID. In previous versions of Mock, the group ownership + was changed to the effective group ID of the user on the host that + executed Mock. This could confuse some tools during the build process, as + they might want to create similarly owned files at the `rpmbuild` time + (but that assumption would be incorrect, such GID doesn't exist + in-chroot). Now, Mock changes the files to the `mockbuild:mock` + ownership, where the `mock` group ID is always 135 (the same GID on host + and in-chroot). This matches the effective GID for the `rpmbuild` + process, ensuring that the tools executed during the build process have + full control over such files. + + While on this, Mock was also [optimized][commit#db64d468202] to do this + ownership change before, and only if, the `rpmbuild` process is started + (so e.g. plain `mock --chroot` commands do not touch the file ownership at + all). + +- The `mock` package has been fixed to depend on precisely the same version + of `mock-filesystem` sub-package (product of the same rpm build). This is + to protect against incompatible Mock sub-package installations. + +- Mock parses the DNF configuration specified in `config_opts["dnf.conf"]` itself + to perform some post-processing tasks, such as bind-mounting the on-host repo + directories into the bootstrap chroot based on this DNF config. In previous + versions of Mock, there was an issue where it failed to parse the DNF config if + some of the DNF options contained the '%' symbol. This was due to Python's + ConfigParser raising an %-interpolation exception that was ignored but Mock, but + resulting in Mock ignoring the rest of the config file and finishing without + performing the mentioned bind-mounts. This bug has been fixed, and the '%' sign + is no longer considered to be a special Python ConfigParser character. + +- The `root_cache` plugin is designed to invalidate the cache tarball whenever the + corresponding Mock configuration changes (any file in the list + `config_opts['config_paths']` changes). This cache invalidation mechanism had + been broken since Mock v3.2 when we rewrote the configuration file loader and + inadvertently broke the `config_opts['config_paths']`. The config loader [has + now been fixed][PR#1322], and the cache invalidation works again as expected. + +### Mock Core Configs 40.1 changes + +- Configuration files for Fedora 40 have been branched from Rawhide, + according to the [Fedora 40 Schedule](https://fedorapeople.org/groups/schedule/f-40/f-40-all-tasks.html). + +- Mageia 7 configs [marked as end-of-life][PR#1316]. + +- Mageia config files started using the `{% raw %}{{ repo_arch }}{% endraw %}` option to fix the + [cross-arch builds][issue#1317]. + +- The OpenMandriva i686 chroots [have been marked as end-of-life][PR#1315], fixing + [issue#987][] and [issue#1012][]. + +- [Added][PR#1283] a config option called "use_host_shadow_utils", to account for situations where + users have host shadow-utils configurations that cannot provision or destroy users and + groups in the buildroot; one example of this kind of configuration is using + FreeIPA-provided subids on the buildhost. The option defaults to True since mock has made a conscious + design decision to prefer using the host's shadow-utils, and we hope that this is a + temporary workaround. Upstream issue is being tracked [here](https://github.com/shadow-maint/shadow/issues/897). + +**Following contributors contributed to this release:** + + * Jakub Kadlcik + * Jani Välimaa + * Martin Jackson + * Tomas Kopecek + * Vít Ondruch + + +Thank you! + +[commit#db64d468202]: https://github.com/rpm-software-management/mock/commit/db64d468202 +[issue#1012]: https://github.com/rpm-software-management/mock/issues/1012 +[commit#27dde5da]: https://github.com/rpm-software-management/mock/commit/27dde5da +[PR#1324]: https://github.com/rpm-software-management/mock/pull/1324 +[PR#1322]: https://github.com/rpm-software-management/mock/pull/1322 +[issue#987]: https://github.com/rpm-software-management/mock/issues/987 +[PR#1315]: https://github.com/rpm-software-management/mock/pull/1315 +[PR#1283]: https://github.com/rpm-software-management/mock/pull/1283 +[issue#1263]: https://github.com/rpm-software-management/mock/issues/1263 +[issue#746]: https://github.com/rpm-software-management/mock/issues/746 +[issue#1317]: https://github.com/rpm-software-management/mock/issues/1317 +[issue#1304]: https://github.com/rpm-software-management/mock/issues/1304 +[issue#1279]: https://github.com/rpm-software-management/mock/issues/1279 +[PR#1316]: https://github.com/rpm-software-management/mock/pull/1316 diff --git a/docs/Release-Notes-5.6.md b/docs/Release-Notes-5.6.md new file mode 100644 index 0000000..4f9c429 --- /dev/null +++ b/docs/Release-Notes-5.6.md @@ -0,0 +1,75 @@ +--- +layout: default +title: Release Notes - Mock 5.6 +--- + +## [Release 5.6](https://rpm-software-management.github.io/mock/Release-Notes-5.6) - 2024-05-14 + + +### New features + +- The Bash completion script for Mock has been improved to pre-compile the list of + available Mock options at package build-time. This enhancement significantly + reduces the time required for option completion from approximately 0.5 seconds + (on a reasonably fast laptop) to just 0.05 seconds. [rhbz#2259430][]. + + +### Bugfixes + +- When a `mock --chain --recurse` fails to built at least one package, it is + unable to print a list of failed packages and displays `AttributeError: type + object 'FileDownloader' has no attribute 'backmap'` instead. The `original_name` + method of `FileDownloader` class has been fixed, and the chain build results + displayed as expected ([issue#1345][]). +- Don't use the `--allowerasing` parameter for DNF subcommands such as + `repoquery`, `makecache`, `search`, and `info`. +- A missing bash completion script for `mock-parse-buildlog` command [has + been added][PR#1353]. +- In the [issue#1257][] it was suggested that we do not change recursively + ownership every run. This was implemented and landed in Mock 5.5. + But in the [issue#1364][] we found that for fresh chroots the homedir + is not writable for unpriv user. + We changed the behaviour that ownership of homedir is changed always (that was + a behaviour prior 5.5 release) and the ownership is changed recursively only for + rebuilds. +- The `nosync` logic was preparing temporary directories even when + `config_opts["nosync"] = False` (meaning nosync was disabled). This logic has + been optimized out. Works around [issue#1351][]. +- No more ugly tracebacks for "no space left on device" (and similar + `OSError`s) related to copying built artifacts to `--resultdir`, + [rhbz#2261758][]. +- The SCM plugin's option `git_timestamps` has been updated to work with Python 3 + and to handle Git repositories with non-Unicode data. ([PR#1355][]) + + +### Mock Core Configs changes + +- Add configuration for Circle Linux 9 configurations ([PR#1366][]). +- Add i686 configuration for Mageia Cauldron and Mageia 10, and remove + corresponding i586 configurations ([PR#1360][]). +- The Fedora ELN configuration [has been updated to use DNF5 for Mock chroot + package management][issue#1292]. + + +#### Following contributors contributed to this release: + +- Bella Zhang +- David Michael +- Jakub Kadlcik +- Jani Välimaa +- Miroslav Suchý +- Nikita Gerasimov +- Pavel Raiskup +- Takuya Wakazono + +[rhbz#2259430]: https://bugzilla.redhat.com/2259430 +[issue#1292]: https://github.com/rpm-software-management/mock/issues/1292 +[issue#1345]: https://github.com/rpm-software-management/mock/issues/1345 +[issue#1351]: https://github.com/rpm-software-management/mock/issues/1351 +[issue#1364]: https://github.com/rpm-software-management/mock/issues/1364 +[PR#1366]: https://github.com/rpm-software-management/mock/pull/1366 +[PR#1353]: https://github.com/rpm-software-management/mock/pull/1353 +[issue#1257]: https://github.com/rpm-software-management/mock/issues/1257 +[PR#1355]: https://github.com/rpm-software-management/mock/pull/1355 +[PR#1360]: https://github.com/rpm-software-management/mock/pull/1360 +[rhbz#2261758]: https://bugzilla.redhat.com/2261758 diff --git a/docs/Release-Notes-5.7.md b/docs/Release-Notes-5.7.md new file mode 100644 index 0000000..6cfa6b7 --- /dev/null +++ b/docs/Release-Notes-5.7.md @@ -0,0 +1,189 @@ +--- +layout: default +title: Release Notes - Mock 5.7 (+configs v41.3) +--- + +## [Release 5.7](https://rpm-software-management.github.io/mock/Release-Notes-5.7) - 2024-09-26 + +### New features and important changes + +- Support for [hermetic builds](feature-hermetic-builds) has [been][PR#1393] + [implemented][PR#1449]. This update introduces two new command-line options: + `--calculate-build-deps` and `--hermetic-build`, along with the new + `mock-hermetic-repo(1)` utility. + + Additionally, this change introduces a new [`buildroot_lock` + plugin](Plugin-BuildrootLock), which generates a new artifact in the buildroot—a + buildroot *lockfile*. Users can enable this plugin explicitly by setting + `config_opts["plugin_conf"]["buildroot_lock_enable"] = True`. + +- This version addresses [issue#521][], which requested a cleanup option for + all chroots. A [new][PR#1337] option, `--scrub-all-chroots`, has been + added. It can detect leftovers in `/var/lib/mock` or `/var/cache/mock` + and make multiple `mock --scrub=all` calls accordingly. + +- Alias `dnf4` added for the `package_manager = dnf`. + + The options specific to DNF4, previously prefixed with `dnf_*`, have been + renamed to `dnf4_*` too to avoid confusion with `dnf5_*` options. For backward + compatibility, the `dnf_*` prefixed variants still work, so these config pairs + are equivalent: + + ```python + config_opts['dnf4_install_cmd'] = 'install python3-dnf python3-dnf-plugins-core' + config_opts['dnf_install_cmd'] = 'install python3-dnf python3-dnf-plugins-core' + + config_opts['package_manager'] = 'dnf4' + config_opts['package_manager'] = 'dnf' + ``` + + Some of the `dnf_*` options remain unchanged because they are universal and used + with DNF4, DNF5, or YUM, e.g., `dnf_vars`. + + While working on this rename, the rarely used `system__command` options have + been changed to `_system_command` to visually align with the rest of the + package-manager-specific options. The old variants are still accepted. + +- The `--addrepo` option has been updated to affect both the bootstrap chroot + installation and the buildroot installation, as requested in [issue#1414][]. + However, be cautious, as Mock [aggressively caches the bootstrap][issue#1289]. + Always remember to run `mock -r --scrub=bootstrap` first. + Additionally, as more chroots are being switched to `bootstrap_image_ready = + True`, you'll likely need to use `--addrepo` **in combination with** + `--no-bootstrap-image`; otherwise, the bootstrap chroot installation will remain + unaffected. + +- There's a new `config_opts['bootstrap_image_skip_pull']` option that allows you + to skip image pulling (running the `podman pull` command by Mock) when preparing + the bootstrap chroot. This is useful if `podman pull` is failing, for example, + when the registry is temporarily or permanently unavailable, but the local image + exists, or if the image reference is pointing at a local-only image. + +- There's a new [ccache](Plugin-CCache) plugin option + `config_opts['plugin_conf']['ccache_opts']['show_stats']`; if set to `True`, + Mock prints the ccache statistics (hits/misses) to logs. + +- A new option `debug` has been [added][PR#1408] to the ccache plugin. Setting + it to `True` creates per-object debug files that are helpful when debugging + unexpected cache misses, see [ccache docs][ccache-docs-debug]. + +- A new option `hashdir` has been [added][PR#1399] to the ccache plugin. Setting + it to `False` excludes the build working directory from the hash used to + distinguish two compilations when generating debuginfo. While this allows the + compiler cache to be shared across different package NEVRs, it might cause the + debuginfo to be incorrect. + The option can be used for issue bisecting if running the debugger is + unnecessary ([issue#1395][]). + +- New Mock RPM package provides the systemd-sysusers drop-in configuration file + for automatic `mock 135` group ID allocation. See [rpm docs][] for more info. + + +### Bugfixes + +- The `installed_pkgs.log` — generated by the `package_state` plugin — was + [previously generated too early][issue#1429], after the static build + requirements were installed but **before** the dynamic build requirements were + resolved and installed. This led to incorrect chroot introspection for the end + user, as the reported set of packages needed to build the given package was + incomplete. The new Mock version generates the `installed_pkgs.log` file after + the dynamic build requirements are installed. + + +- De-duplicating bootstrap mount points for local repositories used with `--chain` + or `--localrepo=file:///repo/on/host`. Caused eventual problems during + `--scrub=bootstrap`. This bug existed in Mock <= 5.6, but after fixing the + [issue#1414][], it got exposed by our test suite. Related issues include + [issue#357][] ([commit#a0a2cba3][]) and [issue#381][] ([commit#16462acc][]). + +- Previously, mock.rpm created file in `/usr/share/doc/mock` directory but did + did own this directory. + +- The fuse-overlayfs package is not installed in the Fedora container images + by default. We need to explicitly install it, otherwise running Mock inside of + a container won't work. + +- Previously, the `nspawn_args` configuration value was not applied in multiple + internal `doChroot()` calls. This could cause issues when custom nspawn + arguments were needed everywhere (see [PR#1410][]). Now, `doChroot()` + automatically applies `nspawn_args`, shifting the responsibility from callers to + callee. + +- Several internal code locations attempt to ensure that the result directory + exists, creating it if necessary. However, these locations handled it + inconsistently, sometimes neglecting to [change the ownership][issue#1467] of + the result directory. Now, all locations use a single method dedicated to + result directory preparation. + + +### Mock Core Configs changes + +- The GPG key locations for the CentOS Stream 10 Appstream debuginfo + and Extras Common repositories were updated to point to the correct + GPG keys. + +- Anolis-7 has been EOLed at 2024-06-30. We moved the configs to `eol` directory. + +- The CentOS Stream 10 configuration has been updated to use + `quay.io/centos/centos:stream10-development` as its bootstrap image. Since + this image [already has the `python3-dnf-plugins-core` package + installed](https://issues.redhat.com/browse/CS-2506), the configuration is also + updated to set `bootstrap_image_ready = True`. This means the image can be + used "as is" to bootstrap the DNF stack without installing any additional + packages into the prepared bootstrap chroot, significantly speeding up + bootstrap preparation. + +- The centos-stream-9 (and transitively centos-stream+epel-9) configuration has + been fixed to rely on "bootstrap image" readiness for Mock builds, see + [issue#1442][]. + +- OpenSuse Leap 15.4 EOLed at 07 Dec 2023 so we moved the configs to `eol` directory. + +- We [updated the configuration][PR#1195] files for chroots that still use older + RPM versions (v4.18 and earlier), which affects RHEL/CentOS 9 and older. These + older RPM versions were built with an incorrect default for the `%_host_cpu` + macro in `ppc64le` chroots, where the macro incorrectly resolved to + `powerpc64le` instead of `ppc64le`. + + This incorrect value caused issues during architecture validation, such as when + checking `ExclusiveArch: ppc64le` for `BuildArch: noarch` packages (done + in-chroot by `/bin/rpmbuild`). + + The incorrect macro value has now been overridden in the relevant `ppc64le` + configuration files in Mock, ensuring that `ExcludeArch` and `ExclusiveArch` + validations resolve correctly. + +#### Following contributors contributed to this release: + +- Brian J. Murrell +- Carl George +- Jakub Kadlcik +- Jiri Kyjovsky +- Julian Sikorski +- Miroslav Suchý +- Nils Philippsen +- Thomas Mendorf + +Thank you! + + +[PR#1195]: https://github.com/rpm-software-management/mock/pull/1195 +[issue#521]: https://github.com/rpm-software-management/mock/issues/521 +[PR#1399]: https://github.com/rpm-software-management/mock/pull/1399 +[commit#a0a2cba3]: https://github.com/rpm-software-management/mock/commit/a0a2cba3 +[PR#1337]: https://github.com/rpm-software-management/mock/pull/1337 +[issue#1429]: https://github.com/rpm-software-management/mock/issues/1429 +[commit#16462acc]: https://github.com/rpm-software-management/mock/commit/16462acc +[issue#1442]: https://github.com/rpm-software-management/mock/issues/1442 +[PR#1408]: https://github.com/rpm-software-management/mock/pull/1408 +[issue#1289]: https://github.com/rpm-software-management/mock/issues/1289 +[issue#1414]: https://github.com/rpm-software-management/mock/issues/1414 +[issue#381]: https://github.com/rpm-software-management/mock/issues/381 +[PR#1393]: https://github.com/rpm-software-management/mock/pull/1393 +[issue#357]: https://github.com/rpm-software-management/mock/issues/357 +[issue#1467]: https://github.com/rpm-software-management/mock/issues/1467 +[issue#1395]: https://github.com/rpm-software-management/mock/issues/1395 +[PR#1410]: https://github.com/rpm-software-management/mock/pull/1410 +[PR#1449]: https://github.com/rpm-software-management/mock/pull/1449 +[ccache-docs-debug]: https://ccache.dev/manual/4.10.html#config_debug +[rpm docs]: https://rpm-software-management.github.io/rpm/manual/users_and_groups.html diff --git a/docs/Release-Notes-5.8.md b/docs/Release-Notes-5.8.md new file mode 100644 index 0000000..3fb7014 --- /dev/null +++ b/docs/Release-Notes-5.8.md @@ -0,0 +1,23 @@ +--- +layout: default +title: Release Notes - Mock 5.8 +--- + +## [Release 5.8](https://rpm-software-management.github.io/mock/Release-Notes-5.8) - 2024-09-27 + + +### Bugfixes + +- Mock v5.7 introduced a regression in the `chroot_scan` plugin that prevented the + result directory from being created properly. This issue has been + [fixed][PR#1472] - and is the major reason for doing v5.8 bugfix release. + +- The ownership of the tarball provided by `chroot_scan` (when `write_tar = + True`) has been corrected, ensuring the file is no longer root-owned. + +- The `chroot_scan` plugin now consistently uses the `util.do` method instead of + custom `subprocess.call` calls. This ensures that the `mock --verbose` output + properly displays the commands (like `cp`, or `tar`) being executed. + + +[PR#1472]: https://github.com/rpm-software-management/mock/pull/1472 diff --git a/docs/Release-Notes-5.9.md b/docs/Release-Notes-5.9.md new file mode 100644 index 0000000..abb087d --- /dev/null +++ b/docs/Release-Notes-5.9.md @@ -0,0 +1,25 @@ +--- +layout: default +title: Release Notes - Mock 5.9 (+configs v41.4) +--- + +## [Release 5.9](https://rpm-software-management.github.io/mock/Release-Notes-5.9) - 2024-09-30 + +### Bugfixes + +- A fix for the DNF → DNF4 fallback has been applied. Now Mock correctly + selects DNF4, even when the `--no-bootstrap-chroot` command is used. See + [issue#1475][] for more info. + +### Mock Core Configs changes + +- The Fedora ELN configuration has been updated to download repositories using + mirrors from the Fedora MirrorManager system. + +#### Following contributors contributed to this release: + +- Yaakov Selkowitz + +Thank you! + +[issue#1475]: https://github.com/rpm-software-management/mock/issues/1475 diff --git a/docs/Release-Notes-6.0.md b/docs/Release-Notes-6.0.md new file mode 100644 index 0000000..1c94354 --- /dev/null +++ b/docs/Release-Notes-6.0.md @@ -0,0 +1,112 @@ +--- +layout: default +title: Release Notes - Mock 6.0 (+ mock-core-configs v41.5) +--- + +## [Release 6.0](https://rpm-software-management.github.io/mock/Release-Notes-6.0) - 2024-12-19 + + +### New features + +- [A new plugin, `export_buildroot_image`](Plugin-Export-Buildroot-Image), has + been added. This plugin can export the generated Mock chroot as an OCI image + archive once all the build dependencies have been installed (when the chroot + is fully prepared to run `/bin/rpmbuild -bb`). + + A [new complementary feature](Feature-buildroot-image) has also been + implemented in Mock and can be enabled using the following option: + + --buildroot-image /tmp/buildroot-oci.tar + + This feature allows the use of generated OCI archives as the source for the + build chroot, similar to how `bootstrap_image` is used as the base for the + bootstrap chroot. + + Additionally, this feature can be utilized with an online image: + + --buildroot-image registry.access.redhat.com/ubi8/ubi + + In both cases, it is essential to use chroot-compatible images! + +- Hermetic build process is enhanced by adding used imaged digests into the + metadata and confirming that exactly same image is used in the next step. + +- The mock-hermetic-repo command now implements a retry mechanism for + downloading files. + +- The `podman load` mechanism for loading OCI archive bootstrap images has been + replaced with `podman pull oci-archive:/path/to-the.tar`. + +### Bugfixes + +- The [chroot_scan plugin](Plugin-ChrootScan) has been fixed so it + no longer (re)creates resultdir below the global `basedir`, but under the + per-package resultdir (by default in a `/var/tmp/` sub-directory). In turn, the + resultdir is no longer created with improper ownership, [issue#1490][]. + +- Make `--dnf-cmd` compatible with DNF5, fixes [issue#1400][]. + +- The `libexec/create_default_route_in_container.sh` file shipped with Mock has + been removed, it was never used in practice (relates to [issue#113][]). + +- The [hermetic mode](feature-hermetic-builds) no longer fallbacks to a manual + bootstrap installation using the hosts DNF stack; it doesn't make sense + because we don't have the bootstrap packages pre-downloaded in the local + "offline" repository. Fixes [issue#1522][]. + +- The error message in the `podman_check_native_image_architecture()` method has + been fixed to correctly indicate the expected (system) architecture and the + image architecture. + +- Regression introduced in Mock v5.7 that ignored the `dnf_builddep_opts` + configuration option has been fixed, [issue#1496][]. + +- The `%pre` scriptlet installing the `mock` group is newly not used for modern + distributions like Fedora 39+ or Mageia (group/user additions are handled by + an RPM built-in feature). + +### Mock Core Configs v41.5 changes + +- The Fedora ELN template has been updated for the new pullspec of the bootstrap + image. + +- The Fedora ELN ResilientStorage repositories are obsolete and have been + removed from the ELN template. + +- The EPEL 10 configuration has been updated to include the epel and epel-testing + repos with the appropriate metalinks. Previously it only included the koji + local repo. + +- Fedora 39 is now [end-of-live](https://fedorapeople.org/groups/schedule/f-39/f-39-all-tasks.html); + the corresponding Mock configuration files have been moved under the + `/etc/mock/eol` directory. + +- Fix [openSUSE-tumbleweed update failure][issue#1506] during the second build. + +- The CentOS Stream 10 configuration has been updated to use + `quay.io/centos/centos:stream10` as its bootstrap image, instead of the + previously used development image. + + +#### The following contributors have contributed to this release: + +- Addisu Z. Taddese +- Carl George +- cheese1 +- duli +- Jakub Kadlcik +- Maksym Kondratenko +- Miroslav Suchý +- Ralph Bean +- Romain Geissler +- Tomas Kopecek +- Yaakov Selkowitz + +Thank You! + +[issue#113]: https://github.com/rpm-software-management/mock/issues/113 +[issue#1506]: https://github.com/rpm-software-management/mock/issues/1506 +[issue#1490]: https://github.com/rpm-software-management/mock/issues/1490 +[issue#1522]: https://github.com/rpm-software-management/mock/issues/1522 +[issue#1400]: https://github.com/rpm-software-management/mock/issues/1400 +[issue#1496]: https://github.com/rpm-software-management/mock/issues/1496 diff --git a/docs/Release-Notes-6.1.md b/docs/Release-Notes-6.1.md new file mode 100644 index 0000000..2155b0a --- /dev/null +++ b/docs/Release-Notes-6.1.md @@ -0,0 +1,68 @@ +--- +layout: default +title: Release Notes - Mock 6.1 +--- + +## [Release 6.1](https://rpm-software-management.github.io/mock/Release-Notes-6.1) - 2025-02-27 + + +### New features + +- The buildroot lockfile generator [has been modified][PR#1548] to include + additional bootstrap image metadata that can be later used for a precise image + pulling. + + The mock-hermetic-repo script has also been modified, to respect the additional + metadata. This allows us to, e.g., download bootstrap image of a different + (cross) architecture then the platform/host architecture is. In turn, the + script is now fully arch-agnostic (any host arch may be used for downloading + files from any arch specific lockfile). + + +### Bugfixes + +- Previous versions of Mock did not install local RPMs (specified as filenames) + via --additional-package when bootstrap was enabled (default). This bug has + been fixed. [issue#1532][]. + + +### Mock Core Configs 43.1 changes + +- Add AlmaLinux Kitten 10 configs to enable building packages for AlmaLinux Kitten 10. +- Add AlmaLinux Kitten 10 + EPEL 10 configs to enable building packages for EPEL 10 using AlmaLinux Kitten 10 as a base. +- Add Azure Linux 2.0 configuration (x68_64, aarch64). The distribution changed name mid lifecycle, it was originally called "CBL Mariner 2.0", replacing "Common Base Linux 1.0". That's why the distribution tag is still "cm2" and has "mariner" references in the repository. +- Add Azure Linux 3.0 configuration (x86_64, aarch64). +- EuroLinux is end-of-life now, so we [EOLed][issue#1537] also the corresponding Mock configuration. +- Add Kylin 10 mock configuration files (x86_64, aarch64, loongarch64). +- Navy Linux 8 configuration [fixed][issue#1538] +- Bugfix: upgrade openeuler chroots to latest release and fix gpg check failed in 20.03 for issue#1539 +- Add openSUSE Leap 15.6 configurations [issue#1516][] + Move openSUSE Leap 15.5 configurations to eol (since 31st December 2024) [issue#1516][] +- Expand Oracle Linux distro_id from `ol` to `oraclelinux` when looking for configuration files [issue#1545] + +#### The following contributors have contributed to this release: + +- Evan Goode +- cheese1 +- Pavel Raiskup + +and to release of configs: + +- Adil Hussain +- Avi Miller +- Li Chaoran +- Miroslav Suchý +- Neal Gompa +- Pavel Raiskup +- Simone Caronni + +Thank You! + + + +[issue#1516]: https://github.com/rpm-software-management/mock/issues/1516 +[issue#1545]: https://github.com/rpm-software-management/mock/issues/1545 +[issue#1537]: https://github.com/rpm-software-management/mock/issues/1537 +[issue#1532]: https://github.com/rpm-software-management/mock/issues/1532 +[issue#1538]: https://github.com/rpm-software-management/mock/issues/1538 +[PR#1548]: https://github.com/rpm-software-management/mock/pull/1548 diff --git a/docs/Release-Notes-6.2.md b/docs/Release-Notes-6.2.md new file mode 100644 index 0000000..d07b1da --- /dev/null +++ b/docs/Release-Notes-6.2.md @@ -0,0 +1,54 @@ +--- +layout: default +title: Release Notes - Mock 6.2 +--- + +## [Release 6.2](https://rpm-software-management.github.io/mock/Release-Notes-6.2) - 2025-05-22 + + +### New features + +- Disables copying /etc/pki/ca-trust and /usr/share/pki/ca-trust-source on + Azure Linux 3.0 via a new config options ('ssl_copied_ca_trust_dirs'). + This avoids file ownership conflicts with a symlink installed by the + ca-certificates-shared packages on that distro. Behavior should be unchanged + for other configurations. + +### Bugfixes + +- Mock will now fall back smoothly to chroot installation by DNF if Podman + image pull fails entirely. + + +### Mock Core Configs changes + +- Mock chroots for RHEL 10 and RHEL+EPEL 10 have been added. + +- Fedora 40 is now + [EOL](https://fedorapeople.org/groups/schedule/f-40/f-40-key-tasks.html), and + we marked Fedora 40 configuration EOL, too. + +- The EPEL 10 configuration has been updated to have separate templates + available for use on CentOS Stream 10 (epel-10.tpl) and RHEL 10 + (epel-z-10.tpl). Relates to [issue#1427][]. + +- Update the list of packages installed by default in the Azure Linux 2 & 3 + chroot to include shadow-utils. Due to recent changes, the useradd/groupadd + commands are required by other packages in the list, but the requirements are + not specified correctly. Add %dist macro to the Azure Linux 3 template. + +- The openSuse Leap 15.6 is now correctly using a bootstrap image of Leap 15.6. + The Leap configurations now refer to bootstrap images using the $releasever + value, instead of hardcoding, to avoid potential misalignment in the future. + + +#### The following contributors have contributed to this release: + +- Adam Williamson +- Carl George +- Reuben Olinsky +- Simone Caronni + +Thank You! + +[issue#1427]: https://github.com/rpm-software-management/mock/issues/1427 diff --git a/docs/Release-Notes-6.3.md b/docs/Release-Notes-6.3.md new file mode 100644 index 0000000..b976f32 --- /dev/null +++ b/docs/Release-Notes-6.3.md @@ -0,0 +1,60 @@ +--- +layout: default +title: Release Notes - Mock 6.3 +--- + +## [Release 6.3](https://rpm-software-management.github.io/mock/Release-Notes-6.3) - 2025-06-18 + + +### New features + +- The `hw_info` plugin now reports memory info units in [human readable scale][PR#1587]. + + +### Bugfixes + +- The `/bin/dnf` command is no longer hardcoded + [in the --calculate-build-dependencies code][issue#1592], and we + use the standard `config_opts['dnf5_command']` (or + `config_opts['dnf_command']`, respectively). + +- The mechanism for creating the `mock` group has been fixed again on + Fedora/RHEL, because the built-in RPM user creation mechanism only works on + F42 and newer. Mock on older distributions returned back to using `%pre` + scriptlet. + +- The `mock-core-configs` package files were previously owned by the `mock` + group, same as a few files in `mock` package — which was unnecessary. These + files are intended to be read-only and accessible by anyone. This issue was + actually [causing install-order problems][issue#1588] when `mock-core-configs` + or `mock` was installed before `mock-filesystem`. So newly those files have + the default `0644, root, root` ownership. + +- This release includes a fix for a Python 3.14 [incompatibility][issue#1594]. + + Mock refused to start as non-root user with Python 3.14. This was because of + the change in behaviour of ProcessPoolExecutor in Python. The code has been + altered to work with both old and new Python. + +### Mock Core Configs changes + +- Added AlmaLinux 10 configs +- Add AlmaLinux kitten 10 x86_64_v2 config +- Added Rocky Linux 10 configs + +#### The following contributors have contributed to this release: + +- FeRD (Frank Dana) +- Javier Hernández +- Jonathan Wright +- Konstantin Shalygin +- Louis Abel +- Miroslav Suchý + +Thank You! + + +[issue#1592]: https://github.com/rpm-software-management/mock/issues/1592 +[issue#1588]: https://github.com/rpm-software-management/mock/issues/1588 +[PR#1587]: https://github.com/rpm-software-management/mock/pull/1587 +[issue#1594]: https://github.com/rpm-software-management/mock/issues/1594 diff --git a/docs/Release-Notes-Configs-39.3.md b/docs/Release-Notes-Configs-39.3.md new file mode 100644 index 0000000..265be85 --- /dev/null +++ b/docs/Release-Notes-Configs-39.3.md @@ -0,0 +1,27 @@ +--- +layout: default +title: Release Notes - Mock Core Configs 39.3 +--- + +## [Release 39.3](https://rpm-software-management.github.io/mock/Release-Notes-Configs-39.3) - 2023-12-01 + +### Mock Core Configs changes + +- Per the approved [Fedora 40 change](https://fedoraproject.org/wiki/Changes/BuildWithDNF5), + [we switched][PR#1256] the default `package_manager` configuration for Fedora 40 + (Rawhide at that point in time) to `dnf5`. DNF5 is [the future replacement for + DNF4](https://fedoraproject.org/wiki/Changes/ReplaceDnfWithDnf5), aiming to be + much faster than its predecessor. Hence, the effect of this change is a + significantly faster buildroot preparation. +- The default `fedora-eln-*` bootstrap image `quay.io/fedoraci/fedora:eln` + [has been fixed](https://github.com/fedora-eln/eln/issues/166) to provide + the `dnf builddep` command. It means it is now "ready for bootstrap" right + after the image download (no additional packages need to be installed inside) + which makes the buildroot preparation + [much faster](https://rpm-software-management.github.io/mock/Feature-container-for-bootstrap). +- The OpenMandriva chroots provide `python-dnf` and `python-dnf-plugins-core` + packages, not `python3-dnf` and `python3-dnf-plugins-core`. That's why we + [had to fix][issue#1251] the `dnf_install_command` config option appropriately. + +[PR#1256]: https://github.com/rpm-software-management/mock/pull/1256 +[issue#1251]: https://github.com/rpm-software-management/mock/issues/1251 diff --git a/docs/Release-Notes-Configs-39.4.md b/docs/Release-Notes-Configs-39.4.md new file mode 100644 index 0000000..eae3517 --- /dev/null +++ b/docs/Release-Notes-Configs-39.4.md @@ -0,0 +1,21 @@ +--- +layout: default +title: Release Notes - Mock Core Configs 39.4 +--- + +## [Release 39.4](https://rpm-software-management.github.io/mock/Release-Notes-Configs-39.4) - 2024-01-11 + +### Mock Core Configs changes + +- The DNF4 caches downloaded RPMs in `/var/cache/dnf`. This path has been + changed in DNF5 to `/var/cache/libdnf5` and this change confuses some packages + [rhbz#2256945][]. Thus, the Fedora mock configuration files have been fixed + to use the same cache directory even with DNF5 — using the + `system_config=/var/cache/dnf` DNF5 option. See [PR#1150][] for more info. + +- The chroot configuration for Fedora 37 [has been EOLed][PR#1270], according to + the [F39 schedule](https://fedorapeople.org/groups/schedule/f-39/f-39-all-tasks.html). + +[PR#1270]: https://github.com/rpm-software-management/mock/pull/1270 +[PR#1150]: https://github.com/rpm-software-management/mock/pull/1150 +[rhbz#2256945]: https://bugzilla.redhat.com/2256945 diff --git a/docs/Release-Notes-Configs-40.2.md b/docs/Release-Notes-Configs-40.2.md new file mode 100644 index 0000000..88cd779 --- /dev/null +++ b/docs/Release-Notes-Configs-40.2.md @@ -0,0 +1,21 @@ +--- +layout: default +title: Release Notes - Mock Core Configs 40.2 +--- + +## [Release 40.2](https://rpm-software-management.github.io/mock/Release-Notes-Configs-40.2) - 2024-02-16 + +### Mock Core Configs changes + +- Per the approved [Fedora 40 change](https://fedoraproject.org/wiki/Changes/BuildWithDNF5), + [we switched][PR#1332] the default `package_manager` configuration + for Fedora 40 or newer to `dnf5`. + This was previously done when Fedora 40 was Rawhide, + but it [regressed][rhbz#2264535] when Fedora 40 branched. + +**Following contributors contributed to this release:** + + * Miro Hrončok + +[PR#1332]: https://github.com/rpm-software-management/mock/pull/1332 +[rhbz#2264535]: https://bugzilla.redhat.com/2264535 diff --git a/docs/Release-Notes-Configs-40.3.md b/docs/Release-Notes-Configs-40.3.md new file mode 100644 index 0000000..e9ee1ea --- /dev/null +++ b/docs/Release-Notes-Configs-40.3.md @@ -0,0 +1,34 @@ +--- +layout: default +title: Release Notes - Mock 40.3 +--- + +## [Release 40.3](https://rpm-software-management.github.io/mock/Release-Notes-40.3) - 2024-04-05 + + +### Mock Core Configs changes + +- Mock chroots for CentOS Stream 10 were added. + They only use the Koji buildroot for now. +- The modular repositories have been dropped from Fedora releases, + as Fedora Modularity has been retired, and these repositories + are no longer maintained. ([PR#1340][]) +- The modular repositories have been dropped from Fedora Rawhide, + as Fedora Modularity has been retired, and these repositories + are no longer maintained. ([PR#1340][]) +- The openSUSE config files have been updated to use the [new `repo_arch` Jinja + template](Release-Notes-5.5) instead of `target_arch`. This change ensures that + the bootstrap-from-image feature works correctly, always choosing the native + architecture (regardless of multilib or forcearch builds). It also ensures that + multilib bootstrap installation works correctly even when the bootstrap image is + OFF. + + +#### Following contributors contributed to this release: + +- Miro Hrončok +- Neal Gompa +- Pavel Raiskup + + +[PR#1340]: https://github.com/rpm-software-management/mock/pull/1340 diff --git a/docs/Release-Notes-Configs-40.5.md b/docs/Release-Notes-Configs-40.5.md new file mode 100644 index 0000000..3c53156 --- /dev/null +++ b/docs/Release-Notes-Configs-40.5.md @@ -0,0 +1,15 @@ +--- +layout: default +title: Release Notes - Mock Configs-40.5 +--- + +## [Release Configs-40.5](https://rpm-software-management.github.io/mock/Release-Notes-Configs-40.5) - 2024-06-05 + + +### Mock Core Configs changes + +- CentOS Stream 8 is EOL. The repositories has been moved to vault. + The corresponding repo file has been updated with new URL. +- The Fedora 38 configuration files have been marked EOL because [Fedora 38 itself + is EOL](https://fedorapeople.org/groups/schedule/f-40/f-40-all-tasks.html). + diff --git a/docs/Release-Notes-Configs-40.6.md b/docs/Release-Notes-Configs-40.6.md new file mode 100644 index 0000000..c319c1a --- /dev/null +++ b/docs/Release-Notes-Configs-40.6.md @@ -0,0 +1,13 @@ +--- +layout: default +title: Release Notes - Mock Configs 40.6 +--- + +## [Release Configs 40.6](https://rpm-software-management.github.io/mock/Release-Notes-Configs-40.6) - 2024-06-15 + + +### Mock Core Configs changes + +- Mock chroots for CentOS Stream 10 now use the mirrored metalinks. + The Koji buildroot is still available with `--enablerepo=local`. + diff --git a/docs/Release-Notes-Configs-41.1.md b/docs/Release-Notes-Configs-41.1.md new file mode 100644 index 0000000..42b7b17 --- /dev/null +++ b/docs/Release-Notes-Configs-41.1.md @@ -0,0 +1,53 @@ +--- +layout: default +title: Release Notes - Mock Configs 41.1 (and typofix 41.2) +--- + +## [Release Configs 41.1](https://rpm-software-management.github.io/mock/Release-Notes-Configs-41.1) - 2024-08-14 + + +### Mock Core Configs changes + +- Configuration files for Fedora 41 have been branched from Rawhide, according + to the [Fedora 41 Schedule](https://fedorapeople.org/groups/schedule/f-41/f-41-all-tasks.html). +- The "early" CentOS Stream 10 + EPEL 10 configuration files have been added, + [issue#1421][]. These chroots only work with Fedora EPEL Koji buildroot(s). +- Add configuration for openEuler 24.03 LTS. +- Mock chroots for CentOS Stream 10 now use the mirrored repositories also for + baseos-source, baseos-debuginfo, appstream-source, appstream-debuginfo, + crb-source, and crb-debuginfo. +- The CentOS 7 [is now EOL](https://www.redhat.com/en/topics/linux/centos-linux-eol) + and the mirroring is disabled. Corresponding configuration files have been + moved to `/etc/mock/eol` and are pointing now to vault.centos.org. + + Similarly, EPEL 7 [goes EOL](https://pagure.io/epel/issue/238), too. Moving + EPEL 7 configuration to `/etc/mock/eol`, too. +- The Fedora ELN ix86 config has been removed, as 32-bit multilibs are no longer + built for ELN. +- Fedora Rawhide configurations, such as releasever=41 now, accept GPG keys from + Fedora releasever+1 (for example, 42, not yet used for RPM signatures). This + change is implemented to address the typically short and unnecessary + inconvenience during [the Fedora branching process][issue#1338] in the future. +- The Fedora Rawhide configuration (F41+) has been updated to use the + `bootstrap_image_ready = True` configuration. The default container images are + [already shipped with the `dnf5-plugins` package](https://pagure.io/fedora-kiwi-descriptions/pull-request/63). + + This means we use the container image "as is" to bootstrap the DNF5 stack + without installing any additional packages into the prepared bootstrap chroot. + Consequently, the bootstrap preparation is much faster (bootstrap preparation + basically equals the image download, if not pre-downloaded, and its subsequent + "extraction"). + +#### Following contributors contributed to this release: + +- Daan De Meyer +- Jakub Kadlcik +- Jiri Kyjovsky +- Miro Hrončok +- nucleo +- Robert Scheck +- Yaakov Selkowitz + +Thank you! + +[issue#1338]: https://github.com/rpm-software-management/mock/issues/1338 diff --git a/docs/Release-Notes-Configs-42.1.md b/docs/Release-Notes-Configs-42.1.md new file mode 100644 index 0000000..79c5613 --- /dev/null +++ b/docs/Release-Notes-Configs-42.1.md @@ -0,0 +1,17 @@ +--- +layout: default +title: Release Notes - Mock Configs 42.1 +--- + +## [Release Configs 42.1](https://rpm-software-management.github.io/mock/Release-Notes-Configs-42.1) - 2025-01-16 + + +### Mock Core Configs changes + +- Configuration files for Fedora 42 have been branched from Rawhide. This has + been done before the actual branching, to make the users' transition fluent + before, during and after the branching event according to the [Fedora 42 + Schedule](https://fedorapeople.org/groups/schedule/f-42/f-42-all-tasks.html). + See [PR#1529][]. + +[PR#1529]: https://github.com/rpm-software-management/mock/pull/1529 diff --git a/docs/Release-Notes-Configs-43.1.md b/docs/Release-Notes-Configs-43.1.md new file mode 100644 index 0000000..c511738 --- /dev/null +++ b/docs/Release-Notes-Configs-43.1.md @@ -0,0 +1,14 @@ +--- +layout: default +title: Release Notes - Mock core configs 43.1 +--- + +## [Release 43.1](https://rpm-software-management.github.io/mock/Release-Notes-Configs-43.1) - 2025-08-12 + +- Configuration files for Fedora 43 have been branched from Rawhide. This has + been done before the actual branching, to make the users' transition fluent + before, during and after the branching event according to the [Fedora 43 + Schedule](https://fedorapeople.org/groups/schedule/f-43/f-43-all-tasks.html). + See [PR#1617][]. + +[PR#1617]: https://github.com/rpm-software-management/mock/pull/1617 diff --git a/docs/Release-Notes-New-Entry.md b/docs/Release-Notes-New-Entry.md new file mode 100644 index 0000000..1dd00a9 --- /dev/null +++ b/docs/Release-Notes-New-Entry.md @@ -0,0 +1,67 @@ +--- +layout: default +title: Maintaining ChangeLog +--- + +# TL;DR - quick start + +You typically want to create a markdown snippet with: + + $ towncrier create failed-selinux-mountpoint.bugfix + Created news fragment at ./releng/release-notes-next/failed-selinux-mountpoint.bugfix + $ vim ./releng/release-notes-next/failed-selinux-mountpoint.bugfix + ... document ... + $ git add ./releng/release-notes-next/failed-selinux-mountpoint.bugfix + +Please refer to issues, PRs, bugs, commits using the +`[reference#ID][]` or `[some text][reference#ID]` syntax described below. + +# Maintaining ChangeLog + +Mock uses the [towncrier](https://github.com/twisted/towncrier) project for +maintaining release notes (aka changelog). For adding a new Release Notes +entry, provide a markdown file in the +[releng/release-notes-next](https://github.com/rpm-software-management/mock/tree/main/releng/release-notes-next) +drop-in directory. + +Each drop-in file is markdown, and the filename must have +`.` pattern. The "unique filename" is +important, but the name is not used anywhere (choose wisely to not collide with +other changes in the next release). For example, let's have a file + + releng/release-notes-next/ssl-certs-fixed.bugfix + +with contents like: + + The SSL certificate copying has been fixed [once more][PR#1113] to use our + own `update_tree()` logic because the `distutils.copy_tree()` was removed + from the Python stdlib, and the new stdlib alternative `shutil.copytree()` + is not powerful enough for the Mock use-cases ([issue#1107][]). + +## Change categories + +Documentation for categories configured in +[towncrier.toml](https://github.com/rpm-software-management/mock/blob/main/towncrier.toml). + + +1. `breaking`: Incompatible change done. This is mentioned at the beginning of + the changelog file to get extra attention. + +1. `bugfix`: Some important bug has been fixed in Mock. + +1. `feature`: New feature in Mock has been implemented. + +1. `config`: Change related to the `mock-core-configs` package. + + +## Referencing issues or pull-requests + +The snippets/drop-in files are in markdown format, so you may simply reference +issues with `[#][]` or `[custom placeholder][#]`. For +example `[rhbz#123456][]` or `[dumping packages][PR#1210]`. Currently +implemented types: + +1. `rhbz#ID`: generates `https://bugzilla.redhat.com/ID +1. `issue#ID`: generates: `https://github.com/rpm-software-management/mock/issues/ID` +1. `PR#ID`: generates: `https://github.com/rpm-software-management/mock/pull/ID` +1. `commit#HASH`: generates: `https://github.com/rpm-software-management/mock/commit/HASH` diff --git a/docs/Releasing-Mock.md b/docs/Releasing-Mock.md new file mode 100644 index 0000000..befb20f --- /dev/null +++ b/docs/Releasing-Mock.md @@ -0,0 +1,189 @@ +--- +layout: default +title: How do we release Mock +--- + +# Mock versions + +We keep the "major.minor" versioning scheme, and we bump the "major" number when +some important/notable new feature appears, or when breaking change happens. + +For the `mock-core-configs` package, the `major` number reflects the latest +Fedora **branched** version from Rawhide. + +Note we maintain multiple Mock versions in parallel. See `README.md` file +for more info. You might need to repeat the steps described in this document +multiple times. Even though `tito` should have appropriate releasers +configured, please be careful when updating various distributions and use the +correct branch version. + +## Fedora Rawhide branching + +When you plan to release a new mock-core-configs for a new Fedora version being +branched from Rawhide, there's a script in releng/rawhide-branching.sh that +helps you to setup a correct configuration layout. + +For Rawhide releases, you'll primarily be updating mock-core-configs only. Follow the +steps in the release checklist, but focus only on the mock-core-configs parts. +Be sure to use the `--config-only` flag when generating release notes. + +## Release checklist overview + +0. Make sure all GitHub CI checks are passing for the latest commit + +1. Change to the correct local branch, e.g. `main` + + $ git checkout main + +2. Fetch git remote, and propose local-only patches (if any) + + $ git pull --rebase main + +3. If you want to propose the release through a PR, switch to a new branch + + $ git checkout -b release-2024-05-15 + +5. Prepare release notes. Use the going-to-be-released version, not + the current version. + + For a full release (both mock and mock-core-configs): + + $ sudo dnf install towncrier + $ ./releng/generate-release-notes --use-version 5.1 + $ vim docs/Release-Notes-5.1.md # modify manually! + + For a mock-core-configs only release: + + $ ./releng/generate-release-notes --use-version 40.3 --config-only + $ vim docs/Release-Notes-Configs-40.3.md # modify manually! + + Don't forget to manually modify ./docs/index.md and mention the new release. + + Add list of contributing authors: + + For mock: + $ git log mock-4.1-1..HEAD --format="%aN" mock/ | sort | uniq + + For mock-core-configs: + $ git log mock-core-configs-38.1-1..HEAD --format="%aN" mock-core-configs/ | sort | uniq + +6. Commit all the pending changes + +7. On your box (you need push-access rights), tag the git tree: + + For a full release (both mock and mock-core-configs): + + # First tag mock-core-configs + $ cd ./mock-core-configs + $ tito tag --use-version 40.3 # major.minor according to policy + + # Then update mock's spec and tag mock + # Update 'Conflicts: mock-core-configs < ??' in mock.spec + $ cd ./mock + $ tito tag --use-version 5.1 # major.minor according to policy + + For a mock-core-configs only release: + + $ cd ./mock-core-configs + $ tito tag --use-version 40.3 # major.minor according to policy + + For a mock only release: + + $ cd ./mock + $ tito tag --use-version 5.1 # major.minor according to policy + +8. Push the tito-generated commits to the upstream repo + + Suggestion: You can calmly avoid doing direct pushes. You can simply remove + the local-only git tag created by tito, and submit PR the normal way: + + $ git tag -d mock-4.1-1 # drop tag + $ git branch new-version proposal + $ git push yourremote new-proposal + ... + remote: Create a pull request for 'new-proposal' on GitHub by visiting: + remote: https://github.com/praiskup/mock/pull/new/new-proposal + ... + + These tito-generated commits can calmly be squashed, updated, etc. (you may + include documentation fixes there). Just don't forget to re-add the dropped + tag back (if dropped) 'git tag mock-4.1-1'. Alternatively just `git push` + the commits. + +9. Push the git tags upstream, e.g. + + For mock-core-configs: + $ git push origin mock-core-configs-40.4-1 + + For mock: + $ git push origin mock-5.1-1 + +10. Release for EPEL and Fedora + + $ # make sure that .tito/releasers.conf is up to date + + For mock-core-configs: + $ cd ./mock-core-configs + $ tito release fedora-git-all + + For mock: + $ cd ./mock + $ tito release fedora-git-all + +11. publish tgz + + - The `tito release` calls above uploaded the tarball to Fedora DistGit + lookaside cache. You can simply use those. Alternative you should be able + to generate the same (byte-by-byte) tarball via `tito build --tgz` + + - Go to the [releases page](https://github.com/rpm-software-management/mock/releases), + + - click "Draft new release" + + - Choose existing tag. E.g., `mock-1.4.9-1 @ main` + + - Enter the same tag as release "title" + + - Attach the tarball binary + +12. Once the builds finish (successfully) you should push the just built packages + to their respective testing repositories. This can be done either with the + Bodhi WebUI at https://bodhi.fedoraproject.org/ or if there are no other + package dependencies, using the 'fedpkg update' command (you can specify + multiple builds, even for multiple fedora/epel versions, like + `[mock-1.35-1.fc38, mock-1.35-1.fc37, ...]`). Note that you do not need to + do this for the Fedora Rawhide build since it automatically gets pushed to + testing. + +13. Announce the release + + We typically send announcement e-mails to + [bulidsys@lists.fedoraproject.org](https://lists.fedoraproject.org/archives/list/buildsys@lists.fedoraproject.org/), + and depending on the severity of the release also to + [devel@lists.fedoraproject.org](https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.org/). + + We also announce in [Fosstodon CPT's space](https://fosstodon.org/@fedoracpt). + + For a full release (both mock and mock-core-configs): + + Subject: Mock v4.0 released (and mock-core-configs v38.5) + + Hello maintainers! + + I'm glad I can announce that we have a new release of Mock v + (the chroot build environment manager for building RPMs). + + + + Full release notes: + + https://rpm-software-management.github.io/mock/Release-Notes-4.0 + + The updated packages are in Bodhi: + + [Fedora 38]: + [Fedora 37]: + [EPEL 9]: + [EPEL 8]: + + Happy building! diff --git a/docs/Why-Mock.md b/docs/Why-Mock.md new file mode 100644 index 0000000..e732b74 --- /dev/null +++ b/docs/Why-Mock.md @@ -0,0 +1,138 @@ +# Why Mock? + +This document tries to answer two questions: + +1. Why you should use Mock instead of lower-level tooling like `rpmbuild` +2. Why you should not try to reimplement Mock and rather join the project + + +## Mock builds packages in a dedicated environment + +Builds are done in a dedicated buildroot that contains only a minimal set of +packages. As a consequence, build failures are deterministic and reproducible +because they cannot be affected by something that user installed/configured on +their host system. + +Additionally, building RPM packages is inherently dangerous because specfiles +are written in a Turing-complete language. Building an untrusted package can +easily lead to a compromised system. This is also true for self-written +packages. One small bug like `rm -rf %{?nonexisting}/` can wipe your entire +system if you are unwise enough to run `rpmbuild` as `root`. Or at least your +user home directory. + +Mock does many parts of the build process inside a `systemd-nspawn` +container (or at least [chroot][chroot]) to avoid such disasters as much as +possible. + +For fully offline builds, Mock supports [hermetic builds][hermetic-builds]. + +## Mock can build packages for different distributions + +Mock comes with the `mock-core-configs` package which provides a list +[configuration files for many RPM-based distributions][mock-core-configs]. +Mock allows building a package for any of them, regardless of what distribution +is installed on your host system. + +Users can easily create and distribute +[their own configuration files][mock-configuration]. + +Let's say you use Fedora 41 as your operating system and want to build a package +for RHEL 8 (or any other distribution). Mock then reads the configuration file +to know what repositories should use, what macros to define, what packages +to install into the minimal buildroot, etc. It does all that, and then builds +the package inside this chroot. + + +## Mock installs build-time dependencies + +Unlike `rpmbuild`, Mock automatically installs all `BuildRequires` dependencies +needed by the package. It also supports [dynamic dependencies][DynamicBuildRequires] +through `%generate_buildrequires`, which is the standard way of dealing with +dependencies for Python, Go, Rust, and other languages. + +Additionally, as an opt-in, Mock can install +[dependencies from non-RPM sources][external-dependencies] like PyPI, RubyGems, +etc. + + +## Mock is not constrained by the host system + +You can easily prepare minimal buildroot with something like this: + +``` +dnf --installroot /tmp/buildroot --use-host-config install @buildsys-build +``` + +But what if, instead of using your system configuration, you want to use +repositories for a different distribution. In some cases, it works just fine. + +But if the distribution is older (e.g. RHEL) than your host system, it may use a +different package manager (e.g. Yum) which can install a different set of +packages, the metadata can use different hashing algorithms, etc. + +Conversely, if the distribution is newer (e.g. Fedora Rawhide) it may require +some RPM features or macros, that are not yet available on the host system. + +Mock solves this by using a [bootstrap chroot][bootstrap-chroot]. + +This feature unlocked a lot of innovation in RPM because the development is not +limited by the OS used in the builder infrastructure anymore. + + +## Mock prepares a directory structure for rpmbuild + +Before `rpmbuild` can be used, it requires a specific directory structure to be +created. Users typically do it by running `rpmdev-setuptree` on their systems, +but inside of a chroot, the structure needs to be created manually with correct +location and permissions. + + +## Mock elevates permissions responsibly + +Mock does a lot of things to prepare a buildroot. For some of them +(e.g. installing build dependencies) Mock needs to elevate permissions to +root. But it does so only for a limited time-frame, and drops back to normal +user as soon as possible. + + +## Mock sets build-specific macros + +Some distributions or architectures require specific macros to be set, so that +RPM packages are built correctly. For example, some chroots set `%_host_cpu` to +`ppc64le`, some chroots adjust the `%dist` macro, etc. The `mock-core-configs` +package provides reasonable defaults, but on top of that, users define or change +any macro they want. + +The same goes for DNF/Yum placeholders like `$releasever`, `$arch`, etc. + + +## Mock prepares special devices + +Some packages require special devices like `/dev/fuse` `/dev/loop-control`, +`/dev/hwrng`, `/dev/prandom`, etc to build successfully. Mock makes them +available inside of the buildroot. + + +## Mock caches downloaded RPM packages + +Not all dependencies have to be downloaded every time. This brings significant +performance improvements especially when consecutively rebuilding the same RPM +package or a package from the same ecosystem (e.g. multiple Python packages). + +Mock also caches DNF metadata (often tens of megabytes), which is significant +because downloading metadata was a well-known bottleneck before DNF5. + + +## Also worth mentioning: + +- Mock uses the same timezone as the host system + + + +[bootstrap-chroot]: https://rpm-software-management.github.io/mock/Feature-bootstrap +[DynamicBuildRequires]: https://fedoraproject.org/wiki/Changes/DynamicBuildRequires +[external-dependencies]: https://rpm-software-management.github.io/mock/Feature-external-deps +[mock-core-configs]: https://rpm-software-management.github.io/mock/Mock-Core-Configs +[hermetic-builds]: https://rpm-software-management.github.io/mock/feature-hermetic-builds +[mock-configuration]: https://rpm-software-management.github.io/mock/configuration +[chroot]: https://en.wikipedia.org/wiki/Chroot diff --git a/docs/_Sidebar.md b/docs/_Sidebar.md new file mode 100644 index 0000000..c56ec91 --- /dev/null +++ b/docs/_Sidebar.md @@ -0,0 +1,15 @@ +--- +layout: default +--- + + +- [[Home]] +- [[Release Notes|Home#release-notes]] +- [[FAQs|FAQ]] +- [[Plugins|Home#plugins]] +- [[Features|Home#features]] +- [Developers](https://github.com/rpm-software-management/mock) +- [Nightlies](https://copr.fedorainfracloud.org/coprs/g/mock/mock/) + +### Help +- [Need help?](https://github.com/rpm-software-management/mock#communication) diff --git a/docs/_config.yml b/docs/_config.yml new file mode 100644 index 0000000..c8e4a45 --- /dev/null +++ b/docs/_config.yml @@ -0,0 +1,15 @@ +title: Mock +description: A 'simple' chroot build environment manager for building RPMs. +show_downloads: true +google_analytics: +theme: jekyll-theme-slate + +show_downloads: false +github: + is_project_page: true + repository_url: https://github.com/rpm-software-management/mock/ + owner_name: Red Hat + owner_url: https://www.redhat.com + +plugins: + - jemoji diff --git a/docs/_includes/head-custom-google-analytics.html b/docs/_includes/head-custom-google-analytics.html new file mode 100644 index 0000000..8a3ae5c --- /dev/null +++ b/docs/_includes/head-custom-google-analytics.html @@ -0,0 +1,10 @@ +{% if site.google_analytics %} + +{% endif %} diff --git a/docs/_includes/head-custom.html b/docs/_includes/head-custom.html new file mode 100644 index 0000000..f7187e7 --- /dev/null +++ b/docs/_includes/head-custom.html @@ -0,0 +1,9 @@ + + + +{% include head-custom-google-analytics.html %} + + + + + diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html new file mode 100644 index 0000000..45d8396 --- /dev/null +++ b/docs/_layouts/default.html @@ -0,0 +1,52 @@ + + + + + + + + + +{% seo %} + {% include head-custom.html %} + + + + + +
+
+ {% if site.github.is_project_page %} + View on GitHub + {% endif %} + +

{{ site.title | default: site.github.repository_name }} » {{ page.title }}

+

{{ site.description | default: site.github.project_tagline }}

+ + {% if site.show_downloads %} +
+ Download this project as a .zip file + Download this project as a tar.gz file +
+ {% endif %} +
+
+ + +
+
+ {{ content }} +
+
+ + + + + diff --git a/docs/_layouts/title.html b/docs/_layouts/title.html new file mode 100644 index 0000000..7cdf2bf --- /dev/null +++ b/docs/_layouts/title.html @@ -0,0 +1,52 @@ + + + + + + + + + +{% seo %} + {% include head-custom.html %} + + + + + +
+
+ {% if site.github.is_project_page %} + View on GitHub + {% endif %} + +

{{ site.title | default: site.github.repository_name }}

+

{{ site.description | default: site.github.project_tagline }}

+ + {% if site.show_downloads %} +
+ Download this project as a .zip file + Download this project as a tar.gz file +
+ {% endif %} +
+
+ + +
+
+ {{ content }} +
+
+ + + + + diff --git a/docs/_sass/jekyll-theme-slate.scss b/docs/_sass/jekyll-theme-slate.scss new file mode 100644 index 0000000..b721960 --- /dev/null +++ b/docs/_sass/jekyll-theme-slate.scss @@ -0,0 +1,488 @@ +@import "rouge-github"; + +/******************************************************************************* +MeyerWeb Reset +*******************************************************************************/ + +html, body, div, span, applet, object, iframe, +h1, h2, h3, h4, h5, h6, p, blockquote, pre, +a, abbr, acronym, address, big, cite, code, +del, dfn, em, img, ins, kbd, q, s, samp, +small, strike, strong, sub, sup, tt, var, +b, u, i, center, +dl, dt, dd, ol, ul, li, +fieldset, form, label, legend, +table, caption, tbody, tfoot, thead, tr, th, td, +article, aside, canvas, details, embed, +figure, figcaption, footer, header, hgroup, +menu, nav, output, ruby, section, summary, +time, mark, audio, video { + margin: 0; + padding: 0; + border: 0; + font: inherit; + vertical-align: baseline; +} + +/* HTML5 display-role reset for older browsers */ +article, aside, details, figcaption, figure, +footer, header, hgroup, menu, nav, section { + display: block; +} + +ol, ul { + list-style: none; +} + +table { + border-collapse: collapse; + border-spacing: 0; +} + +/******************************************************************************* +Theme Styles +*******************************************************************************/ + +body { + box-sizing: border-box; + color:#373737; + background: #212121; + font-size: 16px; + font-family: 'Myriad Pro', Calibri, Helvetica, Arial, sans-serif; + line-height: 1.5; + -webkit-font-smoothing: antialiased; +} + +h1, h2, h3, h4, h5, h6 { + margin: 10px 0; + font-weight: 700; + color:#222222; + font-family: 'Lucida Grande', 'Calibri', Helvetica, Arial, sans-serif; + letter-spacing: -1px; +} + +h1 { + font-size: 36px; + font-weight: 700; +} + +h2 { + padding-bottom: 10px; + font-size: 32px; + background: url('../images/bg_hr.png') repeat-x bottom; +} + +h3 { + font-size: 24px; +} + +h4 { + font-size: 21px; +} + +h5 { + font-size: 18px; +} + +h6 { + font-size: 16px; +} + +p { + margin: 10px 0 15px 0; +} + +footer p { + color: #f2f2f2; +} + +a { + text-decoration: none; + color: #0F79D0; + text-shadow: none; + + transition: color 0.5s ease; + transition: text-shadow 0.5s ease; + -webkit-transition: color 0.5s ease; + -webkit-transition: text-shadow 0.5s ease; + -moz-transition: color 0.5s ease; + -moz-transition: text-shadow 0.5s ease; + -o-transition: color 0.5s ease; + -o-transition: text-shadow 0.5s ease; + -ms-transition: color 0.5s ease; + -ms-transition: text-shadow 0.5s ease; +} + +a:hover, a:focus { + text-decoration: underline; +} + +footer a { + color: #F2F2F2; + text-decoration: underline; +} + +em, cite { + font-style: italic; +} + +strong { + font-weight: bold; +} + +img { + position: relative; + margin: 0 auto; + max-width: 739px; + padding: 5px; + margin: 10px 0 10px 0; + border: 1px solid #ebebeb; + + box-shadow: 0 0 5px #ebebeb; + -webkit-box-shadow: 0 0 5px #ebebeb; + -moz-box-shadow: 0 0 5px #ebebeb; + -o-box-shadow: 0 0 5px #ebebeb; + -ms-box-shadow: 0 0 5px #ebebeb; +} + +p img { + display: inline; + margin: 0; + padding: 0; + vertical-align: middle; + text-align: center; + border: none; +} + +pre, code { + color: #222; + background-color: #fff; + + font-family: Monaco, "Bitstream Vera Sans Mono", "Lucida Console", Terminal, monospace; + font-size: 0.875em; + + border-radius: 2px; + -moz-border-radius: 2px; + -webkit-border-radius: 2px; +} + +pre { + padding: 10px; + box-shadow: 0 0 10px rgba(0,0,0,.1); + overflow: auto; +} + +code { + padding: 3px; + margin: 0 3px; + box-shadow: 0 0 10px rgba(0,0,0,.1); +} + +pre code { + display: block; + box-shadow: none; +} + +blockquote { + color: #666; + margin-bottom: 20px; + padding: 0 0 0 20px; + border-left: 3px solid #bbb; +} + + +ul, ol, dl { + margin-bottom: 15px +} + +ul { + list-style-position: inside; + list-style: disc; + padding-left: 20px; +} + +ol { + list-style-position: inside; + list-style: decimal; + padding-left: 20px; +} + +dl dt { + font-weight: bold; +} + +dl dd { + padding-left: 20px; + font-style: italic; +} + +dl p { + padding-left: 20px; + font-style: italic; +} + +hr { + height: 1px; + margin-bottom: 5px; + border: none; + background: url('../images/bg_hr.png') repeat-x center; +} + +table { + border: 1px solid #373737; + margin-bottom: 20px; + text-align: left; + } + +th { + font-family: 'Lucida Grande', 'Helvetica Neue', Helvetica, Arial, sans-serif; + padding: 10px; + background: #373737; + color: #fff; + } + +td { + padding: 10px; + border: 1px solid #373737; + } + +form { + background: #f2f2f2; + padding: 20px; +} + +kbd { + background-color: #fafbfc; + border: 1px solid #c6cbd1; + border-bottom-color: #959da5; + border-radius: 3px; + box-shadow: inset 0 -1px 0 #959da5; + color: #444d56; + display: inline-block; + font-size: 11px; + line-height: 11px; + padding: 3px 5px; + vertical-align: middle; +} + +/******************************************************************************* +Full-Width Styles +*******************************************************************************/ + +.outer { + width: 100%; +} + +.inner { + position: relative; + max-width: 640px; + padding: 20px 10px; + margin: 0 auto; +} + +#forkme_banner { + display: block; + position: absolute; + top:0; + right: 10px; + z-index: 10; + padding: 10px 50px 10px 10px; + color: #fff; + background: url('../images/blacktocat.png') #0090ff no-repeat 95% 50%; + font-weight: 700; + box-shadow: 0 0 10px rgba(0,0,0,.5); + border-bottom-left-radius: 2px; + border-bottom-right-radius: 2px; +} + +#header_wrap { + background: #212121; + background: -moz-linear-gradient(top, #373737, #212121); + background: -webkit-linear-gradient(top, #373737, #212121); + background: -ms-linear-gradient(top, #373737, #212121); + background: -o-linear-gradient(top, #373737, #212121); + background: linear-gradient(to top, #373737, #212121); +} + +#header_wrap .inner { + padding: 50px 10px 30px 10px; +} + +#project_title { + margin: 0; + color: #fff; + font-size: 42px; + font-weight: 700; + text-shadow: #111 0px 0px 10px; +} + +#project_tagline { + color: #fff; + font-size: 24px; + font-weight: 300; + background: none; + text-shadow: #111 0px 0px 10px; +} + +#downloads { + position: absolute; + width: 210px; + z-index: 10; + bottom: -40px; + right: 0; + height: 70px; + background: url('../images/icon_download.png') no-repeat 0% 90%; +} + +.zip_download_link { + display: block; + float: right; + width: 90px; + height:70px; + text-indent: -5000px; + overflow: hidden; + background: url(../images/sprite_download.png) no-repeat bottom left; +} + +.tar_download_link { + display: block; + float: right; + width: 90px; + height:70px; + text-indent: -5000px; + overflow: hidden; + background: url(../images/sprite_download.png) no-repeat bottom right; + margin-left: 10px; +} + +.zip_download_link:hover { + background: url(../images/sprite_download.png) no-repeat top left; +} + +.tar_download_link:hover { + background: url(../images/sprite_download.png) no-repeat top right; +} + +#main_content_wrap { + background: #f2f2f2; + border-top: 1px solid #111; + border-bottom: 1px solid #111; +} + +#main_content { + padding-top: 40px; +} + +#footer_wrap { + background: #212121; +} + + + +/******************************************************************************* +Small Device Styles +*******************************************************************************/ + +@media screen and (max-width: 992px) { + img { + max-width: 100%; + } +} + +@media screen and (max-width: 480px) { + body { + font-size:14px; + } + + #downloads { + display: none; + } + + .inner { + min-width: 320px; + max-width: 480px; + } + + #project_title { + font-size: 32px; + } + + h1 { + font-size: 28px; + } + + h2 { + font-size: 24px; + } + + h3 { + font-size: 21px; + } + + h4 { + font-size: 18px; + } + + h5 { + font-size: 14px; + } + + h6 { + font-size: 12px; + } + + code, pre { + font-size: 11px; + } + +} + +@media screen and (max-width: 320px) { + body { + font-size:14px; + } + + #downloads { + display: none; + } + + .inner { + min-width: 240px; + max-width: 320px; + } + + #project_title { + font-size: 28px; + } + + h1 { + font-size: 24px; + } + + h2 { + font-size: 21px; + } + + h3 { + font-size: 18px; + } + + h4 { + font-size: 16px; + } + + h5 { + font-size: 14px; + } + + h6 { + font-size: 12px; + } + + code, pre { + min-width: 240px; + max-width: 320px; + font-size: 11px; + } + +} diff --git a/docs/_sass/rouge-github.scss b/docs/_sass/rouge-github.scss new file mode 100644 index 0000000..bd7fc73 --- /dev/null +++ b/docs/_sass/rouge-github.scss @@ -0,0 +1,209 @@ +.highlight table td { padding: 5px; } +.highlight table pre { margin: 0; } +.highlight .cm { + color: #777772; + font-style: italic; +} +.highlight .cp { + color: #797676; + font-weight: bold; +} +.highlight .c1 { + color: #777772; + font-style: italic; +} +.highlight .cs { + color: #797676; + font-weight: bold; + font-style: italic; +} +.highlight .c, .highlight .cd { + color: #777772; + font-style: italic; +} +.highlight .err { + color: #a61717; + background-color: #e3d2d2; +} +.highlight .gd { + color: #000000; + background-color: #ffdddd; +} +.highlight .ge { + color: #000000; + font-style: italic; +} +.highlight .gr { + color: #aa0000; +} +.highlight .gh { + color: #797676; +} +.highlight .gi { + color: #000000; + background-color: #ddffdd; +} +.highlight .go { + color: #888888; +} +.highlight .gp { + color: #555555; +} +.highlight .gs { + font-weight: bold; +} +.highlight .gu { + color: #aaaaaa; +} +.highlight .gt { + color: #aa0000; +} +.highlight .kc { + color: #000000; + font-weight: bold; +} +.highlight .kd { + color: #000000; + font-weight: bold; +} +.highlight .kn { + color: #000000; + font-weight: bold; +} +.highlight .kp { + color: #000000; + font-weight: bold; +} +.highlight .kr { + color: #000000; + font-weight: bold; +} +.highlight .kt { + color: #445588; + font-weight: bold; +} +.highlight .k, .highlight .kv { + color: #000000; + font-weight: bold; +} +.highlight .mf { + color: #009999; +} +.highlight .mh { + color: #009999; +} +.highlight .il { + color: #009999; +} +.highlight .mi { + color: #009999; +} +.highlight .mo { + color: #009999; +} +.highlight .m, .highlight .mb, .highlight .mx { + color: #009999; +} +.highlight .sb { + color: #d14; +} +.highlight .sc { + color: #d14; +} +.highlight .sd { + color: #d14; +} +.highlight .s2 { + color: #d14; +} +.highlight .se { + color: #d14; +} +.highlight .sh { + color: #d14; +} +.highlight .si { + color: #d14; +} +.highlight .sx { + color: #d14; +} +.highlight .sr { + color: #009926; +} +.highlight .s1 { + color: #d14; +} +.highlight .ss { + color: #990073; +} +.highlight .s { + color: #d14; +} +.highlight .na { + color: #008080; +} +.highlight .bp { + color: #797676; +} +.highlight .nb { + color: #0086B3; +} +.highlight .nc { + color: #445588; + font-weight: bold; +} +.highlight .no { + color: #008080; +} +.highlight .nd { + color: #3c5d5d; + font-weight: bold; +} +.highlight .ni { + color: #800080; +} +.highlight .ne { + color: #990000; + font-weight: bold; +} +.highlight .nf { + color: #990000; + font-weight: bold; +} +.highlight .nl { + color: #990000; + font-weight: bold; +} +.highlight .nn { + color: #555555; +} +.highlight .nt { + color: #000080; +} +.highlight .vc { + color: #008080; +} +.highlight .vg { + color: #008080; +} +.highlight .vi { + color: #008080; +} +.highlight .nv { + color: #008080; +} +.highlight .ow { + color: #000000; + font-weight: bold; +} +.highlight .o { + color: #000000; + font-weight: bold; +} +.highlight .w { + color: #bbbbbb; +} +.highlight { + background-color: #f8f8f8; +} diff --git a/docs/_sass/slate.scss b/docs/_sass/slate.scss new file mode 100644 index 0000000..8e0c330 --- /dev/null +++ b/docs/_sass/slate.scss @@ -0,0 +1,4 @@ +// Placeholder file. If your site uses +// @import "{{ site.theme }}"; +// Then using this theme with jekyll-remote-theme will work fine. +@import "jekyll-theme-slate"; diff --git a/docs/assets/css/style.scss b/docs/assets/css/style.scss new file mode 100644 index 0000000..cdc2e1d --- /dev/null +++ b/docs/assets/css/style.scss @@ -0,0 +1,4 @@ +--- +--- + +@import "jekyll-theme-slate"; diff --git a/docs/assets/images/bg_hr.png b/docs/assets/images/bg_hr.png new file mode 100644 index 0000000..514aee5 Binary files /dev/null and b/docs/assets/images/bg_hr.png differ diff --git a/docs/assets/images/blacktocat.png b/docs/assets/images/blacktocat.png new file mode 100644 index 0000000..e160053 Binary files /dev/null and b/docs/assets/images/blacktocat.png differ diff --git a/docs/assets/images/icon_download.png b/docs/assets/images/icon_download.png new file mode 100644 index 0000000..5a793f1 Binary files /dev/null and b/docs/assets/images/icon_download.png differ diff --git a/docs/assets/images/sprite_download.png b/docs/assets/images/sprite_download.png new file mode 100644 index 0000000..f9f8de2 Binary files /dev/null and b/docs/assets/images/sprite_download.png differ diff --git a/docs/configuration.md b/docs/configuration.md index 45e779c..6d103b4 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1,209 +1,109 @@ -# Deb-Mock Configuration +--- +layout: default +title: Mock configuration +--- -Deb-Mock uses YAML configuration files to define build environments and behavior. This document describes all available configuration options. +## Mock configuration files -## Configuration File Format +Syntactically, Mock configuration files are just Python files. But you should +be rather conservative and use just the `config_opts[]` dictionary because we'd +like to [change the format in the future](https://github.com/rpm-software-management/mock/issues/1060). -Deb-Mock configuration files use YAML format. Here's a complete example: +Mock RPM package self-documents all the available options, take a look at this +file: -```yaml -# Basic configuration -chroot_name: bookworm-amd64 -architecture: amd64 -suite: bookworm -output_dir: ./output -keep_chroot: false -verbose: false -debug: false + $ rpm -qd mock | grep site-defaults + /usr/share/doc/mock/site-defaults.cfg -# Chroot configuration -chroot_dir: /var/lib/deb-mock/chroots -chroot_config_dir: /etc/schroot/chroot.d +Mock configuration files can be logically divided into *generic* (used for every +executed Mock command) and *chroot* configuration (used only if the +corresponding chroot is selected, see below). -# sbuild configuration -sbuild_config: /etc/sbuild/sbuild.conf -sbuild_log_dir: /var/log/sbuild +Both the *generic* and *chroot* configuration can be done on either +*system* level (`/etc/mock` directory) or on *user* level (files in +`$HOME/.config` directory). -# Build configuration -build_deps: [] -build_env: - DEB_BUILD_OPTIONS: parallel=4 - DEB_BUILD_PROFILES: nocheck -build_options: - - --verbose - - --debug -# Metadata configuration -metadata_dir: ./metadata -capture_logs: true -capture_changes: true -``` +### Selecting a chroot config -## Configuration Options +For example to initialize a Fedora Rawhide x86_64 chroot (using +`/etc/mock/fedora-rawhide-x86_64.cfg` file), and switch into the chroot, one can +do: -### Basic Configuration + $ mock -r fedora-rawhide-x86_64 --shell -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `chroot_name` | string | `bookworm-amd64` | Name of the chroot environment to use | -| `architecture` | string | `amd64` | Target architecture for builds | -| `suite` | string | `bookworm` | Debian suite (bookworm, sid, bullseye, buster) | -| `output_dir` | string | `./output` | Directory for build artifacts | -| `keep_chroot` | boolean | `false` | Whether to keep chroot after build | -| `verbose` | boolean | `false` | Enable verbose output | -| `debug` | boolean | `false` | Enable debug output | +Note we are not using the `.cfg` suffix in the `-r` option in this case. This +way the *user* level `$HOME/.config` files are searched for the corresponding +`.cfg` file first, and since nothing is found, then the *system* level file is +found in `/etc/mock` (and used). -### Chroot Configuration +One can though use a config pathname with the `-r` option, too. But the +pathname must represent an existing file (accessible from the current working +directory): -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `chroot_dir` | string | `/var/lib/deb-mock/chroots` | Directory for chroot environments | -| `chroot_config_dir` | string | `/etc/schroot/chroot.d` | Directory for schroot configuration files | + $ mock -r ./subdir/existing-config-file.cfg --shell + $ mock -r /etc/mock/fedora-35-x86_64.cfg -### sbuild Configuration +### Generic configuration changes -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `sbuild_config` | string | `/etc/sbuild/sbuild.conf` | Path to sbuild configuration file | -| `sbuild_log_dir` | string | `/var/log/sbuild` | Directory for sbuild logs | +Typically the file `$HOME/.config/mock.cfg` should be used for *generic* +configuration changes for a single user. If a *system* Mock behavior change +is desired (for all system users), then use `/etc/mock/site-defaults.cfg`. -### Build Configuration +The `site-defaults.cfg` is typically empty by default, but contains a basic +documentation and a valid link to a **complete configuration documentation**. +That documentation typically is `/usr/share/doc/mock/site-defaults.cfg` +(location may vary depending on your host system conventions). -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `build_deps` | list | `[]` | Additional build dependencies to install | -| `build_env` | dict | `{}` | Environment variables for builds | -| `build_options` | list | `[]` | Additional sbuild options | -### Metadata Configuration +### Chroot configuration changes -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `metadata_dir` | string | `./metadata` | Directory for build metadata | -| `capture_logs` | boolean | `true` | Whether to capture build logs | -| `capture_changes` | boolean | `true` | Whether to capture .changes files | +There are `/etc/mock/.cfg` files for various build chroots that +contain various compatibility settings related to the target distribution +(location of RPM repositories, if DNF or YUM should be used, working directory +to be used, and so on). -## Supported Architectures +These system files are shipped via the `mock-core-configs` (or other), and users +are discouraged from changing these (change would break the automatic update of +such file with an updated version of the package). It is safer to install an +*override* configuration file: -- `amd64` - 64-bit x86 -- `i386` - 32-bit x86 -- `arm64` - 64-bit ARM -- `armhf` - 32-bit ARM (hard float) -- `ppc64el` - 64-bit PowerPC (little endian) -- `s390x` - 64-bit IBM S390 + $ cat $HOME/.config/mock/fedora-35-x86_64.cfg + # include the default configuration + include("/etc/mock/fedora-35-x86_64.cfg") + # install make into the minimal build chroot + config_opts['chroot_additional_packages'] = 'make' -## Supported Suites +You may also copy and edit an existing configuration file into a new one: -- `bookworm` - Debian 12 (stable) -- `sid` - Debian unstable -- `bullseye` - Debian 11 (oldstable) -- `buster` - Debian 10 (oldoldstable) + $ cp /etc/mock/fedora-rawhide-x86_64.cfg ~/.config/mock/foo.cfg -## Environment Variables +The default chroot configuration file is `/etc/mock/default.cfg`, which is +usually a symlink to one of the installed chroot configuration files. You may +create another symlink to an installed configuration file to change the default +chroot config file: -Common environment variables you can set in `build_env`: + $ ln -s /etc/mock/fedora-rawhide-x86_64.cfg ~/.config/mock/default.cfg -```yaml -build_env: - DEB_BUILD_OPTIONS: parallel=4,nocheck - DEB_BUILD_PROFILES: nocheck - DEB_CFLAGS_SET: -O2 - DEB_CXXFLAGS_SET: -O2 - DEB_LDFLAGS_SET: -Wl,-z,defs -``` +If Koji is already using a config you need, then you can use the Koji client +tool for generating the file: -## Build Options + $ koji mock-config --tag f21-build --arch=aarch64 f21 > ~/.config/mock/foo.cfg -Common sbuild options you can add to `build_options`: +Similar functionality has the Copr client tool: -```yaml -build_options: - - --verbose - - --debug - - --no-clean-source - - --no-run-lintian - - --no-run-autopkgtest -``` + $ copr mock-config @copr/copr-dev fedora-21-x86_64 > ~/.config/mock/foo.cfg -## Configuration File Locations +When your file `foo.cfg` is installed, you can just do `mock -r foo [...]`. -Deb-Mock looks for configuration files in the following order: +### Order of loading the files -1. Command line specified file (`--config` option) -2. `./deb-mock.conf` -3. `~/.config/deb-mock/config.yaml` -4. `/etc/deb-mock/config.yaml` -5. Default configuration +The order of reading and evaluating configuration files in Mock is the following: -## Example Configurations +1. `/etc/mock/site-defaults.cfg` +1. `/etc/mock/.cfg` or `~/.config/mock/.cfg` +1. `~/.mock/user.cfg` +1. `~/.config/mock.cfg` (since `mock-1.2.15`) -### Minimal Configuration - -```yaml -chroot_name: bookworm-amd64 -architecture: amd64 -suite: bookworm -``` - -### Development Configuration - -```yaml -chroot_name: sid-amd64 -architecture: amd64 -suite: sid -output_dir: ./build-output -keep_chroot: true -verbose: true -debug: true -build_env: - DEB_BUILD_OPTIONS: parallel=8,nocheck - DEB_BUILD_PROFILES: nocheck -build_options: - - --verbose - - --no-run-lintian -``` - -### Production Configuration - -```yaml -chroot_name: bookworm-amd64 -architecture: amd64 -suite: bookworm -output_dir: /var/lib/deb-mock/output -keep_chroot: false -verbose: false -debug: false -build_env: - DEB_BUILD_OPTIONS: parallel=4 -build_options: - - --run-lintian - - --run-autopkgtest -metadata_dir: /var/lib/deb-mock/metadata -``` - -## Validation - -Deb-Mock validates configuration files and will report errors for: - -- Invalid architectures -- Invalid suites -- Missing required directories -- Invalid file paths - -## Command Line Overrides - -Most configuration options can be overridden on the command line: - -```bash -# Override chroot -deb-mock build --chroot=sid-amd64 package.dsc - -# Override architecture -deb-mock build --arch=arm64 package.dsc - -# Override output directory -deb-mock build --output-dir=/tmp/build package.dsc - -# Keep chroot for debugging -deb-mock build --keep-chroot package.dsc -``` \ No newline at end of file +I.e. the value set in the later configuration file overrides the value set by +previously loaded files. diff --git a/docs/docs/CODE_OF_CONDUCT.md b/docs/docs/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..ab1a9a1 --- /dev/null +++ b/docs/docs/CODE_OF_CONDUCT.md @@ -0,0 +1,46 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at opensource@github.com. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, available at [http://contributor-covenant.org/version/1/4][version] + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/docs/docs/CONTRIBUTING.md b/docs/docs/CONTRIBUTING.md new file mode 100644 index 0000000..04b6db8 --- /dev/null +++ b/docs/docs/CONTRIBUTING.md @@ -0,0 +1,88 @@ +# Contributing to the Slate theme + +Hi there! We're thrilled that you'd like to contribute to the Slate theme. Your help is essential for keeping it great. + +the Slate theme is an open source project supported by the efforts of an entire community and built one contribution at a time by users like you. We'd love for you to get involved. Whatever your level of skill or however much time you can give, your contribution is greatly appreciated. There are many ways to contribute, from writing tutorials or blog posts, improving the documentation, submitting bug reports and feature requests, helping other users by commenting on issues, or writing code which can be incorporated into the Slate theme itself. + +Following these guidelines helps to communicate that you respect the time of the developers managing and developing this open source project. In return, they should reciprocate that respect in addressing your issue, assessing changes, and helping you finalize your pull requests. + + +## Looking for support? + +We'd love to help. Check out [the support guidelines](SUPPORT.md). + +## How to report a bug + +Think you found a bug? Please check [the list of open issues](https://github.com/pages-themes/slate/issues) to see if your bug has already been reported. If it hasn't please [submit a new issue](https://github.com/pages-themes/slate/issues/new). + +Here are a few tips for writing *great* bug reports: + +* Describe the specific problem (e.g., "widget doesn't turn clockwise" versus "getting an error") +* Include the steps to reproduce the bug, what you expected to happen, and what happened instead +* Check that you are using the latest version of the project and its dependencies +* Include what version of the project your using, as well as any relevant dependencies +* Only include one bug per issue. If you have discovered two bugs, please file two issues +* Even if you don't know how to fix the bug, including a failing test may help others track it down + +**If you find a security vulnerability, do not open an issue. Please email security@github.com instead.** + +## How to suggest a feature or enhancement + +If you find yourself wishing for a feature that doesn't exist in the Slate theme, you are probably not alone. There are bound to be others out there with similar needs. Many of the features that the Slate theme has today have been added because our users saw the need. + +Feature requests are welcome. But take a moment to find out whether your idea fits with the scope and goals of the project. It's up to you to make a strong case to convince the project's developers of the merits of this feature. Please provide as much detail and context as possible, including describing the problem you're trying to solve. + +[Open an issue](https://github.com/pages-themes/slate/issues/new) which describes the feature you would like to see, why you want it, how it should work, etc. + + + +## Your first contribution + +We'd love for you to contribute to the project. Unsure where to begin contributing to the Slate theme? You can start by looking through these "good first issue" and "help wanted" issues: + +* [Good first issues](https://github.com/pages-themes/slate/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) - issues which should only require a few lines of code and a test or two +* [Help wanted issues](https://github.com/pages-themes/slate/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22) - issues which may be a bit more involved, but are specifically seeking community contributions + +*p.s. Feel free to ask for help; everyone is a beginner at first* :smiley_cat: + +## How to propose changes + +Here's a few general guidelines for proposing changes: + +* If you are making visual changes, include a screenshot of what the affected element looks like, both before and after. +* Follow the [Jekyll style guide](https://ben.balter.com/jekyll-style-guide). +* If you are changing any user-facing functionality, please be sure to update the documentation +* Each pull request should implement **one** feature or bug fix. If you want to add or fix more than one thing, submit more than one pull request +* Do not commit changes to files that are irrelevant to your feature or bug fix +* Don't bump the version number in your pull request (it will be bumped prior to release) +* Write [a good commit message](http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html) + +At a high level, [the process for proposing changes](https://guides.github.com/introduction/flow/) is: + +1. [Fork](https://github.com/pages-themes/slate/fork) and clone the project +2. Configure and install the dependencies: `script/bootstrap` +3. Make sure the tests pass on your machine: `script/cibuild` +4. Create a new branch: `git checkout -b my-branch-name` +5. Make your change, add tests, and make sure the tests still pass +6. Push to your fork and [submit a pull request](https://github.com/pages-themes/slate/compare) +7. Pat your self on the back and wait for your pull request to be reviewed and merged + +**Interesting in submitting your first Pull Request?** It's easy! You can learn how from this *free* series [How to Contribute to an Open Source Project on GitHub](https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github) + +## Bootstrapping your local development environment + +`script/bootstrap` + +## Running tests + +`script/cibuild` + +## Code of conduct + +This project is governed by [the Contributor Covenant Code of Conduct](CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. + +## Additional Resources + +* [Contributing to Open Source on GitHub](https://guides.github.com/activities/contributing-to-open-source/) +* [Using Pull Requests](https://help.github.com/articles/using-pull-requests/) +* [GitHub Help](https://help.github.com) diff --git a/docs/docs/SUPPORT.md b/docs/docs/SUPPORT.md new file mode 100644 index 0000000..5ec80b8 --- /dev/null +++ b/docs/docs/SUPPORT.md @@ -0,0 +1,9 @@ +## Where to get help + +If you think you've found a bug in the Slate theme, please [check the existing issues](https://github.com/pages-themes/slate/issues), and if no one has reported the problem, [open a new issue](https://github.com/pages-themes/slate/issues/new). + +If you have a general question about the theme, how to implement it, or how to customize it for your site you have two options: + +1. Search for your query on [`support.github.com`](https://support.github.com/?q=pages+Slate+theme), which will also look for similar topics on [`github.community`](https://github.community/search?q=pages+Slate+theme) +2. Ask your question of the Jekyll community on [talk.jekyllrb.com](https://talk.jekyllrb.com/) +3. [Contact GitHub Support](https://github.com/contact?form%5Bsubject%5D=GitHub%20Pages%20theme%20pages-themes/slate) diff --git a/docs/feature-gpg-and-ssl.md b/docs/feature-gpg-and-ssl.md new file mode 100644 index 0000000..602d43a --- /dev/null +++ b/docs/feature-gpg-and-ssl.md @@ -0,0 +1,49 @@ +--- +layout: default +title: GPG keys and SSL certificates +--- + +## GPG Keys + +When you want to verify GPG keys during installation in build root you can use something like in the config: +```python +config_opts['dnf.conf'] = """ +... SNIP +[appstream] +name=CentOS Stream $releasever - AppStream +metalink=https://mirrors.centos.org/metalink?repo=centos-appstream-$releasever-stream&arch=$basearch +gpgkey=file:///usr/share/distribution-gpg-keys/centos/RPM-GPG-KEY-CentOS-Official +gpgcheck=1 +... SNIP +""" +``` + +The path in `gpgkey` refers to the path in a buildchroot. How do you get your GPG key to buildchroot? There are several ways: + +### Distribution-GPG-Keys + +The package `distribution-gpg-key` is a requirement of Mock and is installed into buildchroot. The easiest way +is to add your package to [distribution-gpg-key project](https://github.com/xsuchy/distribution-gpg-keys/) and then +your key will be automatically present in both of host and buildroot. + +This is the preferred way for any new config added to `mock-core-configs`. + +### Local key + +Any file named `RPM-GPG-KEY-*` in the `/etc/pki/mock/` on the host is copied to buildchroot to the same path. + +You can use it for your personal GPG keys. + +## SSL certificates + + +Mock copy whole `/etc/pki/ca-trust/extracted` directory from the host to chroot. So the +chroot environment should recognize all SSL certificates your host knows. +If you need to add some specific certificate, you can add this part in your config: + + # Copy host's SSL certificate bundle ('/etc/pki/tls/certs/ca-bundle.crt') into + # specified location inside chroot. This usually isn't needed because we copy + # the whole /etc/pki/ca-trust/extracted directory recursively by default, and + # Fedora or EL systems work with that. But some destination chroots can have + # different configuration, and copying the bundle helps. + #config_opts['ssl_ca_bundle_path'] = None diff --git a/docs/feature-hermetic-builds.md b/docs/feature-hermetic-builds.md new file mode 100644 index 0000000..63a2383 --- /dev/null +++ b/docs/feature-hermetic-builds.md @@ -0,0 +1,211 @@ +--- +layout: default +title: Hermetic builds with Mock +--- + +Hermetic builds with Mock +========================= + +Mock (v5.7+) supports hermetic RPM builds, sometimes referred to as "isolated" +or "offline" builds. For more details, see the +[SLSA "hermetic" definition][SLSA future]. + +Quick start +----------- + +For the impatient, the TL;DR steps are as follows: + + # we want to build this package + srpm=your-package.src.rpm + + # we'll create a local repository with pre-fetched RPMs/bootstrap + repo=/tmp/local-repo + + # resolve build deps for the given SRPM, in this case for Fedora Rawhide + mock --calculate-build-dependencies -r fedora-rawhide-x86_64 "$srpm" + + # find the lockfile in Mock's resultdir + lockfile=/var/lib/mock/fedora-rawhide-x86_64/result/buildroot_lock.json + + # create a local RPM repository (+ download bootstrap image) + mock-hermetic-repo --lockfile "$lockfile" --output-repo "$repo" + + # perform the hermetic build! + mock --hermetic-build "$lockfile" "$repo" "$srpm" + +What an "hermetic build" is.. +----------------------------- + +The term "isolated build" is often used in different contexts within +Mock's terminology. Historically, when we said that "Mock isolates the build," +we typically meant that Mock creates a *buildroot* (also referred to as a *build +directory* or *build chroot*) and runs the (Turing-complete, and thus +potentially insecure) *RPM build* process (i.e., a call to `/usr/bin/rpmbuild`) +inside it. In this sense, Mock "isolates" the RPM build process from the rest +of the system, or protects the system from potential mishaps. However, the +**buildroot preparation** process was never "isolated" in this manner—only the +*RPM build* was. Also, the *RPM build* "isolation" was always performed on a +best-effort basis. For more details, see [Mock's Scope](index). + +This document focuses on making builds and their corresponding artifacts safer, +more predictable, and more reproducible. When we refer to *isolation*, we are +specifically referencing the [SLSA platform isolation][SLSA]. SLSA outlines +various security levels, and for the future, it introduces the concept of +[*hermetic builds*][SLSA future]. This is where Mock steps in, enabling builds +to be performed in a *hermetic* environment, free from unintended external +influences. + +Mock itself doesn't aim to provide this level of *isolation*. Mock is still +just a tool that runs in "some" build environment to perform the `SRPM → RPM` +translation. In such an environment, the Mock process can be tampered with by +other processes (potentially even root-owned), and as a result, the artifacts +may be (un)intentionally altered. Therefore, the preparation of the environment +to **run Mock** and the **isolation** itself is the responsibility of a +different tool (for example, `podman run --privileged --network=none`). + +So, what does Mock `--hermetic-build` do if it doesn't isolate? Essentially, it +just does less work than it usually does! It optimizes out any action +(primarily during the *buildroot* preparation) that would rely on "external" +factors—specifically, it never expects Internet connectivity. +However, for the eventual build to succeed, **something else** still needs to +perform these omitted actions. Every single component/artifact required for +*buildroot* preparation must be prepared in advance for the `mock +--hermetic-build` call (within the properly *isolated* or *hermetic* +environment, of course). + + +Challenges +---------- + +You've probably noticed that what used to be a simple command—like +`mock -r "$chroot" "$srpm"`—has now become a more complicated set of commands. +This complexity arises because the *buildroot* in Mock is always prepared by +installing a set of RPMs (Mock calls DNF, DNF calls RPM, ...), which normally +requires a network connection. + +Additionally, it’s not always guaranteed that the DNF/RPM variant on the build +host is sufficient or up-to-date for building the target distribution (e.g., +building the newest *Fedora Rawhide* packages on *EPEL 8* host). Therefore, we +need network access [to obtain the appropriate bootstrap +tooling](Feature-bootstrap). + +[Dynamic build dependencies][] add further complexity to the process. Without +them, we could potentially make the `/bin/rpmbuild` process fully offline—but +with their inclusion, it becomes much more challenging. Mock must interrupt the +ongoing *RPM build* process, resolve additional `%generate_buildrequires` +(installing more packages on demand), restart the *RPM build*, and potentially +repeat this cycle. This process also requires an (intermittent) network +connection! + +All of this is further complicated by the goal of making the *buildroot* as +*minimal* as possible—the fewer packages installed, the better. We can’t even +afford to install DNF into the buildroot, and as you've probably realized, we +definitely don’t want to blindly install all available RPMs. + + +The solution +------------ + +To address the challenges, we needed to separate the online +(`--calculate-build-dependencies`) and offline (`--hermetic-build`) tasks +that Mock performs. + +1. **Online Tasks:** These need to be executed first. We let Mock prepare the + *buildroot #1* for the given *SRPM* (using the standard "online" method) and + record its *lockfile*—a list of all the resources obtained from the network + during the process. + + The format of lockfile is defined by provided JSON Schema file(s), see + documentation for the [buildroot_lock plugin](Plugin-BuildrootLock). + + **Note:** The *buildroot* preparation includes the installation of dynamic + build dependencies! That's why we have to **initiate** `rpmbuild`. + But we don’t **finish** the build—we terminate it once the + `%generate_buildrequires` section is resolved, before reaching the `%build` + phase. + +2. **Offline Repository Creation:** With the *lockfile* from the previous step, + we can easily retrieve the referenced components from the network. The Mock + project provides an example implementation for this step in the + `mock-hermetic-repo(1)` utility. This tool downloads all the referenced + components from the internet and places them into a single local + directory—let's call it an *offline repository*. + + **Note:** This step doesn't necessarily have to be done by the Mock project + itself. The *lockfile* is concise enough for further processing and + validation (e.g., ensuring the set of RPMs and the buildroot image come from + trusted sources) and could be parsed by build-system-specific tools like + [cachi2][] (potentially in the future). + +3. **Offline Build:** With the *srpm* and the *offline repository*, we can + instruct Mock to restart the build using the `--hermetic-build + LOCKFILE OFFLINE_REPO SRPM` command. The *lockfile* is still needed at this + stage because it contains some of the configuration options used in step 1 + that must be inherited by the current Mock call. + + This step creates a new *buildroot #2* using the pre-downloaded RPMs in the + *offline repository* (installing them all at once) and then (re)starts the + RPM build process. This `rpmbuild` run **finishes** though, and provides the + binary RPM artifacts as usually. + +You might notice that some steps are performed twice, specifically downloading +the RPMs (steps 1 and 2) and running the RPM build (steps 1 and 3). This +duplication is a necessary cost (in terms of more resources and time spent on +the build) to ensure that step 3 is _fully offline_. In step 3, the *offline* +RPM build is no longer interrupted by an *online* `%generate_buildrequires` +process—dependencies are already installed! + +Also, while you can calmly experiment with + + + mock --calculate-build-dependencies -r fedora-rawhide-x86_64 "$srpm" + mock --no-clean -r fedora-rawhide-x86_64 "$srpm" + +This approach might seem similar to the TL;DR version, but it's not the same! +There is no *buildroot #1* and *buildroot #2*, only one buildroot. And that one +was prepared while Mock was online, meaning that something could **have +influenced** the environment preparation, and the subsequent **build**. + +Limitations +----------- + +- Let us stress out that this feature itself, while related or at least a bit + helpful for, doesn't provide reproducible builds. For reproducible builds, + build systems need to take in account state of host machine, the full + software/hardware stack. There's still a big influence of external factors! + +- We rely heavily on + the [Bootstrap Image feature](Feature-container-for-bootstrap). This allows + us to easily abstract the bootstrap preparation tasks, which would otherwise + depend heavily on the system's RPM/DNF stack, etc. + + For now, we also require the Bootstrap Image to be *ready*. This simplifies + the implementation, as we don't need to recall the set of commands (or list of + packages to install into) needed for bootstrap preparation. + +- It is known fact that *normal builds* and *hermetic builds* may result in + slightly different outputs (at least in theory). This issue relates to the + topic of *reproducible builds*. Normally, the *buildroot* is installed using + several DNF commands (RPM transactions), whereas the *hermetic build* installs + all dependencies in a single DNF command (single RPM transaction). While this + difference might cause the outputs of *normal* and *hermetic* builds to vary + (in theory, because the chroot shape depends on the complex RPM installation + order), the *hermetic* variant introduces more determinism! + +- The *lockfile* provides a list of the required RPMs, referenced by URLs. + These URLs point to the corresponding RPM repositories (online) from which + they were installed in step 1. However, in many cases, RPMs are downloaded + from `metalink://` or `mirrorlist://` repositories, meaning the URL might be + selected non-deterministically, and the specific mirrors chosen could be + rather ephemeral. For this reason, users should—for *hermetic* builds, for + now—avoid using mirrored repositories (and prefer Koji buildroots only) and + avoid making large delays between step 1 and step 2. Especially that, at the + time of writing this document, we know about [two][bug1] [bugs][bug2] that + will complicate the *lockfile* generation. + +[SLSA]: https://slsa.dev/spec/v1.0/requirements +[SLSA future]: https://slsa.dev/spec/v1.0/future-directions +[dynamic build dependencies]: https://github.com/rpm-software-management/mock/issues/1359 +[cachi2]: https://github.com/containerbuildsystem/cachi2 +[bug1]: https://github.com/rpm-software-management/dnf/issues/2130 +[bug2]: https://github.com/rpm-software-management/dnf5/issues/1673 diff --git a/docs/feature-isolated-builds.md b/docs/feature-isolated-builds.md new file mode 100644 index 0000000..ff4b0e3 --- /dev/null +++ b/docs/feature-isolated-builds.md @@ -0,0 +1,6 @@ +--- +layout: default +title: Isolated builds with Mock +--- + +This page been moved, see the [Hermetic Builds feature](feature-hermetic-builds). diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..7f647aa --- /dev/null +++ b/docs/index.md @@ -0,0 +1,373 @@ +--- +layout: title +--- + +# Mock + +Mock is a tool for building packages. It can build packages for different architectures and different [Fedora](https://getfedora.org/), [RHEL](https://www.redhat.com/en/technologies/linux-platforms/enterprise-linux), and [Mageia](https://www.mageia.org/) versions than the build host have. Mock creates chroots and builds packages in them. Its only task is to reliably populate a chroot and attempt to build a package in that chroot. + +``` +$ mock -r fedora-35-x86_64 package.src.rpm +... +Finish: rpmbuild packagei-1.98-1.fc35.src.rpm +Finish: build phase for package-1.98-1.fc35.src.rpm +INFO: Done(package.src.rpm) Config(fedora-35-x86_64) 2 minutes 14 seconds +INFO: Results and/or logs in: /var/lib/mock/fedora-35-x86_64/result +$ ls /var/lib/mock/fedora-35-x86_64/result +build.log package-1.98-1.fc35.noarch.rpm package-1.98-1.fc35.src.rpm hw_info.log installed_pkgs.log root.log state.log + +$ mock -r centos-stream+epel-9-s390x package.src.rpm +... +$ mock -r alma+epel-8-x86_64 package.src.rpm +... +``` + +Mock also offers a multi-package command (`--chain`), that can build chains of packages that depend on each other. + +Mock is capable of building SRPMs from source configuration management if the `mock-scm` package is present, then building the SRPM into RPMs. See `--scm-enable` in the documentation. + +## Scope + + * Mock is a [`rpmbuild(8)`](https://linux.die.net/man/8/rpmbuild) wrapper. Mock tries to simplify some steps, which would otherwise be boring or complicated. + * Mock runs `rpmbuild(8)` in an isolated environment consisting of a minimal set of packages. + * Mock helps you find missing [`BuildRequires`](https://rpm-packaging-guide.github.io/#what-is-a-spec-file) - if it is missing and the package needs it, then the build fails. + * Mock can build packages for various platforms and architectures. Some combinations of hosts and targets may need an additional configuration depending on your host platform. + * Mock can prepare a fresh build/development environment for specific RPM-based operating systems. + * Mock needs to execute some tasks under root privileges, therefore malicious RPMs can put your system at risk. Mock is not safe for unknown RPMs. If you want to build packages from untrusted sources, then use some wrapper around Mock like [OBS](https://openbuildservice.org/), [Copr](https://pagure.io/copr/copr/) or run Mock in a virtual machine. + * Mock is neither container nor VM. Mock does some isolation, but it does not aim to be fully isolated. + * Mock helps you open a shell within the buildroot to retrieve artifacts, or run commands for the purpose of debugging the build. It is not intended to run any production or developer application from there. For such purposes, you can use [podman](https://podman.io/) or [Flatpak](https://www.flatpak.org/). + +## Content + +* [Why Mock?](Why-Mock) +* [Status](#status) +* [Release Notes](#release-notes) +* [Tarballs](#tarballs) +* [Download](#download) +* [Setup](#setup) +* [Chroot configuration files](Mock-Core-Configs) +* [Plugins](#plugins) +* [Features](#features) +* [Using Mock outside your git sandbox](#using-mock-outside-your-git-sandbox) +* [Mock inside Podman, Fedora Toolbox or Docker container](#mock-inside-podman-fedora-toolbox-or-docker-container) +* [FAQ](#faq) +* [Exit codes](#exit-codes) +* [Problems](#problems) +* [Mock configuration](configuration) +* [See Also](#see-also) + +## Status + +Mock is currently being used for all Fedora builds. It is called by [Koji](https://fedoraproject.org/wiki/Koji) and [Copr](https://copr.fedorainfracloud.org) to build chroots and packages. + +Versions in Linux distributions: + +
+mock versions + +mock-core-configs versions +
+ + +## Release Notes +* [Configs 43.1](Release-Notes-Configs-43.1) (2025-08-12) - Fedora 43 branched. +* [6.3 and Configs 42.4](Release-Notes-6.3) (2025-06-18) - Compatibility bug-fix for Python 3.14, configuration is owned by root group, AlmaLinux and Rocky Linux 10 added. +* [6.2 and Configs 42.3](Release-Notes-6.2) (2025-05-22) - RHEL 10 configuration, configurable ca-trust paths that Mock copies to chroots. +* [6.1 and Configs 42.2](Release-Notes-6.1) (2025-02-27) - more hermetic-build improvements, AlmaLinux Kitten 10, Azure Linux 2.0 and 3.0, Kylin 10, Navy Linux 8, openSUSE Leap 15.6 +* [Configs 42.1](Release-Notes-Configs-42.1) (2025-01-16) - Fedora 42 branched. +* [6.0 and Configs 41.5](Release-Notes-6.0) (2024-12-19) - New OCI image support for buildroot, hermetic build mode fixes, Fedora 39 EOL. +* [5.9 and Configs 41.4](Release-Notes-5.9) (2024-09-30) - A --no-bootstrap-chroot regression with DNF4 chroots fixed. +* [5.8](Release-Notes-5.8) (2024-09-27) - Bug-fixed regression in `chroot_scan` plugin. +* [5.7 and Configs 41.3](Release-Notes-5.7) (2024-09-26) - Support for isolated builds added. DNF4 = DNF. Option `--scrub-all-chroots` added. +* [Configs 41.1 (and 41.2)](Release-Notes-Configs-41.1) (2024-08-15) - EL7 configs EOL. F38 EOL. F41 branched. +* [Configs 40.6](Release-Notes-Configs-40.6) (2024-06-15) - CentOS Stream 10 uses mirrored repositories. +* [Configs 40.5](Release-Notes-Configs-40.5) (2024-06-05) - Fedora 38 moved to EOL. CentOS Stream 8 moved to vault. +* [5.6](Release-Notes-5.6) (2024-05-15) - Improved performance of bash completion, don't use `--allowerasing` for commands that doesn't provide it, fixed "no space left" tracebacks, new Circle Linux 9 configs, Mageia Cauldron i686, fixed Fedora ELN. +* [Configs 40.3](Release-Notes-Configs-40.3) (2024-04-09) - Added C10s chroots, Dropped Fedora modular repositories, fix bootstrap from image for openSUSE +* [Configs 40.2](Release-Notes-Configs-40.2) (2024-02-16) - Fixed Fedora 40 builds that regressed back to `dnf` (instead of expected `dnf5`). +* [5.5](Release-Notes-5.5) (2024-02-14) - The `{{ repo_arch }}` support added, chroot_scan supports tarballs, ownership during `--init` fixed, fixed `root_cache` tarball invalidation problem. +* [5.4](Release-Notes-5.4) (2024-01-04) - Bugfix the rpmautospec plugin. +* [5.3](Release-Notes-5.3) (2023-12-13) - New "rpmautospec" plugin added, `%generate_buildrequries` fixes landed. +* [Configs 39.3](Release-Notes-Configs-39.3) - Fedora 40+ configuration uses DNF5, Fedora ELN and OpenMandriva fixes. +* [Configs 39.2](Release-Notes-39.2) - Fedora ELN and openSUSE fixes. +* [5.2](Release-Notes-5.2) (2023-09-27) - Compatibility fix with EPEL 8, logging fixes, `--copyout` files with tilde in name. +* [5.1.1](Release-Notes-5.1.1) (2023-09-18) - If Mock does multiple builds at once, root directory is re-created for each of them. +* [5.1](Release-Notes-5.1) (2023-09-15) - Fixes for `--use-bootstrap-image`, it now retries pulling, and fallbacks to a normal bootstrap. +* [5.0](Release-Notes-5.0) (2023-08-09) - The `--use-bootstrap-image` feature enabled by default, using `/sbin/useradd` from host (not in chroot) and configurable. +* [4.1](Release-Notes-4.1) (2023-06-05) - Bug-fix v4.0 for bootstrap with custom SSL certificates, bug-fix 4.0 the --dnf-cmd option. Newly we use /bin/dnf-3 if `package_manager=dnf`, and dnf5 is used to install bootrap (if found on host). +* [4.0](Release-Notes-4.0) (2023-05-22) - Support for DNF5 added, the '--use-bootstrap-image' feature now works even if Mock is run in container. +* [3.5](Release-Notes-3.5) (2022-12-01) - Fixed detection of qemu-user-static* packages for the `--forcearch` feature. +* [3.4](Release-Notes-3.4) (2022-11-15) - Device Mapper control file exposed, better detection for qemu-user-static. +* [3.3](Release-Notes-3.3) (2022-10-17) - Mock can again be run by `root`, even though this is discouraged. +* [3.2](Release-Notes-3.2) (2022-10-14) - Optimized --list-chroots option, directories in `/var/lib` dropped SGID bit, `rpmbuild --noclean` is not used for old chroots (EL6 and older). +* [3.1](Release-Notes-3.1) (2022-07-22) - Fixes for new RPM with --no-cleanup-after, a new config `tar_binary` added, more convenient work with `/bin/bsdtar`. +* [3.0](Release-Notes-3.0) (2022-04-07) - Added --list-chroots command, a new seccomp option, Mock is not installable on EL7, dropped Python 2 compatibility. +* [2.16](Release-Notes-2.16) (2021-12-16) - EPEL 8 chroots removed, alternatives added. New `ssl_extra_certs` option. Bugfixes. +* [2.15](Release-Notes-2.15) (2021-11-18) - Fix for old-style `mock shell -- commands` variant. +* [2.14](Release-Notes-2.14) (2021-11-04) - Fix for broken `--enablerepo` and `--disablerepo` options. +* [2.13](Release-Notes-2.13) (2021-11-02) - New options `--additional-package` and `--debug-config-expanded`. Bugfixing. +* [2.12](Release-Notes-2.12) (2021-07-19) - bugfixes in Mock, but new config files in mock-core-configs +* [2.11](Release-Notes-2.11) (2021-06-09) - introduced `%{_platform_multiplier}` macro +* [2.10](Release-Notes-2.10) (2021-04-27) - smaller bugfixes +* [2.9](Release-Notes-2.9) (2021-01-18) - bugfixes, EOLed Fedora 31 and EPEL 6 chroots +* [2.8](Release-Notes-2.8) (2020-12-15) - bugfix in --isolation=nspawn, --isolation=simple was used instead +* [2.7](Release-Notes-2.7) (2020-12-01) - external build requires implemented, new rpkg_preprocessor plugin, bugfixes +* [2.6](Release-Notes-2.6) (2020-09-15) - bugfixing --chain mode and --isolation=nspawn +* [2.5](Release-Notes-2.5) (2020-09-03) - setting DNFs user_agent, a new showrc plugin added, new mock-filesystem package introduced +* [2.4](Release-Notes-2.4) (2020-07-21) - exposed btrfs-control, `module_setup_commands` configuration option, copy source CA certificates +* [2.3](Release-Notes-2.3) (2020-05-22) - bugfixes, mostly related to (by default on) bootstrap +* [2.2](Release-Notes-2.2) (2020-04-02) - bugfixing, mostly --bootstrap-chroot issues and mock-in-container use-cases +* [2.1](Release-Notes-2.1) (2020-03-11) - bugfixing +* [2.0](Release-Notes-2.0) (2020-02-07) - new major version, default --bootstrap-chroot +* [1.4.21](Release-Notes-1.4.21) (2019-11-01) - bugfixing +* [1.4.20](Release-Notes-1.4.20) (2019-10-04) - Container image for bootstrap, Mockchain removed, New config option package_manager_max_attempts, Bind mount local repos to bootstrap chroot +* [1.4.19](Release-Notes-1.4.19) (2019-09-10) - bugfixing +* [1.4.18](Release-Notes-1.4.18) (2019-08-27) - subscription-manager support; procenv plugin; automatic forcearch; signals propagated in chroot +* [1.4.17](Release-Notes-1.4.17) (2019-08-08) - Toolbox support, OpenMandriva, `mock --chain`, Dynamic Build Requires enabled by default +* [1.4.16](Release-Notes-1.4.16) (2019-05-22) - python3 on el7 +* [1.4.15](Release-Notes-1.4.15) (2019-04-22) - Dynamic Build Requires; configurable list of disabled plugins; nice error for people not in mock group +* [1.4.14](Release-Notes-1.4.14) (2019-02-19) - Jinja2 templates; choose decompress program for root_cache +* [1.4.13](Release-Notes-1.4.13) (2018-08-13) - rawhide is gpg checked; new option `print_main_output`; proxy environmnet variables passed to mock; improved bash completation +* [1.4.11](Release-Notes-1.4.11) (2018-06-12) - new options `--force-arch`, `--spec`, `chrootuser`; MicroDNF support; BSDTar support +* [1.4.10](Release-Notes-1.4.10) (2018-05-10) - new overlayfs plugin; bind_mount can mount even files; chroot_scan can retrieve artifacts even from failed builds; introduced symlinks to rawhide configs +* [1.4.9](Release-Notes-1.4.9) (2018-02-12) - split of stdout and stderr; new option `optstimeout` +* [1.4.8](Release-Notes-1.4.8) (2017-12-22) - new option `--config-opts` +* [1.4.7](Release-Notes-1.4.7) (2017-10-31) - new option `chrootgroup`; config options for bootstrap; recognize DeskOS; handle network namespace in systemd container on our own +* [1.4.6](Release-Notes-1.4.6) (2017-09-15) - separation of mock-core-configs; new command `--debug-config`; short option `-N` for `--no-cleanup-after` +* [1.4.4](Release-Notes-1.4.4) (2017-08-22) - rename group inside of chroot from mockbuild to mock +* [1.4.3](Release-Notes-1.4.3) (2017-08-7) +* [1.4.2](Release-Notes-1.4.2) (2017-06-20) +* [1.4.1](Release-Notes-1.4.1) (2017-04-26) - introduced systemd-nspawn and `--old-chroot`option; new option `/dev/hwrng`, `/dev/prandom`; added `%distro_section` for Mageia; bugfixing +* [1.3.5](Release-Notes-1.3.5) (2017-03-02)- only for EL6; change path to the “df” in hw-info plugin +* [1.3.4](Release-Notes-1.3.4) (2017-02-27) - `.log` extension for the `available_pkgs` and `installed_pkgs` log files, support for custom nspawn args, new hw_info plugin, best=1 used for Rawhide, added Fedora 26 configs +* [1.3.3](Release-Notes-1.3.3) (2017-01-01) - bugfixing; a new config option for the builder hostname; upgraded temporary directories, chroot contains `best=1` +* [1.3.2](Release-Notes-1.3.2) (2016-10-17) - move /usr/sbin/mock/ to /usr/libexec/mock/mock; script in /usr/libexec/ are not in $PATH; F22 configs have been removed; --nocheck works; run mock in Docker; a lot of flake8/pep8/pycodestyle clean-ups +* [1.2.21](Release-Notes-1.2.21) (2016-09-12) - fix privilege escalation via mock-scm; rename of mageia pubkey +* [1.2.20](Release-Notes-1.2.20) (2016-08-17) - just a bugfix release, which uses correct gpg keys for epel in epel* configs. +* [1.2.19](Release-Notes-1.2.19) +* [1.2.18](Release-Notes-1.2.18) (2016-06-10) - Unconditional setup resolver config, added MPS personalities, requires rpm=pyton, use root name instead config name for backups dir, use DNF for F24 chroot, scm plugging handles better submodes and improve prompt +* [1.2.17](Release-Notes-1.2.17) +* [1.2.16](Release-Notes-1.2.16) +* [1.2.15](Release-Notes-1.2.15) +* [1.2.14](Release-Notes-1.2.14) +* [1.2.13](Release-Notes-1.2.13) (2016-08-17) + +### Tarballs + +Tarballs can be found at https://github.com/rpm-software-management/mock/releases + +You can retrieve tarball from the command line: + +``` +git checkout --hard mock-1.4.20-1 +cd mock +tito build --tgz +``` + +## Download + +If you want to contribute to the code, please checkout https://github.com/rpm-software-management/mock for more information. + +Otherwise, just run + + dnf install mock + +For nightly builds, please refer to [developer documentation](https://github.com/rpm-software-management/mock#nightly) + + +## Setup + +All users that are to use mock must be added to the *mock* group. + + usermod -a -G mock [User name] + +:warning: _Mock runs some parts of its code with root privileges. There are known ways to get root access once a user is in the mock group (and once he is able to run mock). This is possible when a user abuses the mock configuration options. Please do not add anyone who is not trustworthy to the mock group!_ + +:notebook: To have this change take effect you have to either log out and log back in or run command `newgrp -` + +Mock caches the downloaded rpm packages (via the `yum_cache` plugin), which +speeds up subsequent builds by a considerable margin. Nevertheless, you may +wish to [change the default configuration](configuration) to point to local +repositories to speed up builds. + +By default, builds are done in `/var/lib/mock`, so be sure you have room there. You can change this via the `basedir` config option. + +## Chroot config files + +See a [separate document](Mock-Core-Configs). + +## Plugins + +* [bind_mount](Plugin-BindMount) - bind mountpoints inside the chroot +* [buildroot_lock](Plugin-BuildrootLock) - provide a buildroot lockfile +* [ccache](Plugin-CCache) - compiler cache plugin +* [chroot_scan](Plugin-ChrootScan) - allows you to retrieve build artifacts from buildroot (e.g. additional logs, coredumps) +* [compress_logs](Plugin-CompressLogs) - compress logs +* [export_buildroot_image](Plugin-Export-Buildroot-Image) - export prepared build chroot as an OCI image +* [hw_info](Plugin-HwInfo) - prints HW information of builder +* [lvm_root](Plugin-LvmRoot) - caching buildroots using LVM +* [mount](Plugin-Mount) - allows you to mount directories into chroot +* [overlayfs](Plugin-Overlayfs) - plugin implementing snapshot functionality (similary to lvm_root) +* [package_state](Plugin-PackageState) - dumps list of available and installed packages +* [pm_request](Plugin-PMRequest) - executes package manager commands requested by processes running in the chroot +* [procenv](Plugin-ProcEnv) - dumps the build process runtime within the chroot. +* [rpkg_preprocessor](Plugin-rpkg-preprocessor) - preprocess the input spec file just before srpm build starts +* [root_cache](Plugin-RootCache) - cache buildroots (as tar file) +* [scm](Plugin-Scm) - SCM integration module - builds directly from Git or Svn +* [selinux](Plugin-SELinux) - on SELinux enabled box, this plugin will pretend, that SELinux is disabled in build environment +* [showrc](Plugin-Showrc) - Log the content of `rpm --showrc` for capturing all defined macros +* [sign](Plugin-Sign) - call command on the produced rpm +* [tmpfs](Plugin-Tmpfs) - mount buildroot directory as tmpfs +* [yum_cache](Plugin-YumCache) - mount `/var/cache/{dnf,yum}` of your host machine to chroot + +Plugins can be enabled on command line e.g `--enable-plugin=chroot_scan`. And you can set plugin options using e.g. `'--plugin-option=root_cache:age_check=False'` + +Every plugin has a corresponding wiki page with docs. + +[Order of plugins hooks](Plugin-Hooks). + +## Features + +* [bootstrap](Feature-bootstrap) - bootstrapping chroot. I.e., when building F28 on RHEL7, then first install very minimal bootstrap chroot with DNF and rpm from F28 and then use F28's rpm to install final F28 chroot. +* [container image for bootstrap](Feature-container-for-bootstrap) - set up bootstrap chroot using Podman. +* [container image for buildroot](Feature-buildroot-image) - pre-create buildroot from an OCI image +* [external dependencies](Feature-external-deps) - use of external dependencies, e.g., `BuildRequires external:pypi:foo`. +* [forcearch](Feature-forcearch) - build for foreign architecture using emulated virtualization. +* [nosync](Feature-nosync) - speed up build by making `fsync`(2) no-op. +* [modularity](Feature-modularity) - support for Fedora Modularity. +* [package managers](Feature-package-managers) - supported package managers +* [rhel chroots](Feature-rhelchroots) - builds for RHEL +* [GPG keys and SSL](feature-gpg-and-ssl) - how to get your GPG keys and SSL certificates to buildroot +* [Hermetic (offline) Builds](feature-hermetic-builds) - doing hermetic builds with Mock + +## Using Mock outside your git sandbox + +Create your SRPM using `rpmbuild -bs`. Then change to the directory where your SRPM was created. + +Now you can start mock with +``` +mock -r --rebuild package-1.2-3.src.rpm +``` + +where `` is the name of a configuration file from `/etc/mock/`, without the `/etc/mock` path prefix and without the `.cfg` suffix. + +Note that you can track the progress of mock using the logs stored in `/var/lib/mock//result` + +## Mock inside Podman, Fedora Toolbox or Docker container + +First, we need to state that Mock, in a nutshell, is a tool that (a) prepares an +appropriate RPM build environment (aka "build chroot" or "buildroot"), and then +it (b) just executes the RPM build inside (`rpmbuild` run). + +The build environment is (again a bit simplified) defined by a set of RPM +packages that need to be installed in such environment (build dependencies, or +also `BuildRequires:` in RPM spec files). + +Mock is a generic tool to build **any** RPM out there. And each RPM has a +different set of requirements (so we can not just pre-generate one environment +for all packages and share it). Here comes the important implication: If you +want to **build an RPM**, you need to install **other RPMs**, and for that, +you need to have **root access**. + +Normally, Mock uses `dnf --installroot /some/directory install ...` (on host) +to prepare the environment. Then it switches into the environment using the +[systemd-nspawn](https://www.freedesktop.org/software/systemd/man/systemd-nspawn.html) +container (default `--isolation=nspawn`, but you can fallback to `--isolation=simple` +which is just `man (2) chroot`). Again [simplified a bit](Feature-bootstrap). + +The build itself (`rpmbuild` process) is a non-root operation, Mock +intentionally drops the privileges there. + +All that said, using Mock inside a container to build RPMs is totally possible! +You just need to have permissions to install RPMs, and be able to switch UID +when needed (from `root` to non-privileged and back, `man (2) seteuid`). As a +benefit, we don't need to run `systemd-nspawn` and still have even better +isolation because now even the `dnf --installroot` is executed inside the +container. This statement assumes the container is dedicated to the Mock build +and no other task(s) that could be compromised by a rogue build (even subsequent +builds!). So ideally, considering how easy is to start new containers from +images, each build should have its own dedicated container (especially if you +are a generic build system where you can not fully trust all your users, or even +the packages that your users with the best intentions build or install). Then +the build can only affect the container, not the whole host. + +So, Mock can be run in a rootless Podman container (with [user +namespaces](https://man7.org/linux/man-pages/man7/user_namespaces.7.html)) +without any special tweaks. The only necessary step is to run the container +with `--privileged` option. Read the podman-run manual page for more info, but +`--privileged` - by the Podman nature - can not give the process more +permissions than the UID running the podman process already has; in other words +- `podman run --privileged` is a completely different thing from `docker run +--privileged`! + +So simply, as any **non-privileged system user**, do: + +``` +$ podman run --rm --privileged -ti fedora:32 bash +# dnf install -y mock +# useradd mockbuilder +# usermod -a -G mock mockbuilder +# su - mockbuilder +$ mock https://some/online.src.rpm +$ mock --shell +#> ... +# etc +``` + +And similarly in `toolbox enter`. + +But running Mock in an OpenShift POD isn't [typically +allowed](https://access.redhat.com/solutions/6375251). +Cluster admin typically keeps `SETUID` and `SETGID` [capabilities +dropped](https://docs.openshift.com/container-platform/4.12/authentication/managing-security-context-constraints.html), +`allowPrivilegeEscalation` disabled (no root access). User namespaces are +[not yet available](https://access.redhat.com/solutions/6977863). Per previous +implications, with the default security configuration, you can not install RPMs +in OpenShift POD containers and thus neither build RPMs. + +You can run Mock in Docker container, however, you need to add `SYS_ADMIN` +capability to the docker container (or use `--privileged`). I.e. run the +container like: + +``` +docker run --cap-add=SYS_ADMIN ... +``` + +:warning: Please note that Mock run inside of Docker container skips unsharing +of a namespace, so it runs in the same namespace as any other program in the +same container. That means you should not run any other application inside of +that container. Mock prints warning about this. You can suppress this warning +when you put in the config + +``` +config_opts['docker_unshare_warning'] = False +``` + +## FAQ + +See separate page: [FAQ](FAQ) + +## Exit codes + +Mock has various exit codes to signal a problem in the build. See https://github.com/rpm-software-management/mock/blob/master/mock/py/mockbuild/exception.py#L26 + +## Problems + +[List of known issues](https://bugzilla.redhat.com/buglist.cgi?bug_status=NEW&bug_status=ASSIGNED&component=mock&known_name=mock-all&list_id=6164173&product=Fedora&product=Fedora%20EPEL&query_based_on=mock-all&query_format=advanced) + +If you encounter a bug running mock, please file it in [Bugzilla](https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora&component=mock): product "Fedora", component mock ([Open Bugs](https://bugzilla.redhat.com/buglist.cgi?query_format=advanced&product=Fedora&component=mock&bug_status=NEW&bug_status=ASSIGNED&bug_status=MODIFIED&bug_status=ON_DEV&bug_status=ON_QA&bug_status=VERIFIED&bug_status=FAILS_QA&bug_status=RELEASE_PENDING&bug_status=POST)). + +If your problem is specific to EPEL, then [file it](https://bugzilla.redhat.com/enter_bug.cgi?product=Fedora%20EPEL&component=mock) against the "Fedora EPEL" product instead ([Open Bugs](https://bugzilla.redhat.com/buglist.cgi?query_format=advanced&product=Fedora%20EPEL&component=mock&bug_status=NEW&bug_status=ASSIGNED&bug_status=MODIFIED&bug_status=ON_DEV&bug_status=ON_QA&bug_status=VERIFIED&bug_status=FAILS_QA&bug_status=RELEASE_PENDING&bug_status=POST)). + +## Users + +If you use Mock, we'd love to hear from you and add you to this [wiki page](https://github.com/rpm-software-management/mock/wiki/Users). It will motivate our future work. + +## See Also + +* [Using Mock to test package builds](https://fedoraproject.org/wiki/Using_Mock_to_test_package_builds) has some useful tips for using mock. +* [Mock Setup Using Local Mirror](https://fedoraproject.org/wiki/Docs/Drafts/MockSetupUsingLocalMirror) Setting up a local mirror using Mock. +* [Legacy/Mock](https://fedoraproject.org/wiki/Archive:Legacy/Mock?rd=Legacy/Mock) has some useful tips for building packages in mock for older Fedora and Red Hat Linux releases. +* [Increase Mock performance](http://miroslav.suchy.cz/blog/archives/2015/05/28/increase_mock_performance_-_build_packages_in_memory/index.html). +* [RPM Packaging Guide](https://rpm-packaging-guide.github.io/) +* [Modularity Features in Mock](http://frostyx.cz/posts/modularity-features-in-mock) diff --git a/docs/jekyll-theme-slate.gemspec b/docs/jekyll-theme-slate.gemspec new file mode 100644 index 0000000..1ecb0ed --- /dev/null +++ b/docs/jekyll-theme-slate.gemspec @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +Gem::Specification.new do |s| + s.name = "jekyll-theme-slate" + s.version = "0.2.0" + s.license = "CC0-1.0" + s.authors = ["Jason Costello", "GitHub, Inc."] + s.email = ["opensource+jekyll-theme-slate@github.com"] + s.homepage = "https://github.com/pages-themes/slate" + s.summary = "Slate is a Jekyll theme for GitHub Pages" + + s.files = `git ls-files -z`.split("\x0").select do |f| + f.match(%r{^((_includes|_layouts|_sass|assets)/|(LICENSE|README)((\.(txt|md|markdown)|$)))}i) + end + + s.required_ruby_version = ">= 2.4.0" + + s.platform = Gem::Platform::RUBY + s.add_runtime_dependency "jekyll", "> 3.5", "< 5.0" + s.add_runtime_dependency "jekyll-seo-tag", "~> 2.0" + s.add_development_dependency "html-proofer", "~> 3.0" + s.add_development_dependency "rubocop-github", "~> 0.16" + s.add_development_dependency "w3c_validators", "~> 1.3" +end diff --git a/docs/script/bootstrap b/docs/script/bootstrap new file mode 100755 index 0000000..492e553 --- /dev/null +++ b/docs/script/bootstrap @@ -0,0 +1,6 @@ +#!/bin/sh + +set -e + +gem install bundler +bundle install diff --git a/docs/script/cibuild b/docs/script/cibuild new file mode 100755 index 0000000..165dd48 --- /dev/null +++ b/docs/script/cibuild @@ -0,0 +1,9 @@ +#!/bin/sh + +set -e + +bundle exec jekyll build +bundle exec htmlproofer ./_site --check-html --check-sri +bundle exec rubocop -D --config .rubocop.yml +bundle exec script/validate-html +gem build jekyll-theme-slate.gemspec diff --git a/docs/script/release b/docs/script/release new file mode 100755 index 0000000..a4a7332 --- /dev/null +++ b/docs/script/release @@ -0,0 +1,42 @@ +#!/bin/sh +# Tag and push a release. + +set -e + +# Make sure we're in the project root. + +cd $(dirname "$0")/.. + +# Make sure the darn thing works + +bundle update + +# Build a new gem archive. + +rm -rf jekyll-theme-slate-*.gem +gem build -q jekyll-theme-slate.gemspec + +# Make sure we're on the master branch. + +(git branch | grep -q 'master') || { + echo "Only release from the master branch." + exit 1 +} + +# Figure out what version we're releasing. + +tag=v`ls jekyll-theme-slate-*.gem | sed 's/^jekyll-theme-slate-\(.*\)\.gem$/\1/'` + +# Make sure we haven't released this version before. + +git fetch -t origin + +(git tag -l | grep -q "$tag") && { + echo "Whoops, there's already a '${tag}' tag." + exit 1 +} + +# Tag it and bag it. + +gem push jekyll-theme-slate-*.gem && git tag "$tag" && + git push origin master && git push origin "$tag" diff --git a/docs/script/validate-html b/docs/script/validate-html new file mode 100755 index 0000000..43d8fca --- /dev/null +++ b/docs/script/validate-html @@ -0,0 +1,28 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require "w3c_validators" + +def validator(file) + extension = File.extname(file) + if extension == ".html" + W3CValidators::NuValidator.new + elsif extension == ".css" + W3CValidators::CSSValidator.new + end +end + +def validate(file) + puts "Checking #{file}..." + + path = File.expand_path "../_site/#{file}", __dir__ + results = validator(file).validate_file(path) + + return puts "Valid!" if results.errors.empty? + + results.errors.each { |err| puts err.to_s } + exit 1 +end + +validate "index.html" +validate File.join "assets", "css", "style.css" diff --git a/docs/thumbnail.png b/docs/thumbnail.png new file mode 100644 index 0000000..cb179a4 Binary files /dev/null and b/docs/thumbnail.png differ diff --git a/mock/.gitignore b/mock/.gitignore new file mode 100644 index 0000000..6350e98 --- /dev/null +++ b/mock/.gitignore @@ -0,0 +1 @@ +.coverage diff --git a/mock/COPYING b/mock/COPYING new file mode 100644 index 0000000..623b625 --- /dev/null +++ b/mock/COPYING @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/mock/Makefile b/mock/Makefile new file mode 100644 index 0000000..209074c --- /dev/null +++ b/mock/Makefile @@ -0,0 +1,45 @@ +# vim:noexpandtab:autoindent:tabstop=8:shiftwidth=8:filetype=make:nocindent:tw=0: +check: + ./integration-tests/runtests.sh + +AUTHORS: + (GIT_DIR=.git git log | grep ^Author | sort |uniq > .authors.tmp && mv .authors.tmp AUTHORS; rm -f .authors.tmp) || (touch AUTHORS; echo 'git directory not found: installing possibly empty AUTHORS.' >&2) + +install-exec-hook: + for i in $(REPLACE_VARS_ON_INSTALL); do \ + file=$(DESTDIR)/$$i ;\ + perl -p -i -e 's|^__VERSION__\s*=.*|__VERSION__="$(RELEASE_VERSION)"|' $$file ;\ + perl -p -i -e 's|^SYSCONFDIR\s*=.*|SYSCONFDIR="$(sysconfdir)"|' $$file ;\ + perl -p -i -e 's|^PYTHONDIR\s*=.*|PYTHONDIR="$(pythondir)"|' $$file ;\ + perl -p -i -e 's|^PKGPYTHONDIR\s*=.*|PKGPYTHONDIR="$(mockbuilddir)"|' $$file ;\ + perl -p -i -e 's|^PKGDATADIR\s*=.*|PKGDATADIR="$(pkgdatadir)"|' $$file ;\ + perl -p -i -e 's|^LIBDIR\s*=.*|LIBDIR="$(libdir)"|' $$file ;\ + done + mv $(DESTDIR)/$(sbindir)/mock.py $(DESTDIR)/$(sbindir)/mock + [ -d $(DESTDIR)/$(bindir) ] || mkdir $(DESTDIR)/$(bindir) + mv $(DESTDIR)/$(sbindir)/mockchain.py $(DESTDIR)/$(bindir)/mockchain + + +.PHONY: rpm srpm help install-devel-packages +rpm: + tito build --test --rpm + +srpm: + tito build --test --srpm + +install-devel-packages: + yum --disablerepo='beaker*' install mock rpm-build fedora-packager vim-enhanced git-all + +help: + @echo + @echo "Mock Makefile targets:" + @echo " dist - generate Authors file" + @echo " rpm - build binary RPM" + @echo " srpm - build source RPM" + @echo " help - print this message" + @echo "Additionally, all standard automake targets are supported" + @echo + +# Tell versions [3.59,3.63) of GNU make to not export all variables. +# Otherwise a system limit (for SysV at least) may be exceeded. +.NOEXPORT: diff --git a/mock/README.md b/mock/README.md new file mode 100644 index 0000000..4c2e54a --- /dev/null +++ b/mock/README.md @@ -0,0 +1,215 @@ +# deb-mock + +## Overview + +**deb-mock** is Debian's equivalent to Fedora's Mock - a chroot build environment manager that provides isolated, reproducible build environments for Debian packages. It's designed to work seamlessly with `deb-bootc-compose` and `deb-orchestrator` to create Debian's complete bootc ecosystem. + +## What deb-mock Does + +### **Core Purpose** +deb-mock is fundamentally a **chroot environment manager** - it doesn't build packages directly, but rather creates and manages isolated build environments where packages can be built safely. Think of it as the "sandbox creator" for Debian package building, providing clean, reproducible environments for each build. + +### **Primary Functions** + +#### **1. Build Environment Management** +- **Chroot Creation**: Creates isolated chroot environments for builds +- **Buildroot Management**: Manages buildroot lifecycle (create, populate, cleanup) +- **Package Installation**: Installs build dependencies in the chroot +- **Environment Isolation**: Ensures builds don't interfere with host system + +#### **2. Build Process Orchestration** +- **Build Execution**: Runs build commands within the chroot +- **Dependency Resolution**: Manages package dependencies for builds +- **Result Collection**: Collects build artifacts from the chroot +- **Cleanup Management**: Handles post-build cleanup and caching + +#### **3. Security & Isolation** +- **User Management**: Manages UIDs/GIDs within the chroot +- **Mount Management**: Controls filesystem access and mounts +- **Network Isolation**: Provides network isolation during builds +- **Resource Control**: Limits resource usage within chroots + +## Architecture + +### **Single-Process Architecture** + +deb-mock uses a **single-process, multi-stage architecture** with clear separation of concerns: + +``` +┌─────────────────────────────────────────────────────────────┐ +│ deb-mock Process │ +├─────────────────────────────────────────────────────────────┤ +│ Configuration │ Buildroot │ Package │ Command │ +│ Manager │ Manager │ Manager │ Executor │ +├─────────────────────────────────────────────────────────────┤ +│ Plugin System │ Mount │ UID │ State │ +│ │ Manager │ Manager │ Manager │ +└─────────────────────────────────────────────────────────────┘ + │ + ▼ + ┌─────────────────┐ + │ Chroot │ + │ Environment │ + └─────────────────┘ +``` + +### **Core Components** + +#### **1. Main Entry Point (`deb_mock/core.py`)** +The command-line interface that orchestrates all operations: +- **Command Parsing**: Handles user commands and arguments +- **Configuration Loading**: Loads and validates configuration +- **Buildroot Management**: Creates and manages build environments +- **Command Execution**: Executes build commands in chroots + +#### **2. Buildroot Manager (`deb_mock/chroot.py`)** +The core component that manages chroot environments: +- **Chroot Lifecycle**: Creates, populates, and destroys chroots +- **Package Management**: Installs and manages packages in chroots +- **Plugin Integration**: Coordinates plugin execution +- **State Management**: Tracks buildroot state and progress + +#### **3. Package Manager (`deb_mock/package_manager.py`)** +Handles package installation and dependency resolution: +- **Package Installation**: Installs packages in chroots +- **Dependency Resolution**: Resolves package dependencies +- **Repository Management**: Manages package repositories +- **Fallback Support**: Provides fallback to alternative package managers + +## Integration with deb-bootc-compose and deb-orchestrator + +### **The deb-ecosystem Workflow** + +``` +┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ +│ deb-bootc- │ │ deb-orchestrator│ │ deb-mock │ +│ compose │ │ Build System │ │ Build Environment│ +└─────────────────┘ └─────────────────┘ └─────────────────┘ + │ │ │ + │ 1. "I need packages" │ │ + │──────────────────────▶│ │ + │ │ │ + │ │ 2. "Build this .deb" │ + │ │──────────────────────▶│ + │ │ │ + │ │ 3. "Environment ready"│ + │ │◀──────────────────────│ + │ │ │ + │ │ 4. "Build complete" │ + │ │◀──────────────────────│ + │ │ │ + │ 5. "Packages ready" │ │ + │◀──────────────────────│ │ +``` + +### **How deb-mock Fits** + +1. **deb-orchestrator** requests a build environment for package building +2. **deb-mock** creates an isolated chroot environment +3. **deb-mock** installs build dependencies in the chroot +4. **deb-orchestrator** executes build commands in the deb-mock environment +5. **deb-mock** provides build results back to deb-orchestrator + +## Development Status + +### **Current Phase** +- **Status**: Foundation development +- **Phase**: Phase 1 (Weeks 1-8 of parallel development) +- **Timeline**: Parallel development with deb-bootc-compose and deb-orchestrator + +### **Development Goals** +- [ ] Basic chroot management system +- [ ] Package installation using apt/dpkg +- [ ] Basic environment isolation +- [ ] Simple plugin framework +- [ ] YAML-based configuration system + +### **Success Criteria** +- Can create isolated chroot environments +- Can install packages in chroots +- Basic isolation working +- Simple plugin system functional +- Configuration system working + +## Technology Stack + +### **Primary Language** +- **Python**: For rapid development and Debian tooling integration + +### **Key Dependencies** +- **debootstrap**: For creating base chroot environments +- **schroot**: For chroot management and isolation +- **sbuild**: For package building within chroots +- **apt/dpkg**: For package management within chroots + +### **Integration Points** +- **deb-orchestrator**: REST API for build environment requests +- **deb-bootc-compose**: Direct integration for specific build operations +- **Debian build tools**: Integration with sbuild, schroot, debootstrap +- **Package repositories**: Debian package sources and results + +## Getting Started + +### **Prerequisites** +- Python 3.9+ installed +- Debian build tools (debootstrap, schroot, sbuild) +- Basic understanding of Debian packaging and chroot environments + +### **Development Setup** +```bash +# Navigate to the project directory +cd parallel_projects/deb-mock + +# Install Python dependencies +pip install -r requirements.txt + +# Run basic tests +python -m pytest tests/ + +# Run the CLI +python -m deb_mock.cli --help +``` + +### **Configuration** +Configuration is handled through YAML files. See `config.yaml` for examples. + +## Key Differences from Fedora's Mock + +### **Package Management** +- **Mock**: Uses RPM, DNF/YUM +- **deb-mock**: Uses DEB, apt/dpkg + +### **Build Tools** +- **Mock**: Integrates with rpmbuild +- **deb-mock**: Integrates with sbuild, debuild + +### **Repository Structure** +- **Mock**: RPM repositories with metadata +- **deb-mock**: Debian package pools with Sources/Packages files + +### **Configuration** +- **Mock**: Python .cfg files +- **deb-mock**: YAML configuration files + +## Contributing + +This project is part of Debian's bootc ecosystem development. Contributions are welcome! + +### **Development Guidelines** +- Follow Python best practices and PEP 8 +- Write comprehensive tests for all functionality +- Document all public APIs and interfaces +- Ensure compatibility with Debian's development standards + +### **Contact** +- **Project**: Part of deb-bootc-compose ecosystem +- **Goal**: Create Debian's equivalent to Fedora's Mock build environment manager +- **Timeline**: Phase 1 development (Weeks 1-8) + +## License + +[License information to be added] + +--- + +**Note**: This is a development project. The project is currently in early development and not ready for production use. It's being developed in parallel with deb-bootc-compose and deb-orchestrator to create Debian's complete bootc ecosystem. diff --git a/mock/docs/buildroot-lock-schema-1.0.0.json b/mock/docs/buildroot-lock-schema-1.0.0.json new file mode 100644 index 0000000..a8cd03a --- /dev/null +++ b/mock/docs/buildroot-lock-schema-1.0.0.json @@ -0,0 +1,102 @@ +{ + "$id": "https://raw.githubusercontent.com/rpm-software-management/mock/main/mock/docs/buildroot-lock-schema-1.0.0.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "type": "object", + "title": "Mock buildroot_lock.json file specification", + "description": "Version 1.0.0; last updated 2024-09-02", + "additionalProperties": false, + "properties": { + "version": { + "description": "Version of the https://raw.githubusercontent.com/rpm-software-management/mock/main/mock/docs/buildroot-lock-schema.json schema the document conforms to. Semantic versioned. Mock that implements v2.Y.Z versions no longer reads v1.Y.Z.", + "const": "1.0.0" + }, + "buildroot": { + "description": "The object that describes the Mock buildroot", + "type": "object", + "additionalProperties": false, + "properties": { + "rpms": { + "description": "List of RPM packages installed in the buildroot", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "arch": { + "description": "Architecture for which the package was built, 'noarch' for arch agnostic packages", + "type": "string" + }, + "epoch": { + "description": "Epoch number of the package", + "type": ["string", "null"] + }, + "license": { + "description": "The distribution license(s) of the package", + "type": "string" + }, + "name": { + "description": "Name of the package", + "type": "string" + }, + "release": { + "description": "Release (downstream) number of the package", + "type": "string" + }, + "sigmd5": { + "description": "The SIGMD5 tag from the rpm header.", + "type": "string" + }, + "signature": { + "description": "The signature used to sign the rpm (if any), last 8 characters from the \"rpm -q --qf '%{sigpgp:pgpsig}\n'\" output", + "type": ["string", "null"] + }, + "url": { + "description": "Uniform Resource Locator that points to additional information on the packaged software", + "type": "string" + }, + "version": { + "description": "Version (upstream) of the package", + "type": "string" + } + }, + "required": [ + "arch", + "epoch", + "license", + "name", + "release", + "sigmd5", + "signature", + "url", + "version" + ] + } + } + }, + "required": [ + "rpms" + ] + }, + "bootstrap": { + "description": "The object that describes the Mock bootstrap chroot. Optional, only provided when bootstrap (image) is used.", + "type": "object", + "additionalProperties": false, + "properties": { + "image_digest": { + "description": "SHA256 digest concatenated RootFS layer digests and Config section from 'podman image inspect' command, sha256 string", + "type": "string" + } + } + }, + "config": { + "description": "A set of important Mock configuration options used when the buildroot was generated (Mock's internal)", + "type": "object", + "properties": {} + } + }, + "required": [ + "buildroot", + "config", + "version" + ] +} diff --git a/mock/docs/buildroot-lock-schema-1.1.0.json b/mock/docs/buildroot-lock-schema-1.1.0.json new file mode 100644 index 0000000..4c7c5bf --- /dev/null +++ b/mock/docs/buildroot-lock-schema-1.1.0.json @@ -0,0 +1,120 @@ +{ + "$id": "https://raw.githubusercontent.com/rpm-software-management/mock/main/mock/docs/buildroot-lock-schema-1.1.0.json", + "$schema": "http://json-schema.org/draft-06/schema#", + "type": "object", + "title": "Mock buildroot_lock.json file specification", + "description": "Version 1.1.0; last updated 2025-02-03", + "additionalProperties": false, + "properties": { + "version": { + "description": "Version of the https://raw.githubusercontent.com/rpm-software-management/mock/main/mock/docs/buildroot-lock-schema.json schema the document conforms to. Semantic versioned. Mock that implements v2.Y.Z versions no longer reads v1.Y.Z.", + "const": "1.1.0" + }, + "buildroot": { + "description": "The object that describes the Mock buildroot", + "type": "object", + "additionalProperties": false, + "properties": { + "rpms": { + "description": "List of RPM packages installed in the buildroot", + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "arch": { + "description": "Architecture for which the package was built, 'noarch' for arch agnostic packages", + "type": "string" + }, + "epoch": { + "description": "Epoch number of the package", + "type": ["string", "null"] + }, + "license": { + "description": "The distribution license(s) of the package", + "type": "string" + }, + "name": { + "description": "Name of the package", + "type": "string" + }, + "release": { + "description": "Release (downstream) number of the package", + "type": "string" + }, + "sigmd5": { + "description": "The SIGMD5 tag from the rpm header.", + "type": "string" + }, + "signature": { + "description": "The signature used to sign the rpm (if any), last 8 characters from the \"rpm -q --qf '%{sigpgp:pgpsig}\n'\" output", + "type": ["string", "null"] + }, + "url": { + "description": "Uniform Resource Locator that points to additional information on the packaged software", + "type": "string" + }, + "version": { + "description": "Version (upstream) of the package", + "type": "string" + } + }, + "required": [ + "arch", + "epoch", + "license", + "name", + "release", + "sigmd5", + "signature", + "url", + "version" + ] + } + } + }, + "required": [ + "rpms" + ] + }, + "bootstrap": { + "description": "The object that describes the Mock bootstrap chroot. Optional, only provided when bootstrap (image) is used.", + "type": "object", + "additionalProperties": false, + "required": [ + "image_digest", + "pull_digest", + "architecture", + "id" + ], + "properties": { + "image_digest": { + "description": "SHA256 digest concatenated RootFS layer digests and Config section from 'podman image inspect' command, sha256 string", + "type": "string" + }, + "pull_digest": { + "description": "Image digest, as reported by podman inspect, can be used for podman pull.", + "type": "string" + }, + "architecture": { + "description": "OCI architecture string, as reported by podman inspect .Architecture field.", + "type": "string" + }, + "id": { + "type": "string", + "description": "Image ID, as reported by podman inspect .Id" + } + } + }, + "config": { + "description": "A set of important Mock configuration options used when the buildroot was generated (Mock's internal)", + "type": "object", + "properties": {} + } + }, + "required": [ + "buildroot", + "config", + "version" + ] +} diff --git a/mock/docs/mock.1 b/mock/docs/mock.1 new file mode 100644 index 0000000..616e0c2 --- /dev/null +++ b/mock/docs/mock.1 @@ -0,0 +1,588 @@ +.TH "mock" "1" "@VERSION@" "Seth Vidal" "" +.SH "NAME" +.LP +mock \- build SRPMs in a chroot +.SH "SYNTAX" +.LP +mock [options] \fB\-\-rebuild\fR \fISRPM [\fISRPM...\fR] +.LP +mock [options] \fB\-\-chain\fR \fISRPM [\fISRPM...\fR] +.LP +mock [options] \fB\-\-buildsrpm\fR {\-\-spec \fIspec\fR \-\-sources \fIsrc\fR \-\-symlink\-dereference | \-\-scm\-enable} +.LP +mock [options] \fB\-\-chroot\fR \fI\fR +.LP +mock [options] {\fB\-\-init\fR|\fBclean\fR|\fBshell\fR} +.LP +mock [options] \fB\-\-installdeps\fR {SRPM|RPM|SPEC} +.LP +mock [options] \fB\-i\fR, \fB\-\-install\fR PACKAGE [\fIPACKAGE...\fR] +.LP +mock [options] \fB\-\-update\fR [\fIPACKAGE...\fR] +.LP +mock [options] \fB\-\-remove\fR PACKAGE [\fIPACKAGE...\fR] +.LP +mock [options] \fB\-\-orphanskill\fR +.LP +mock [options] \fB\-p\fR, \fB\-\-print\-root\-path\fR +.LP +mock [options] \fB\-\-copyin\fR \fIpath [\fIpath...\fR] \fIdestination\fR +.LP +mock [options] \fB\-\-copyout\fR \fIpath [\fIpath...\fR] \fIdestination\fR +.LP +mock [options] \fB\-\-scm\-enable\fR [\fI\-\-scm\-option key=value ...\fR] +.LP +mock [options] \fB\-l\fR, \fB\-\-list\-snapshots\fR +.LP +mock [options] \fB\-\-snapshot\fR [\fIsnapshot\-name\fR] +.LP +mock [options] \fB\-\-rollback\-to\fR [\fIsnapshot\-name\fR] +.LP +mock [options] \fB\-\-remove\-snapshot\fR [\fIsnapshot\-name\fR] +.LP +mock [options] \fB\-\-umount\fR +.LP +mock [options] \fB\-\-pm\-cmd\fR [\fIarguments ...\fR] +.LP +mock [options] \fB\-\-yum\-cmd\fR [\fIarguments ...\fR] +.LP +mock [options] \fB\-\-dnf\-cmd\fR [\fIarguments ...\fR] +.LP +mock [options] \fB\-\-calculate\-build\-dependencies\fR \fISRPM\fR +.LP +mock [options] \fB\-\-hermetic\-build \fILOCKFILE\fR \fIREPO\fR \fISRPM\fR +.LP +mock [options] {\fB\-\-scrub\fR=\fITYPE\fP,\fB\-\-scrub\-all\-chroots\fR} + + +.SH "DESCRIPTION" +.LP +Mock is a simple program that will build source RPMs inside a chroot. It +doesn't do anything fancy other than populating a chroot with the +contents specified by a configuration file, then build any input SRPM(s) in +that chroot. +.LP +The content of a chroot is specified by the configuration specified with the +\fB\-r\fR option. The default configuration file is /etc/mock/default.cfg, +which is usually a symlink to one of the installed configurations. +.LP +There is a site\-wide configuration file, /etc/mock/site\-defaults.cfg, which can +be used to specify site\-wide options. The shipped version of this file has no +active options, but does have a list of all of the configuration options +examples of how to set them, and their default values. +.LP +To change configuration only for the current user please use ~/.config/mock.cfg +configuration file. +.LP +For backward compatibility, old\-style commands, ("rebuild", "init", "clean", +etc.) without leading '\-\-' are still accepted, but are deprecated. See +COMMANDS section, below, for the detailed listing of all commands. +.LP +To use mock, a user should become a member of the \fBmock\fR group by +adding their username to the \fBmock\fR line in /etc/group. This can +be done with the following command: +.P + \fBsudo /usr/sbin/usermod \-a \-G mock $USER\fR + +Note that mock is not intended to be run directly as root. +.LP +Warning: +Mock is running some parts of code with root privileges. There are known ways to get root access once a user is in mock group (and once he is able to run mock). This is possible when a user abuses the mock configuration options. Please do not add anyone who is not trustworthy to the mock group! + + +.SH "COMMANDS" +.LP +.TP +\fB\-\-buildsrpm\fP +Build the specified SRPM either from a spec file and source file/directory or SCM. The chroot (including the results directory) is cleaned first, unless \-\-no\-clean is specified. +.TP +\fB\-\-calculate\-build\-dependencies\fR \fISRPM\fR +Evaluate and install all the \fISRPM\fR (= file name, path on your system) build +dependencies, including dynamic dependencies in \fI%generate_buildrequires\fR. +This is similar to the \fB\-\-installdeps\fR option which only installs the +static \fIBuildRequires\fR. + +Build chroot-native \fI*.src.rpm\fR and \fI*.nosrc.rpm\fR files (the later only +when \fI%generate_buildrequires\fR is in use!). The \fI*.src.rpm\fR records the +corresponding static list of build dependencies (= \fIBuildRequires\fR). The +\fI*.nosrc.rpm\fR records a full list of build dependencies (static + dynamic). +Test with \fIrpm -qpR [*.src.rpm|*.nosrc.rpm]\fR. + +Additionally, provide a \fIbuildroot_lock.json\fR file; this records the +metadata needed for a hermetic build (see also \fB\-\-hermetic\-build\fR). +.TP +\fB\-\-chain\fR +When passing more than one SRPM, it will try to build failed builds if at least one subsequent SRPM succeed. This mimic the behaviour of deprecated mockchain. +.TP +\fB\-\-clean\fP +Purge the chroot tree. +.TP +\fB\-\-copyin\fP +Copies the source paths (files or directory trees) into the chroot at +the specified destination path. +.TP +\fB\-\-copyout\fP +Copies the source paths (files or directory trees) from the chroot to +the specified destination path. +.TP +\fB\-\-chroot\fP [\fI\-\-\fR] \fICOMMAND\fR [\fIARGS...\fR] +Run the specified command non\-interactively within the chroot (no +\fB\-\-clean\fR is performed). + +This mode is similar to \fishell\fR mode, except that the output \fBis logged\fR +and the \fICOMMAND\fR and \fIARGS\fR arguments are not shell expanded in chroot +when the variant with \fICMD+ARGS\fR is used, see the difference in mock output: + \fBmock --quiet --chroot -- echo '*'\fR + * + \fBmock --quiet --chroot -- 'echo *'\fR + bin boot builddir dev etc home lib lib64 media mnt opt ... +See also \fB\-\-shell\fR. +.TP +\fB\-\-debug-config\fP +Print all options in config_opts. +.TP +\fB\-\-debug-config-expanded\fP +Prints all options in config_opts with jinja template values already expanded. +.TP +\fB\-\-dnf\-cmd\fP +Execute following arguments with DNF with installroot set to the chroot path. DNF must be installed on the system. +It will use the binary which is specified in 'dnf_command' option in site-defaults.cfg config, which by default is /usr/bin/dnf. +This option will enforce \-\-dnf. +.TP +\fB\-\-init\fP +Initialize a chroot (clean, install chroot packages, etc.). +.TP +\fB\-i\fR, \fB\-\-install\fP +Do a yum install PACKAGE inside the chroot. No 'clean' is performed. +.TP +\fB\-\-installdeps\fP +Find out "static" deps for SRPM or RPM, and do a \fIdnf install\fR to put them +into the buildroot. No 'cleanup' is performed. + +Dynamic build dependencies (\fI%generate_buildrequires\fR specfile section) are +not installed, see \fB\-\-calculate\-build\-dependencies\fR. +.TP +\fB\-\-hermetic\-build \fILOCKFILE\fR \fIREPO\fR \fISRPM\fR +Perform a hermetic RPM build (i.e., an offline build without the need to access +the Internet at all) from the given \fISRPM\fR (= file name, path on your +system). After running Mock with the \fB\-\-calculate\-build\-dependencies\fR +option to generate the \fILOCKFILE\fR file (typically named +\fIbuildroot_lock.json\fR in the result directory), and then running the +\fImock\-hermetic\-repo(1)\fR helper to generate \fIREPO\fR (a directory on the +host that provides RPMs with metadata and a bootstrap image tarball), Mock has +all the necessary information to build RPMs from the given \fISRPM\fR fully +offline. More info in the feature page: + +\fIhttps://rpm-software-management.github.io/mock/feature-hermetic-builds\fR +.TP +\fB\-\-list-chroots\fP +List all available chroots names and their description - both system-wide and user ones. +.TP +\fB\-l\fR, \fB\-\-list\-snapshots\fP +List all existing snapshots of the chroot belonging to the current configuration. +Current base snapshot is marked with an asterisk (\fB*\fR) +.TP +\fB\-\-mount\fP +Mount all everything mounted in the chroot path including the root itself +that might have been an LVM volume, TMPFS or overlayfs. +.TP +\fB\-\-orphanskill\fP +No\-op mode that simply checks that no stray processes are running in the chroot. Kills any processes that it finds using the specified root. +.TP +\fB\-\-pm\-cmd\fP +Execute following arguments with the current package manager with installroot set to +the chroot path. +.TP +\fB\-p\fR, \fB\-\-print\-root\-path\fP +Prints a path to the currently used chroot directory. +.TP +\fB\-\-rebuild\fP +If no command is specified, rebuild is assumed. Rebuild the specified SRPM(s). The chroot (including the results directory) is cleaned first, unless \-\-no\-clean is specified. +.TP +\fB\-\-remove\fP +Do a yum remove PACKAGE inside the chroot. No 'clean' is performed. +.TP +\fB\-\-remove\-snapshot\fP +Remove given snapshot freeing the space it occupied. This action cannot be +undone. +This feature is available only when lvm_root or overlayfs plugin is installed and enabled. +.TP +\fB\-\-rollback\-to\fP +Return chroot to the state in the specified snapshot and set it as the current +base to which clean actions will return. It won't delete nor modify the snapshot +that was set as base previously. +This feature is available only when the lvm_root or overlayfs plugin is installed and enabled. +.TP +\fB\-\-scm\-enable\fP +Enable building from an SCM (CVS/Git/SVN/DistGit). The SCM repository must be +configured in site\-defaults.cfg before SCM checkouts are possible. SCM +package and branch can be defined with \fB\-\-scm\-option\fP arguments, +see site\-defaults.cfg for more information. +.TP +\fB\-\-scrub\fR=\fITYPE\fP +Completely remove the specified chroot or cache dir or all of the chroot and cache. \fITYPE\fR is one of all, chroot, bootstrap, cache, root\-cache, c\-cache, yum\-cache or dnf\-cache. In fact, dnf\-cache is just alias for yum\-cache, and both remove Dnf and Yum cache. +.TP +\fB\-\-scrub\-all\-chroots\fP +Run \fBmock \-\-scrub=all \-r <\fIchroot\fB>\fR for all chroots that appear to +have been used previously (some leftovers in \fB/var/lib/mock\fR or +\fB/var/cache/mock\fR were detected by the heuristic). This option cannot clean +leftovers for chroots with configurations in non-standard locations, or if the +configuration is no longer available. It also attempts to detect previous use +of \fB\-\-uniqueext\fR and adjusts the corresponding \fB\-\-scrub=all\fR call +accordingly. +.TP +\fB\-\-shell\fP [\fI\-\-\fR] [\fICOMMAND\fR [\fIARGS...\fR]] +Shell mode. Run the specified command interactively within the chroot (no +\fB\-\-clean\fR is performed). If no command specified, \fB/bin/sh\fR is run +and prompt is provided. + +Be aware that mock first parses all the command-line arguments, so the +\fIARGS\fR could be mistakenly evaluated as mock's options. Thats why you +almost always want to use the \fI\-\-\fR separator. + +This mode does not produce logs (nothing is appended to \fBroot.log\fR in +\fB\-\-resultdir\fR). + +The \fICOMMAND\fR and \fIARGS\fR are shell expanded using the shell in chroot +(unless they mistakenly expand in host's terminal shell). E.g. the following +two commands are equivalent: + \fBmock \-\-shell \-\- ls \-l '*'\fR + \fBmock \-\-shell 'ls \-l *'\fR +.br +But the following is something entierly different: + \fBmock \-\-shell \-\- ls \-l *\fR +.TP +\fB\-\-sources\fR=\fISOURCES\fP +Specifies sources (either a single file or a directory of files) to use to build an SRPM (used only with \-\-buildsrpm). +.TP +\fB\-\-spec\fR=\fISPEC\fP +Specifies spec file to use to build an SRPM. +.TP +\fB\-\-update\fP [\fIPACKAGE...]\fR +Do a package update inside the chroot. The package list is optional, if omitted, all packages will be updated. No 'clean' is performed. +.TP +\fB\-\-snapshot\fP +Make a snapshot of the current state of the chroot. That snapshot will be set +as the current base to which \fV\-\-clean\fP and implicit clean happening during +rebuild command will return. +This feature is available only when the lvm_root or overlayfs plugin is installed and enabled. +.TP +\fB\-\-umount\fP +Umount all everything mounted in the chroot path including the root itself +that might have been an LVM volume, TMPFS or overalyfs. +.TP +\fB\-\-yum\-cmd\fP +Execute following arguments with YUM with installroot set to the chroot path. Yum must be installed on the system. +It will use the binary which is specified in 'yum_command' option in site-defaults.cfg config, which by default is /usr/bin/yum. +Note that in config files for Fedora 22\+ this value is overwritten in chroot config to default to /usr/bin/yum-deprecated. +This option will enforce \-\-yum. +.TP +Note: While you can specify more commands on a command line, only one can be executed. The last command will win. + + +.SH "OPTIONS" +.LP +.TP +\fB\-a\fR, \fB\-\-addrepo\fR=\fIREPO\fP +Add a repo baseurl to the DNF/YUM configuration for both the build chroot and +the bootstrap chroot. This option can be specified multiple times, allowing you +to reference multiple repositories in addition to the default repository set. +.TP +\fB\-\-arch\fR=\fIARCH\fP +Calls the Linux personality() syscall to tell the kernel to emulate a secondary architecture. For example, building i386 packages on an x86_64 buildhost. +.TP +\fB\-\-additional\-package\fR=\fIPACKAGE\fP +An additional package (on top of in-package specified BuildRequires) to be +installed into the buildroot before the build is done. Can be specified +multiple times. Works only with \fB\-\-rebuild\fR. +.TP +\fB\-\-forcearch\fR=\fIARCH\fP +Pass \-\-forcearch to DNF. This will enable to install packages for different architecture. Works only for DNF and you have to have package qemu-user-static installed. +.TP +\fB\-\-cache\-alterations\fR +Rebuild the root cache after making alterations to the chroot (i.e. \-\-install). This option is useful only when using tmpfs plugin. +.TP +\fB\-\-cleanup\-after\fR +Clean chroot after building. Use with \-\-resultdir. Only active for '\-\-rebuild'. +.TP +\fB\-\-configdir\fR=\fICONFIGDIR\fP +Change directory where config files are found +.TP +\fB\-\-config-opts\fR=\fIKEY=VALUE\fP +Override configuration option. Can be used multiple times. + +When used multiple times for the same key, it will create an array (if you need +to specify an array value with just a single item, e.g. +\fIconfig_opts["foo"] = ["baz"]\fR, specify it as array of two items with the +empty string as the second item, e.g. \fI--config-opts=foo=baz +--config-opts=foo=\fR). + +This is evaluated after parsing configs, so command line options override +previously defined options. +.TP +\fB\-\-continue\fR +If a pkg fails to build, continue to the next one, default is to stop. + +Works only with \fB\-\-chain\fR. +.TP +\fB\-\-cwd\fR=\fIDIR\fP +Change to the specified directory (relative to the chroot) before running command when using \-\-chroot or \-\-shell. +.TP +\fB\-D \fR"\fIMACRO EXPR\fP", \fB\-\-define\fR="\fIMACRO EXPR\fP" +Specify macro definitions used for the build. This option may be used multiple times, just as the rpmbuild \-\-define option can be. For example: + +\fB\-\-define "with_extra_cheese 1" \-\-define="packager Monkey"\fR +.TP +\fB\-\-disable\-plugin\fR=\fIPLUGIN\fP +Disable the specified plugin. This option may be used multiple times. +.TP +\fB\-\-disablerepo\fR=\fIREPO\fR +Pass \fB\-\-disablerepo\fR option to package manager to disable a repository. +It can be specified multiple times. +.TP +\fB\-\-dnf\fR +Use DNF as the current package manager. You should have DNF (and dnf-plugins-core) installed on your system. This is the default. +.TP +\fB\-\-enable\-plugin\fR=\fIPLUGIN\fP +Enable the specified plugin. This option may be used multiple times. +.TP +\fB\-\-enablerepo\fR=\fIREPO\fR +Pass \fB\-\-enablerepo\fR option to package manager to enable a repository. +It can be specified multiple times. +.TP +\fB\-\-enable\-network\fR +Enable networking. If you want to have reproducible builds then your builds should run without a network. +This option overrides config_opts['rpmbuild_networking'] and config_opts['use_host_resolv'], setting both True. +.TP +\fB\-\-isolation\fR={\fIauto\fR|\fInspawn\fR|\fIsimple\fR} +What should be used for isolation of chroot. The \fIsimple\fR method uses +chroot() call. The \fInspawn\fR method utilizes systemd-nspawn(1) and runs the +commands inside container. The \fIauto\fR tries to use \fInspawn\fR, and falls +back to \fIsimple\fR if system-nspawn can not be used (e.g. if mock is run in +container). The default is \fIauto\fR. +.TP +\fB\-\-localrepo\fR=\fIREPO\fR\fR +Set the path to put the results/repo in (works only in \fB\-\-chain\fR mode). +Will make a tempdir if not set. +.TP +\fB\-c\fR +If package fails, continue to the next one (works only in \fB\-\-chain\fR mode). +.TP +\fB\-h\fR, \fB\-\-help\fR +Show usage information and exit. +.TP +\fB\-\-macro\-file\fR=\fIFILE\fR +Use pre\-defined rpm macro file. Macros passed to '\-\-define' override macros of the same name from FILE. +.TP +\fB\-\-new\-chroot\fR +Deprecated. Use \fV\-\-isolation=nspawn\fP. +.TP +\fB\-n\fR, \fB\-\-no\-clean\fR +Do not clean chroot before building a package. +.TP +\fB\-\-nocheck\fR +Pass \-\-nocheck to rpmbuild to skip 'make check' tests. +.TP +\fB\-N\fR, \fB\-\-no\-cleanup\-after\fR +Don't clean chroot after building. If automatic cleanup is enabled, use this to disable. +.TP +\fB\-\-offline\fR +Run in an 'offline' mode where we tell 'yum' to run completely from the local cache. Also, disables cache expiry for the mock yum cache. +.TP +\fB\-\-old\-chroot\fR +Deprecated. Use \fV\-\-isolation=simple\fP. +.TP +\fB\-\-plugin\-option \fR\fIPLUGIN\fR\fB:\fR\fIKEY\fR\fB=\fR\fIVALUE\fP +Set plugin specific parameter. This option may be used multiple times. +Examples: + +\fB\-\-plugin\-option=root_cache:age_check=False\fR + +\fB\-\-plugin\-option=mount:dirs=("/dev/device", "/mount/path/in/chroot/", "vfstype", "mount_options")\fR +.TP +\fB\-\-postinstall\fR +Try to install built packages in the same buildroot right after the build. +.TP +\fB\-q\fR, \fB\-\-quiet\fR +Be quiet. +.TP +\fB\-\-recurse\fR +Build all pkgs, record the failures and try to rebuild them again and again +until everything gets built (or until the set of pkgs failing to build are the +same over) sets \fB\-\-continue\fR. Works only with \fB\-\-chain\fR. +.TP +\fB\-r\fR \fICONFIG\fP, \fB\-\-root\fR=\fICONFIG\fP +Uses specified chroot configuration as defined in +~/.config/mock/<\fICONFIG\fP>.cfg or /etc/mock/<\fICONFIG\fP>.cfg. +Optionally if CONFIG ends in '.cfg', it is +interpreted as full path to config file. If none specified, uses the chroot +config linked to by /etc/mock/default.cfg. +.TP +\fB\-\-resultdir\fR=\fIRESULTDIR\fP +Change directory where resulting files (RPMs and build logs) are written. Resultdir can contain python\-string substitutions for any variable in the chroot config. For example: + +\fB\-\-resultdir=./my/"{{dist}}"/"{{target_arch}}"/\fR + +This option enables automatic cleanup, this can be changed in config file (by cleanup_on_success, cleanup_on_failure configuration options) or overridden by \-\-no\-cleanup\-after/\-\-cleanup\-after arguments. + +Note that this option does not have an effect for \-\-chain command. You can use \-\-localrepo instead. +.TP +\fB\-\-rootdir\fR=\fIROOTDIR\fP +The path for where the chroot should be built. By default it is created in /var/lib/mock/<\fICONFIG\fP>/root/. +.TP +\fB\-\-rpmbuild\-opts\fR=\fIOPTIONS\fR +Pass additional options to rpmbuild. To pass more options, put them in quotes. +.TP +\fB\-\-rpmbuild_timeout\fR=\fISECONDS\fP +Fail build if rpmbuild takes longer than 'timeout' seconds +.TP +\fB\-\-scm\-option\fR=\fIOPTIONS\fR +define an SCM option (may be used more than once). +.TP +\fB\-\-short\-circuit\fR=\fISTAGE\fR +Use rpmbuild's short\-circuit mechanism to skip already executed stages of the build. +It doesn't produce RPMs, and it's useful only for debugging packaging. Implies +\fI\-\-no\-clean\fR. STAGE specifies which stage will be executed as the first. +Available values: prep, build, install, binary. +.TP +\fB\-\-symlink\-dereference\fR +Follow symlinks in sources (used only with \-\-buildsrpm). +.TP +\fB\-\-target\fR=\fIARCH\fP +This argument is passed to rpmbuild to specify the target arch to build. It defaults to whatever is specified for \-\-arch, or whatever is specified in the config file as config_opts['target_arch']. +.TP +\fB\-\-tmp_prefix\fR=\fIPREFIX\fP +Tmp dir prefix - will default to username-pid if not specified. +.TP +\fB\-\-trace\fR +Enables verbose tracing of function enter/exit with function arguments and return codes. Useful for debugging mock itself. +.TP +\fB\-\-uniqueext\fR=\fItext\fP +Arbitrary, unique extension to append to chroot directory name +.TP +\fB\-\-unpriv\fR +Drop privileges before running command when using \-\-chroot +.TP +\fB\-v\fR, \fB\-\-verbose\fR +Output verbose progress information. +.TP +\fB\-\-version\fR +Show version number and exit. +.TP +\fB\-\-with\fR=\fIOPTION\fP +Enable configure OPTION for the build. This option may be used multiple times. For example: + +\fB\-\-with=extra_cheese\fR +.TP +\fB\-\-without\fR=\fIOPTION\fP +Disable configure OPTION for the build. This option may be used multiple times. For example: + +\fB\-\-without=anchovies\fR +.TP +\fB\-\-yum\fR +Use yum as the current package manager. + +.TP +\fB\-\-bootstrap-chroot\fR +build in two stages, using chroot rpm for creating the build chroot +.TP +\fB\-\-no-bootstrap-chroot\fR +build in a single stage, using system rpm for creating the build chroot + +.TP +\fB\-\-use-bootstrap-image\fR +Instead of creating a bootstrap chroot from scratch, use podman image specified in +\fBconfig_opts['bootstrap_image']\fR, extract it, and use it as a cache for the bootstrap chroot. +This is useful when host \fBrpm\fR version is not compatible with the target system, or when using mock +on non-RPM distributions. This option turns \fB\-\-bootstrap\-chroot\fR on. + +.TP +\fB\-\-no-bootstrap-image\fR +don't create bootstrap chroot from container image + +.TP +\fB\-\-buildroot\-image\fR \fIBUILDROOT_IMAGE\fR +Use an OCI image (or a local file containing an OCI image as a tarball) as the +base for the buildroot. The image must contain a compatible distribution. + +.SH "FILES" +.LP +\fI/etc/mock/\fP \- default configuration directory +.LP +\fI/var/lib/mock\fP \- directory where chroots and results are created. You should not put there your input files. +.SH "EXAMPLES" +.LP +To rebuild test.src.rpm using the Fedora 14 configuration for x86_64 +.LP +.RS 5 +\fBmock \-r fedora\-14\-x86_64 \-\-rebuild /path/to/test.src.rpm +.RE +.LP +Note that the available configurations are found in the /etc/mock +directory with the extension .cfg. To specify a configuration use the +filename without the trailing .cfg extension. +.LP +To place the output RPMs and logs in a specified location. +.LP +.RS 5 +\fBmock \-r fedora\-14\-i386 \-\-resultdir=./my\-results /path/to/your.src.rpm\fR +.RE +.LP +To build a package from the default SCM repository configured in site\-defaults.cfg use the following arguments. +.LP +.RS 5 +\fBmock \-r fedora\-14\-i386 \-\-scm\-enable \-\-scm\-option package=pkg\fR +.RE +.LP +To execute a command inside of chroot. +.LP +.RS 5 +\fBmock \-r fedora\-21\-x86_64 \-\-chroot \-\- rpm --eval %dist\fR +.LP +\fBmock \-r fedora\-21\-x86_64 \-\-chroot 'rpm --eval %dist'\fR +.RE +.LP +To build rawhide package using yum: +.LP +.RS 5 +\fBmock \-r fedora\-rawhide\-x86_64 \-\-yum \-\-rebuild your.src.rpm\fR +.RE +.LP +Query rpm database inside chroot using Yum: +.LP +.RS 5 +\fBmock \-r fedora\-rawhide\-x86_64 \-\-yum-cmd whatprovides foo\fR +.RE +.LP +List package manager history using package manager which is configured in chroot config (can be either DNF or YUM): +.LP +.RS 5 +\fBmock \-r fedora\-rawhide\-x86_64 \-\-pm-cmd history list\fR +.RE + +.SH "BUGS" +.LP +To report an issue with Mock, go to: +.LP +.RS 5 +\fIhttps://github.com/rpm-software-management/mock/issues\fR +.RE +.LP +Search through the list of existing issues. If there is a similar +issue to the one you are seeing, add your information in new comments. +If not, press \fBNew issue\fR and fill in the form. +.SH "AUTHORS" +.LP +Michael Brown +.LP +Clark Williams +.LP +Seth Vidal +.LP +and a cast of...tens +.SH "SEE ALSO" +.LP +rpmbuild(8), +yum(8), +dnf(8), +https://rpm-software-management.github.io/mock/ diff --git a/mock/docs/mock.cheat b/mock/docs/mock.cheat new file mode 100644 index 0000000..f082e36 --- /dev/null +++ b/mock/docs/mock.cheat @@ -0,0 +1,14 @@ +# to build a package +mock -r fedora-30-x86_64 foo.src.rpm + +# to enter shell in different distribution +mock -r epel-7-x86_64 --shell + +# to run command in different distribution +mock -r epel-7-x86_64 --chroot cat /etc/os-release + +# to build for different architecture +mock -r fedora-30-s390x --forcearch=s390x foo.src.rpm + +# to clean everything +mock -r fedora-30-x86_64 --scrub=all diff --git a/mock/docs/site-defaults.cfg b/mock/docs/site-defaults.cfg new file mode 100644 index 0000000..622eae3 --- /dev/null +++ b/mock/docs/site-defaults.cfg @@ -0,0 +1,759 @@ +# mock defaults +# vim:tw=0:ts=4:sw=4:et: +# +# If you want to alter just some setting for one user, you can put the +# configuration in: +# ~/.config/mock.cfg +# +# This config file is for site-specific default values that apply across all +# configurations. Options specified in this config file can be overridden in +# the individual mock config files. +# +# The site-defaults.cfg delivered by default has NO options set. Only set +# options here if you want to override the defaults. +# +# It's possible to use include statement in order to make one config included to another: +# include('/path/to/included/config.cfg') +# +# Entries in this file follow the same format as other mock config files. +# config_opts['foo'] = 'bar' +# +# You can use jinja templates, e.g.: +# config_opts['foobar'] = '{{ foo }} bar' +# which will result in 'bar bar' (using value defined few lines above) +# more complicated example: +# config_opts['foo'] = "{{ plugin_conf['package_state_enable'] }}" +# which will result in "True" + +############################################################################# +# +# Things that we recommend you set in site-defaults.cfg: +# +# config_opts['basedir'] = '/var/lib/mock/' +# config_opts['cache_topdir'] = '/var/cache/mock' +# Note: the path pointed to by basedir and cache_topdir must be owned +# by group 'mock' and must have mode: g+rws +# config_opts['rpmbuild_timeout'] = 0 +# Using --enable-network sets use_host_resolv True, overriding the value set here. +# config_opts['use_host_resolv'] = False + +# You can configure log format to pull from logging.ini formats of these names: +# config_opts['build_log_fmt_name'] = "unadorned" +# config_opts['root_log_fmt_name'] = "detailed" +# config_opts['state_log_fmt_name'] = "state" +# +# By default, mock only prints the build log to stderr if it is a tty. You can +# force it on here (for CI builds where there is no tty, for example) by +# setting this to True, or force it off by setting it to False. Setting it to +# None or leaving it undefined uses the default behavior. +# config_opts['print_main_output'] = None +# +# This option sets the beginning of the line with stderr output in build.log file. +# By default it's an empty string. +# config_opts['stderr_line_prefix'] = "" +# +# mock will normally set up a minimal chroot /dev. +# If you want to use a pre-configured /dev, disable this and use the bind-mount +# plugin to mount your special /dev +# config_opts['internal_dev_setup'] = True +# +# the cleanup_on_* options allow you to automatically clean and remove the +# mock build directory, but only take effect if --resultdir is used. +# config_opts provides fine-grained control. cmdline only has big hammer +# +# config_opts['cleanup_on_success'] = True +# config_opts['cleanup_on_failure'] = True + +# The build user's homedir is partially cleaned up even when --no-clean is +# specified in order to prevent garbage from previous builds from altering +# successive builds. Mock can be configured to exclude certain files/directories +# from this. Default is SOURCES directory to support nosrc rpms. Paths are +# relative to build user's homedir +# config_opts['exclude_from_homedir_cleanup'] = ['build/SOURCES'] + +# Choose the isolation method for running commands in buildroot. Possible +# values are 'nspawn' (mock uses systemd-nspawn(1)), 'simple' (simple +# os.chroot() is used) or 'auto' (use 'nspawn' when possible, and fallback to +# 'chroot'). +#config_opts['isolation'] = 'auto' + +# If you're using isolation='nspawn', then by default networking will be turned +# off for rpmbuild. This helps ensure more reproducible builds. +#config_opts['rpmbuild_networking'] = False +# Additional args for nspawn +# config_opts['nspawn_args'] = ['--capability=cap_ipc_lock'] +## When RPM is build in container then build hostname is set to name of +## container. This sets the build hostname to name of container's host. +## Works only in F25+ chroots +# config_opts['use_container_host_hostname'] = True + +# This works unconditionally by calling sethostname(), however +# variable use_container_host_hostname or %_buildhost macro can override this +# config_opts['hostname'] = 'my.own.hostname' + +# The default package manager is DNF4. The options are: +# - 'dnf4' for DNF4 (Python) https://github.com/rpm-software-management/dnf +# - 'dnf5' for DNF5 (C++) https://github.com/rpm-software-management/dnf5 +# The major version of DNF5 will remain '5' indefinitely, see #1271. +# - 'yum' for YUM https://github.com/rpm-software-management/yum +#config_opts['package_manager'] = 'dnf4' + +# Number of attempts to execute package manager's action, and delay between +# attempts (seconds). This is useful e.g. if the build is done against +# non-reliable mirrors (downloading of metadata failed, package download +# failed, ...). +#config_opts['package_manager_max_attempts'] = 1 +#config_opts['package_manager_attempt_delay'] = 10 + +# Dynamic BuildRequires, available since RPM 4.15 +# config_opts['dynamic_buildrequires'] = True +# config_opts['dynamic_buildrequires_max_loops'] = 10 + +# Allows use of external buildrequires. I.e. when dependencies are installed +# from PyPI, Rubygems... +# config_opts['external_buildrequires'] = False + +# Default maximum number of dev loops in chroot is 12 +# config_opts['dev_loop_count'] = 12 + +# rpmbuild/rpm executable path if you need to use different version that the +#config_opts['rpmbuild_command'] = '/usr/bin/rpmbuild' +#config_opts['rpm_command'] = '/bin/rpm' + +# By default a Yum/DNF update is performed before each rebuild +# config_opts['update_before_build'] = True + +# Sometimes the rpm/yum/dnf ecosystem on the host machine isn't really +# compatible with the rpm/yum/dnf ecosystem in mock chroot (the system we +# build for). Typically when host is yum-based and target system is dnf-based. +# Such scenario may cause an error when preparing the target mock buildroot, or +# even worse an unexpected install transaction that is hard to detect. +# Therefore, with `use_bootstrap=True` default, we first install a minimal +# "bootstrap" chroot that only contains rpm/yum/dnf stack, and from that +# "bootstrap" chroot we initialize the target chroot. +#config_opts['use_bootstrap'] = True + +# The bootstrap chroot is normally installed using a package manager from host, +# but in some cases even this isn't possible (e.g. installing rather new ZSTD +# compressed dnf.rpm package by 'dnf4_install_command' on epel-7 host). In such +# case, you may need to have "bootstrap" chroot pre-populated from a container +# image first (where the package manager stack is already pre-installed, so +# mock doesn't have to). +#config_opts['use_bootstrap_image'] = True +#config_opts['bootstrap_image'] = 'fedora:latest' + +# Mock in a nutshell needs to have the selected config_opts["package_manager"] +# executable in bootstrap, and "builddep" command working. That's why Mock +# automatically installs appropriate packages (e.g. dnf5/dnf5-plugins, if +# package_manager==dnf5). If the config_opts["bootstrap_image"] though points +# to a well-prepared image (no additional packages need to be installed), we +# can set bootstrap_image_ready to True and Mock then avoids installing +# packages into the bootstrap chroot. The initial package manager operation is +# very expensive (downloading metadata, initializing caches, ...) so setting +# this to True significantly speeds the bootstrap-from-bootstrap_image +# preparation. This can also help work-around some preparation issues, see +# https://github.com/rpm-software-management/mock/issues/1088 +# "bootstrap_module_setup_commands" and "bootstrap_chroot_additional_packages" +# options invalidate the effect of this option. +#config_opts['bootstrap_image_ready'] = False + +# If 'use_bootstrap_image' is True, Mock is instructed download the configured +# container image from image registry. This option controls the behavior when +# the image can not be downloaded. When set to False, Mock fails hard. When +# set to True, Mock falls-back to normal bootstrap chroot installation using +# package manager (e.g. using dnf --installroot). +#config_opts['bootstrap_image_fallback'] = True + +# When 'use_bootstrap_image' is True, bootstrap image must be downloaded and it +# may fail. Mock's logic is to retry downloads, using this option you can +# configure how long should Mock keep trying (using exponential algorithm with +# full jitter, see python-backoff docs for more info). +#config_opts['bootstrap_image_keep_getting'] = 120 # seconds + +# Skip the "podman pull" and rely on the image already being in the local cache. +#config_opts["bootstrap_image_skip_pull"] = False + +# If provided, Mock performs a 'podman image inspect --format {{ .Digest }}' +# call and asserts that the downloaded/imported bootstrap_image has expected +# Digest (SHA256 string). +#config_opts["bootstrap_image_assert_digest"] = None + +# anything you specify with 'bootstrap_*' will be copied to bootstrap config +# e.g. config_opts['bootstrap_system_yum_command'] = '/usr/bin/yum-deprecated' will become +# config_opts['system_yum_command'] = '/usr/bin/yum-deprecated' for bootstrap config +# These three are overrided in bootstrap by default as we usually do not want additional packages +# and modules in bootstrap chroot. +#config_opts['bootstrap_chroot_additional_packages'] = [] +#config_opts['bootstrap_module_setup_commands'] = [] + +# if you want mock to automatically run createrepo on the rpms in your +# resultdir. +# config_opts['createrepo_on_rpms'] = False +# config_opts['createrepo_command'] = '/usr/bin/createrepo_c -d -q -x *.src.rpm' + + +# What tar binary should be used by Mock. +#config_opts['tar_binary'] = "/bin/tar" + +# You can configure what is the tar implementation stored on the +# config_opts['tar_binary'] path. Depending on this, Mock will use a different +# set of command-line options for tar commands. Valid options are "gnutar" or +# "bsdtar" (used by root cache and SCM plugin, but also by core Mock for +# unpacking Podman container file-systems with --use-bootstrap-image option). +#config_opts['tar'] = "gnutar" + +# if you want mock to backup the contents of a result dir before clean +# config_opts['backup_on_clean'] = False +# config_opts['backup_base_dir'] = "{{basedir}}/backup" + +# if you want to speed up the package installation and the build process, mock +# can use nosync library to skip fsync and related calls from programs called +# from within mock. It needs nosync library to be installed and for multilib +# target, it requires both architectures of nosync to be present. If these +# conditions aren't met, it won't be enabled +# config_opts['nosync'] = False +# if you cannot or don't want to install both architectures of nosync and still +# want mock to use it, you can force it, but then expect a lot of (generally +# harmless) error messages from ld.so when a 32bit program is executed +# config_opts['nosync_force'] = False + +# By default Mock unshare namespace so it is different from your other application +# in unpriviliged container, this is skipped. We will warn you that running mock +# and some other applications in the same container is not good idea and +# can be security risk. If you are fully aware of this risk or mock is your +# only one application in that container you can disable the warning here. +# config_opts['docker_unshare_warning'] = True + +# Change directory where resulting files (RPMs and build logs) are written. +# Resultdir can contain python-string substitutions for any variable in the chroot config. +# config_opts["resultdir"] = "{{basedir}}/{{root}}/result" + +############################################################################# +# +# plugin related. Below are the defaults. Change to suit your site +# policy. site-defaults.cfg is a good place to do this. +# +# NOTE: Some of the caching options can theoretically affect build +# reproducability. Change with care. +# +# config_opts['plugin_conf']['package_state_enable'] = True +# config_opts['plugin_conf']['package_state_opts'] = {} +# config_opts['plugin_conf']['package_state_opts']['available_pkgs'] = False +# config_opts['plugin_conf']['package_state_opts']['installed_pkgs'] = True +# config_opts['plugin_conf']['ccache_enable'] = False +# config_opts['plugin_conf']['ccache_opts'] = {} +# config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '4G' +# config_opts['plugin_conf']['ccache_opts']['compress'] = None +# config_opts['plugin_conf']['ccache_opts']['dir'] = "{{cache_topdir}}/{{root}}/ccache/u{{chrootuid}}/" +# config_opts['plugin_conf']['ccache_opts']['hashdir'] = True +# config_opts['plugin_conf']['ccache_opts']['show_stats'] = False +# config_opts['plugin_conf']['ccache_opts']['debug'] = False +# config_opts['plugin_conf']['yum_cache_enable'] = True +# config_opts['plugin_conf']['yum_cache_opts'] = {} +# config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] = 30 +# config_opts['plugin_conf']['yum_cache_opts']['max_metadata_age_days'] = 30 +# config_opts['plugin_conf']['yum_cache_opts']['online'] = True +# config_opts['plugin_conf']['root_cache_enable'] = True +# config_opts['plugin_conf']['root_cache_opts'] = {} +# config_opts['plugin_conf']['root_cache_opts']['age_check'] = True +# config_opts['plugin_conf']['root_cache_opts']['max_age_days'] = 15 +# config_opts['plugin_conf']['root_cache_opts']['dir'] = "{{cache_topdir}}/{{root}}/root_cache/" +# config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "pigz" +## decompress_program is needed only for bsdtar, otherwise `compress_program` with `-d` is used +## for bsdtar use "unpigz" or "gunzip" +# config_opts['plugin_conf']['root_cache_opts']['decompress_program'] = "pigz" +# config_opts['plugin_conf']['root_cache_opts']['extension'] = ".gz" +# config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] = ["./proc", "./sys", "./dev", +# "./var/tmp/ccache", "./var/cache/yum", +# "./var/cache/dnf", "./var/log" ] +# config_opts['plugin_conf']['hw_info_enable'] = True +# config_opts['plugin_conf']['hw_info_opts'] = {} +# +# config_opts['plugin_conf']['procenv_enable'] = False +# config_opts['plugin_conf']['procenv_opts'] = {} +# +# config_opts['plugin_conf']['showrc'] = False +# config_opts['plugin_conf']['showrc'] = {} +# +# +# bind mount plugin is enabled by default but has no configured directories to +# mount +# config_opts['plugin_conf']['bind_mount_enable'] = True +# config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path', '/bind/mount/path/in/chroot/' )) +# +# config_opts['plugin_conf']['tmpfs_enable'] = False +# config_opts['plugin_conf']['tmpfs_opts'] = {} +# config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 +# config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '768m' +# config_opts['plugin_conf']['tmpfs_opts']['mode'] = '0755' +# config_opts['plugin_conf']['tmpfs_opts']['keep_mounted'] = False +# +# https://rpm-software-management.github.io/mock/Plugin-ChrootScan +# config_opts['plugin_conf']['chroot_scan_enable'] = False +# config_opts['plugin_conf']['chroot_scan_opts'] = { +## Regexp of files which should be copied from buildroot to resultdir. +# 'regexes': [ "^[^k]?core(\.\d+)?", "\.log$",], +## If set to True files are copied only if build failed. +# 'only_failed': True, +## If set to True, tarball is created instead of directory. +# 'write_tar': False, +#} +# +# lvm_root plugin is not enabled by default and is distributed in separate +# subpackage mock-lvm. If you want to use it, it's recommended to disable the +# root_cache plugin, otherwise you'd be caching twice. +# config_opts['plugin_conf']['lvm_root_enable'] = False +# config_opts['plugin_conf']['lvm_root_opts'] = {} +# You need to give it a volume group with sufficient space. It won't touch any +# of the existing logical volumes, so you can use the same volume group you use +# for other purposes. It requires a name of the VG (not device path). +# config_opts['plugin_conf']['lvm_root_opts']['volume_group'] = 'my_vg' +# You need to specify the size that will mock's thinpool occupy. For regular +# packages with small set of dependencies, 2G should suffice. For large packages +# such as libreoffice, you should set it to at least 5 GB otherwise you may run +# out of space and the build process will be blocked +# config_opts['plugin_conf']['lvm_root_opts']['size'] = '2G' +# You can specify alternative pool metadata size, format is the same as size. +# Default value is determined by lvcreate based on size +# config_opts['plugin_conf']['lvm_root_opts']['poolmetadatasize'] = None +# When thin pool utilization is over 90% mock will refuse to continue. +# Because once it reach 100% utilization weird things will happens. +# config_opts['plugin_conf']['lvm_root_opts']['check_size'] = True +# Specifying whether the buildroot volume should stay mounted after mock exits. +# config_opts['plugin_conf']['lvm_root_opts']['umount_root'] = False +# Filesystem name that will be used for the volume. It will use mkfs.$filesystem binary to create it. +# config_opts['plugin_conf']['lvm_root_opts']['filesystem'] = "ext4" +# The whole command for creating the filesystem that will get the volume path as an argument. When set, overrides above +# option. +# config_opts['plugin_conf']['lvm_root_opts']['mkfs_command'] = None +# Additional arguments passed to mkfs command. +# config_opts['plugin_conf']['lvm_root_opts']['mkfs_args'] = [] +# Will be passed to -o option of mount when mounting the volume. String or None. +# config_opts['plugin_conf']['lvm_root_opts']['mount_opts'] = None +# How long to sleep when waiting for concurrent LVM initialization. +# config_opts['plugin_conf']['lvm_root_opts']['sleep_time'] = 1 +# +# overlayfs plugin +# It is recomended to disable root_cache plugin, when overlayfs plugin +# is enabled since overlayfs plugin implicitly creates postinit snapshot +# ( similary to lvm_root plugin), which makes root cache pointless. +# ( Recomended with: config_opts['plugin_conf']['root_cache_enable'] = False ) +# config_opts['plugin_conf']['overlayfs_enable'] = False +# config_opts['plugin_conf']['overlayfs_opts']['base_dir'] = /some/directory +# config_opts['plugin_conf']['overlayfs_opts']['touch_rpmdb'] = False + +### pm_request plugin can install packages requested from within the buildroot +# It is disabled by default, as it affects build reproducibility. It can be enabled +# by setting the following to True, but it's not advised to have it enabled globally. +# It's better to enable it per build by using --enable-plugin pm_request +# config_opts['plugin_conf']['pm_request_enable'] = False +# config_opts['plugin_conf']['pm_request_opts'] = {} + +### If you want to compress mock logs, enable this plugin +# config_opts['plugin_conf']['compress_logs_enable'] = False +### Command used to compress logs - e.g. "/usr/bin/xz -9 --force" +# config_opts['plugin_conf']['compress_logs_opts']['command'] = "gzip" + +# Configuration options for the sign plugin: +# config_opts['plugin_conf']['sign_enable'] = False +# config_opts['plugin_conf']['sign_opts'] = {} +# config_opts['plugin_conf']['sign_opts']['cmd'] = 'rpmsign' +# The options to pass to the signing command. %(rpms)s will be expanded to +# the rpms in the results folder. +# config_opts['plugin_conf']['sign_opts']['opts'] = '--addsign %(rpms)s -D "%%_gpg_name your_name" -D "%%_gpg_path /home/your_name/.gnupg"' + +# Enable preprocessing step before srpm build by using rpkg utilities +# config_opts['plugin_conf']['rpkg_preprocessor_enable'] = False +# config_opts['plugin_conf']['rpkg_preprocessor_opts']['requires'] = ['preproc-rpmspec'] +# config_opts['plugin_conf']['rpkg_preprocessor_opts']['cmd'] = '/usr/bin/preproc-rpmspec %(source_spec)s --output %(target_spec)s' + +# The rpmautospec plugin is disabled by default and distributed in the separate +# subpackage mock-rpmautospec. +# config_opts['plugin_conf']['rpmautospec_enable'] = True +# config_opts['plugin_conf']['rpmautospec_opts'] = { +# 'requires': ['rpmautospec'], +# 'cmd_base': ['/usr/bin/rpmautospec', 'process-distgit'], +# } + +############################################################################# +# +# environment for chroot +# +# config_opts['environment']['TERM'] = 'vt100' +# config_opts['environment']['SHELL'] = '/bin/bash' +# config_opts['environment']['HOME'] = '/builddir' +# config_opts['environment']['HOSTNAME'] = 'mock' +# config_opts['environment']['PATH'] = '/usr/bin:/bin:/usr/sbin:/sbin' +# config_opts['environment']['PROMPT_COMMAND'] = r'printf "\033]0;\007"' +# config_opts['environment']['PS1'] = r' \s-\v\$ ' +# config_opts['environment']['LANG'] = 'C.UTF-8' +# +## other example for PS1 +# config_opts['environment']['PS1'] = r'[\u@\h/\w]\[\033[01;31m\]${?/#0/}\[\033[00m\]\$' +# feel free to craft your own at: http://bashrcgenerator.com/ + +############################################################################# +# +# Things that you can change, but we dont recommend it: +# +# config_opts['chroothome'] = '/builddir' +# config_opts['clean'] = True +## you could not really use substitution here so it will not work if overridden: +# config['rootdir'] = '/var/lib/mock//root/' +## This works in F25+ chroots. This overrides 'use_container_host_hostname' option +# config_opts['macros']['%_buildhost'] = 'my.own.hostname' +# +# Each Mock run has a unique UUID +#config_opts["mock_run_uuid"] = str(uuid.uuid4()) +# +# These OCI buildroot related options are set&used automatically by +# --buildroot-image option logic. The semantics are similar to the *bootstrap* +# counterparts above, e.g., see `config_opts['bootstrap_image']`. +# +# Use OCI image for build chroot initialization. Requires 'buildroot_image' to be set. +#config_opts['use_buildroot_image'] = False +# Initialize buildroot from this OCI image (image reference). +#config_opts['buildroot_image'] = None +# Mock normally tries to pull up2date buildroot image. Set to True if +# you want to use the local image. +#config_opts['buildroot_image_skip_pull'] = False +# No need to intsall any package into the buildroot extracted from an OCI +# image. TODO: not implemented. +#config_opts['buildroot_image_ready'] = False +# If the 'buildroot_image' above can not be used for any reason, fallback to a +# normal DNF installation. If set to False, it leads to hard failure. +#config_opts['buildroot_image_fallback'] = False +# Keep trying 'podman pull' for at most 120s. +#config_opts['buildroot_image_keep_getting'] = 120 +# If set, mock compares the OCI image digest with the one specified here. +#config_opts['buildroot_image_assert_digest'] = None + +############################################################################# +# +# Things that must be adjusted if SCM integration is used: +# +# config_opts['scm'] = True +# config_opts['scm_opts']['method'] = 'git' +# config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN SCM_PKG' +# if there is "--recurse-submodules" or "--recursive" then git submodules are checkout too +# config_opts['scm_opts']['git_get'] = 'git clone SCM_BRN git://localhost/SCM_PKG.git SCM_PKG' +# config_opts['scm_opts']['svn_get'] = 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG' +# config_opts['scm_opts']['distgit_get'] = 'rpkg clone -a --branch SCM_BRN SCM_PKG SCM_PKG' +# config_opts['scm_opts']['distgit_src_get'] = 'rpkg sources' +# config_opts['scm_opts']['spec'] = 'SCM_PKG.spec' +# config_opts['scm_opts']['int_src_dir'] = None +# config_opts['scm_opts']['ext_src_dir'] = '/dev/null' +# config_opts['scm_opts']['write_tar'] = True +# config_opts['scm_opts']['git_timestamps'] = True +# config_opts['scm_opts']['exclude_vcs'] = True + +# These options are also recognized but usually defined in cmd line +# with --scm-option package= --scm-option branch= +# config_opts['scm_opts']['package'] = 'mypkg' +# config_opts['scm_opts']['branch'] = 'main' + +############################################################################# +# +# Things that are best suited for individual chroot config files: +# +# MUST SET (in individual chroot cfg file): +# config_opts['root'] = 'name-of-yum-build-dir' +# Mock will set architecture to 'target_arch' using personality(2) syscall. +# config_opts['target_arch'] = 'i386' +# When host system architecture is not in 'legal_host_arches' list, mock will refuse to switch to +# 'target_arch' and will raise error. +# config_opts['legal_host_arches'] = ('i386', 'i586', 'i686', 'x86_64') +# Contains content of $CHROOT/etc/yum/yum.conf or $CHROOT/etc/dnf/dnf.conf. If DNF is used and +# 'dnf.conf' is not set, then content of 'yum.conf' is used to populate $CHROOT/etc/dnf/dnf.conf +# and vice versa. But only one of those two can be specified. +# config_opts['yum.conf'] = '' +# or +# config_opts['dnf.conf'] = '' +# Important! You must use 'assumeyes=1' in yum/dnf.conf otherwise Mock will fail. +# +# This is used by DNF and can be used in dnf.conf as $key and will be replaced by its value +# config_opts['dnf_vars'] = { 'key': 'value', 'key2': 'value2' } +# +# Flip this if you want to get rid of warning message on systems which do not +# support the desired package manager (e.g. when only Yum is available on host, +# but the chosen buildroot expects to be installed via Dnf). +# Warning! Setting this to False will automatically use Yum on RHEL{6,7} platforms. +# config_opts['dnf_warning'] = True +# +# CAN SET, defaults usually work ok: +# config_opts['chroot_setup_cmd'] = 'install @buildsys-build' +# @buildsys-build is comps group defined by Fedora +# for chroot_setup_cmd we actually need those packages: +# * rpm-build - mock needs /usr/bin/rpmbuild +# * glibc-minimal-langpack - this is optional, but helps to avoid +# installation of huge glibc-all-langpacks. +# config_opts['chroot_additional_packages'] = [] +# config_opts['log_config_file'] = 'logging.ini' +# config_opts['more_buildreqs']['srpm_name-version-release'] = 'dependency' +# config_opts['more_buildreqs']['srpm_name-version-release'] = ['dependency1', 'dependency2'] +# config_opts['macros']['%Add_your_macro_name_here'] = "add macro value here" +# config_opts['files']['path/name/no/leading/slash'] = "put file contents here." +# config_opts['chrootuid'] = os.getuid() +# config_opts['releasever'] = '20' + + +# Configuration options related to specific package managers. +# +# You can configure how DNF (and other packager managers) will be used with +# Mock. Each option is specific to a concrete package manager, and then it +# is appropriately prefixed by , namely `dnf5_`, `dnf4_`, `yum_` or +# `microdnf_`. Mock attempts to use appropriate package manager depending on +# the `package_manager` configuration option. +# +# Currently there are several options to set (override only if you need to +# use something non-standard). +# +# '_command' (e.g. dnf5_command) - the package manager executable +# path, searched both on host (to install bootstrap chroot) or in +# bootstrap chroot (to install buildroot) +# +# '_system_command' (e.g. yum_system_command) - the package manager +# executable searched on host only in case of _command not found. +# This may be useful for "bootstrap=off" scenarios with the package +# manager installed in a custom location. +# +# '_common_opts' (e.g. dnf5_common_opts) - options that are passed to +# every single command execution of the '_command' by Mock, minus +# '_avoid_opts', see below +# +# '_avoid_opts" (e.g. dnf5_common_opts) - a dictionary where key +# matches a sub-command of the '_command', and value is a list[] +# of options that are going to be filtered out (from _common_opts) and +# newer used for a particular sub-command. +# +# '_install_command' (e.g. dnf5_install_command) - when +# 'use_bootstrap' is enabled, these commands are used to install the +# desired package manager (providing the '_command' file) into +# the "bootstrap" chroot; for installing it, Mock first tries to find +# '_command' on host, and then if not found there is a +# pre-defined fallback so Mock tries other host's managers (dn5 +# => dnf => yum, etc.). But installing the bootstrap chroot might not +# be an easy task in some cases (e.g. installing modern Fedora bootstrap +# where packages are Zstd-compressed with the old Yum from RHEL 7). You +# might want to use 'use_bootstrap_image' instead (then bootstrap chroot +# is just "downloaded", not installed). +# +# '_disable_plugins (e.g. dnf4_disable_plugins) - list of package +# manager plugins that should be always disabled when Mock executes the +# '_command' (e.g. versionlock, with Mock we always want to build +# against the latest package versions available in buildroot) +# +# '_builddep_command (e.g. yum_builddep_command) - normally, +# the '_command builddep' command is used by default to install +# build dependencies. There's no 'yum builddep' (sub)command though, +# but /bin/yum-builddep. Use this option to override the default. +# +# '_builddep_opts (e.g. dnf5_builddep_opts) - list of additional +# options to always use with the builddep command + +# DNF4 specific options (Fedora <= 39, EL 8, 9 and 10) +# +# Don't be confused by the '-3' suffix in the DNF4 script. This is a +# historical artifact from when it was necessary to differentiate between +# 'Python 3' and 'Python 2' scripts. +#config_opts['dnf4_command'] = '/usr/bin/dnf-3' +#config_opts['dnf4_system_command'] = '/usr/bin/dnf' +#config_opts['dnf4_common_opts'] = ['--setopt=deltarpm=False', '--setopt=allow_vendor_change=yes', '--allowerasing'] +#config_opts['dnf4_install_command'] = 'install python3-dnf python3-dnf-plugins-core' +#config_opts['dnf4_disable_plugins'] = ['local', 'spacewalk', 'versionlock'] +#config_opts['dnf4_builddep_opts'] = [] + +# DNF5 specific options (Fedora 40+ https://fedoraproject.org/wiki/Changes/BuildWithDNF5) +# +#config_opts['dnf5_command'] = '/usr/bin/dnf5' +#config_opts['dnf5_system_command'] = '/usr/bin/dnf5' +#config_opts['dnf5_common_opts'] = ['--setopt=deltarpm=False', '--setopt=allow_vendor_change=yes', '--allowerasing'] +#config_opts['dnf5_install_command'] = 'install dnf5 dnf5-plugins' +#config_opts['dnf5_disable_plugins'] = [] +# DF5 sub-command 'builddep' doesn't support the '--allowerasing' option: +# https://github.com/rpm-software-management/dnf5/issues/461 +#config_opts["dnf5_avoid_opts"] = {"builddep": ["--allowerasing"]} + +# YUM (for RHEL7 and older) - https://github.com/rpm-software-management/yum +# +#config_opts['yum_command'] = '/usr/bin/yum' +#config_opts['yum_system_command'] = '/usr/bin/yum' +#config_opts['yum_common_opts'] = [] +#config_opts['yum_install_command'] = 'install yum yum-utils' +#config_opts['yum_builddep_command'] = '/usr/bin/yum-builddep' +#config_opts['yum_builddep_opts'] = [] + +# microdnf (limited DNF functionality for containers) - https://github.com/rpm-software-management/microdnf +## "dnf-install" is special keyword which tells mock to use install but with DNF +#config_opts['microdnf_command'] = '/usr/bin/microdnf' +#config_opts['microdnf_common_opts'] = [] +#config_opts['microdnf_install_command'] = 'dnf-install microdnf dnf dnf-plugins-core' +#config_opts['microdnf_builddep_command'] = '/usr/bin/dnf' +#config_opts['microdnf_builddep_opts'] = [] + +# config_opts['priorities.conf'] = 'put file contents here.' +# config_opts['rhnplugin.conf'] = 'put file contents here.' +## Important! You should register your host machine first! +# config_opts['subscription-manager.conf'] = 'put file contents here.' +## This will only work with DNF and when repo is configured with modules=1 for repo in dnf.conf. + +# List of module commands to be executed when initializing chroot, before +# `chroot_setup_cmd`. Each command is a pair like `(action, module_specs)` +# where `module_specs` is a comma-separated list of module specifications. +# The commands are executed in order they are configured here, and each +# `action` can be executed multiple times. +# +## Artificial example: (a) Disable any potentially enabled postgresql module +## stream, (b) enable _specific_ postgresql and ruby module streams, +## (c) install the development nodejs profile and (d) disable it immediately. +#config_opts['module_setup_commands'] = [ +# ('disable', 'postgresql'), +# ('enable', 'postgresql:12, ruby:2.6'), +# ('install', 'nodejs:13/development'), +# ('disable', 'nodejs'), +#] + +## Use this to force foreing architecture (requires qemu-user-static) +# config_opts['forcearch'] = None +## mapping from target_arch (or forcearch) to arch in /usr/bin/qemu-*-static +# config_opts['qemu_user_static_mapping'] = { +# 'aarch64': 'aarch64', +# 'armv7hl': 'arm', +# 'i386': 'i386', +# 'i686': 'i386', +# 'ppc64': 'ppc64', +# 'ppc64le': 'ppc64le', +# 's390x': 's390x', +# 'x86_64': 'x86_64', +# } +# +## Emulating architecture results in slower builds. Expose it to packagers +## This is set to 1 normally. And to 10 when forcearch is in play. +# config_opts['macros']['%_platform_multiplier'] = 1 +# +# If you change chrootgid, you must also change "mock" to the correct group +# name in this line of the mock PAM config: +# auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet +# config_opts['chrootgid'] = grp.getgrnam("mock")[2] +# name of user that is used when executing commands inside the chroot +# config_opts['chrootuser'] = 'mockbuild' +# name of the group inside of chroot +# config_opts['chrootgroup'] = 'mock' + +# Security related +# config_opts['no_root_shells'] = False +# +# Proxy settings (https_proxy, ftp_proxy, and no_proxy can also be set) +# You can set a specific proxy: 'http://localhost:3128' +# But by default, the existing environment variables are re-used +# config_opts['http_proxy'] = os.getenv("http_proxy") +# config_opts['ftp_proxy'] = os.getenv("ftp_proxy") +# config_opts['https_proxy'] = os.getenv("https_proxy") +# config_opts['no_proxy'] = os.getenv("no_proxy") + +# +# Extra dirs to be created when the chroot is initialized +# This is just a list of strings representing chroot paths such as: +# [ '/run/lock', ] +# config_opts['extra_chroot_dirs'] = [] +# +# Set timeout in seconds for common mock operations +# if 0 is set, then no time limit is used +# config_opts['opstimeout'] = 0 + +# Copy host's ca-trust directories into the specified locations inside the +# chroot. Each item in the list is a pair of (host, chroot) paths for the +# directories to be copied, since some hosts and some destination chroots +# may use different paths. The directories are copied recursively. +#config_opts['ssl_copied_ca_trust_dirs'] = None +# Example: +#config_opts['ssl_copied_ca_trust_dirs'] = [ +# ('/etc/pki/ca-trust', '/etc/pki/ca-trust'), +# ('/usr/share/pki/ca-trust-source', '/usr/share/pki/ca-trust-source') +#] + +# Copy host's SSL certificate bundle ('/etc/pki/tls/certs/ca-bundle.crt') into +# specified location inside chroot. This usually isn't needed because we copy +# the whole /etc/pki/ca-trust/extracted directory recursively by default, and +# Fedora or EL systems work with that. But some destination chroots can have +# different configuration, and copying the bundle helps. +#config_opts['ssl_ca_bundle_path'] = None + +# Copy host's SSL certificates into a specified location inside the chroot if +# mock needs access to repositories which require client certificate +# authentication. Specify the full path to the public certificate on the host +# and the destination directory in the chroot. Do the same for the private key. +# The private key should not be password-protected if you want mock to run +# unattended. +#config_opts['ssl_extra_certs'] = None +# Example: +#config_opts['ssl_extra_certs'] = ['/etc/pki/tls/certs/client.crt', '/etc/pki/tls/certs/', +# '/etc/pki/tls/private/client_nopass.key.crt', '/etc/pki/tls/private/'] + +# user_agent string to identify HTTP request to repositories +# config_opts['user_agent'] = "Mock ({{ root }}; {{ target_arch }})" + +# Seccomp (Linux kernel security facility) isn't utilized by Mock, but is +# utilized by systemd-nspawn and Podman (both tools are used by Mock in +# multiple places). The list of seccomp rules (syscall allow-lists) maintained +# in those tools is often different across distributions or even versions. +# Because Mock does cross-distribution builds, "host" distro rules are not +# often applicable on the "target" distribution. To not complicate things, and +# because by design Mock doesn't have to fully isolate, we disable seccomp for +# those containerization tools by default. +#config_opts["seccomp"] = False + +# Mock code can go into a relatively deep recursion (e.g. when doing chroot +# cleanup via the recursive rmtree() calls). Use this config option to +# override the default Mock's stack call limit (5000). +#config_opts["recursion_limit"] = 5000 + +# Mock internals used by the --calculated-build-dependencies and +# --hermetic-build options. Please do not set these options in Mock +# configuration files. +# config_opts["calculatedeps"] = None +# config_opts["hermetic_build"] = False + +# List of usernames (strings) that will be pre-created in buildroot. The UID +# and GID in-chroot is going to be the same as on-host. This option is for +# example useful for the 'pesign' use-cases that both (a) bind-mount +# pesign-owned socket-files into the chroot and (b) install the +# 'BuildRequires: pesign' package which would overwrite the ownership of the +# socket file. See https://github.com/rpm-software-management/mock/issues/1091 +#config_opts["copy_host_users"] = [] + +# Whether to use host's shadow-utils to provision users and groups in the +# buildroot, which we normally want to do because host shadow-utils are +# newer and more flexible than buildroot ones. However, there is an issue in shadow-utils +# where even using the --prefix (or, even --root if we did it that way) option, the host +# config will "leak" into the chroot. This is not an issue if the configs are +# effectively the same between host and buildroot, but will cause problems if, for +# example, the host is configured to use FreeIPA-provided subids. +# See https://github.com/shadow-maint/shadow/issues/897 +# config_opts["use_host_shadow_utils"] = True + +# The `repo_arch` (read-only) option simplifies DNF configuration with Mock for +# non-trivial architecture selection decisions. Typically, we want to use the +# DNF-native `$basearch` variable to instruct DNF to use the appropriate RPM +# architecture for a given Mock config (for cross-arch builds, bootstrap uses a +# different architecture than the target chroot!). However, `$basearch` often +# doesn't work correctly — some distributions do not align the mirror URLs with +# the `$basearch` content (as known by DNF), causing problems with +# cross-distro/cross-architecture builds. The `repo_arch` internal is then +# exported as a `{{ repo_arch }}` Jinja2 placeholder, aiming to help with this +# problem. Simply replace `$basearch` with `{{ repo_arch }}` in your config. +# +# The `repo_arch` thing is not really an "option" but rather a Mock internal +# exported for read-only use-cases. However, the `repo_arch_map` dictionary +# can be used to affect Mock's background decisions. For example, when +# the configuration claims `target_arch=armv7hnl`, but the repo URLs look like +# 'example.com/arm32/', one can use +# `baseurl=example.com/{{ repo_arch }}/` instead of +# `baseurl=example.com/$basearch/`, together with +# `config_opts["repo_arch_map"] = {"armv7hnl": "arm32"}`. +# In such a case, builds on `x86_64` hosts will expand to `example.com/x86_64` +# URL for the bootstrap (native) chroot installation, but also to +# `example.com/arm32` for the target (cross-arch, emulated) chroot +# installation. +#config_opts["repo_arch"] = "Mock internal, e.g. 'x86_64'" +#config_opts["repo_arch_map"] = {} diff --git a/mock/etc/bash_completion.d/mock b/mock/etc/bash_completion.d/mock new file mode 100644 index 0000000..8de539c --- /dev/null +++ b/mock/etc/bash_completion.d/mock @@ -0,0 +1,213 @@ +# bash >= 3 completion for mock(1) + +_mock_root() +{ + test $# -eq 0 && set -- /etc/mock "$HOME/.config/mock" + + local suggest=$( + shopt -s nullglob + for dir in "$@"; do + # work with absolute paths! + cd "$dir" &>/dev/null || continue + for file in *; do + case $file in + site-defaults*) ;; # skip site defaults + *' '*) ;; # skip files with white spaces + *.cfg) echo "${file%%.cfg}" ;; + esac + done + done + ) + + COMPREPLY+=( $( compgen -W "$suggest" -- "$cur" ) ) + _filedir 'cfg' +} + +_mock_isopt() +{ + [[ ${1:0:1} = - ]] && return 0 + case $1 in + install|remove) + return 0 + ;; + esac + return 1 +} + +_mock_isopt_multiarg() { + local the_option=$1 + _mock_isopt "$the_option" || return 1 + case $the_option in + --*) + the_option=${the_option##--} + ;; + esac + case $the_option in + rebuild|install|installdeps|remove|chain|update|copyin|copyout|pm-cmd|yum-cmd|dnf-cmd|chroot|shell) + return 0 + ;; + esac + return 1 +} + +_mock_parse_help() +{ + # PRECOMPILED_PARSED_MOCK_HELP + _parse_help "$1" +} + +_mock() +{ + local cur prev words cword split + _init_completion -s || return + + local cfgdirs=( /etc/mock "$HOME/.config/mock" ) + local count=0 + local greedyopt=rebuild # the default mode eating srpms + local prevopt=rebuild + + for word in "${words[@]}" ; do + if [[ $count -eq $cword ]] ; then + # If the last (i.e. current) argument is an option, clear prevopt so that we complete + # the current argument as an option instead an argument to prevopt + _mock_isopt "$word" && prevopt= + break + fi + # Record the option argument previous to the current argument to determine the type of + # completion that is needed + if _mock_isopt_multiarg "$word"; then + prevopt=$word + # last greedy option wins + greedyopt=$word + elif _mock_isopt "$word"; then + prevopt=$word + else + # Revert back to the last greedy option. E.g. for 'mock -r fedora-rawhide-x86_64 ', + # we want to react on the default 'rebuild' mode, not `-r`. + prevopt=$greedyopt + fi + + if [[ "$word" == --configdir ]] ; then + cfgdirs=( "${words[((count+1))]}" ) + elif [[ "$word" == --configdir=* ]] ; then + cfgdirs=( ${word/*=/} ) + fi + count=$((++count)) + done + + case "$prevopt" in + -h|--help|--version) + # no further arguments are accepted after the above arguments + return + ;; + --arch|--config-opts|-D|--define|--disablerepo|--enablerepo|--forcearch|--plugin-option|\ + --rpmbuild-opts|--rpmbuild_timeout|--scm-option|--uniqueext|--with|--without) + # argument required but no completions available + return + ;; + -r|--root|--chain) + _mock_root "${cfgdirs[@]}" + return + ;; + --configdir|--cwd|--resultdir|--rootdir) + _filedir -d + return + ;; + --copyin|--copyout|--macro-file|--sources) + _filedir + return + ;; + --spec) + _filedir 'spec' + return + ;; + --target) + # Yep, compatible archs, not compatible build archs + # (e.g. ix86 chroot builds in x86_64 mock host) + # This would actually depend on what the target root + # can be used to build for... + COMPREPLY=( $( compgen -W "$( command rpm --showrc | \ + sed -ne 's/^\s*compatible\s\s*archs\s*:\s*\(.*\)/\1/i p' )" \ + -- "$cur" ) ) + return + ;; + --enable-plugin|--disable-plugin) + COMPREPLY=( $( compgen -W "$( $1 $prev=DOES_NOT_EXIST 2>&1 | \ + sed -ne "s/[',]//g" -e 's/.*[[(]\([^])]*\)[])]/\1/p' )" \ + -- "$cur" ) ) #' unconfuse emacs + return + ;; + --scrub) + COMPREPLY=( $( compgen -W "all chroot cache root-cache c-cache + yum-cache dnf-cache lvm overlayfs bootstrap" -- "$cur" ) ) + return + ;; + -i|--install|install) + _filedir 'rpm' + COMPREPLY=( $( compgen -W '"${COMPREPLY[@]}"' -X '*.src.rpm' ) ) + COMPREPLY=( $( compgen -W '"${COMPREPLY[@]}"' -X '*.nosrc.rpm' ) ) + [[ $cur != */* && $cur != [.~]* ]] && \ + declare -F _yum_list &>/dev/null && _yum_list all "$cur" + return + ;; + --isolation) + COMPREPLY=( $( compgen -W "auto simple nspawn" -- "$cur" ) ) + return + ;; + --remove|remove) + declare -F _yum_list &>/dev/null && _yum_list all "$cur" + return + ;; + --short-circuit) + COMPREPLY=( $( compgen -W "install binary build prep" -- "$cur" ) ) + return + ;; + esac + + $split && return + + if [[ "$cur" == -* ]] ; then + COMPREPLY=( $( compgen -W '$( _mock_parse_help "$1" )' -- "$cur" ) ) + # _parse_help fails to pick up --define (it's a parsing failure due to + # the quoted 'MACRO EXPR' argument) + COMPREPLY+=( $( compgen -W '--define=' -- "$cur" ) ) + [[ $COMPREPLY == *= ]] && compopt -o nospace + else + _filedir '@(?(no)src.r|s)pm' + fi + +} && +complete -F _mock mock mock.py + +_mock_parse_buildlog() +{ + local cur prev cword split + _init_completion -s || return + + case "$prev" in + -h|--help) + # no further arguments are accepted after the above arguments + return + ;; + -p|--path) + _filedir + return + ;; + esac + + $split && return + + if [[ $cword -eq 1 ]] ; then + COMPREPLY=( $( compgen -W '$( _parse_help "$1" )' -- "$cur" ) ) + [[ $COMPREPLY == *= ]] && compopt -o nospace + fi +} && +complete -F _mock_parse_buildlog mock-parse-buildlog mock-parse-buildlog.py + +# Local variables: +# mode: shell-script +# sh-basic-offset: 4 +# sh-indent-comment: t +# indent-tabs-mode: nil +# End: +# ex: ts=4 sw=4 et filetype=sh diff --git a/mock/etc/consolehelper/mock b/mock/etc/consolehelper/mock new file mode 100644 index 0000000..b24d0a0 --- /dev/null +++ b/mock/etc/consolehelper/mock @@ -0,0 +1,6 @@ +USER=root +PROGRAM=/usr/libexec/mock/mock +SESSION=false +FALLBACK=false +KEEP_ENV_VARS=COLUMNS,SSH_AUTH_SOCK,http_proxy,ftp_proxy,https_proxy,no_proxy,MOCK_TRACE_LOG +BANNER=You are not in the `mock` group. See https://rpm-software-management.github.io/mock/#setup diff --git a/mock/etc/mock/hermetic-build.cfg b/mock/etc/mock/hermetic-build.cfg new file mode 100644 index 0000000..79416de --- /dev/null +++ b/mock/etc/mock/hermetic-build.cfg @@ -0,0 +1,38 @@ +# used by https://rpm-software-management.github.io/mock/feature-hermetic-builds + +config_opts['root'] = 'hermetic-build' +config_opts['description'] = 'Configuration file for Hermetic Builds' + +# Hermetic build configuration file is re-used for multiple chroot +# configurations (particular chroot used depends on the previous +# --calculate-build-dependencies run). That's why Mock automatically runs +# --scrub=all with --hermetic-build. It doesn't make sense to waste the time +# with creating caches. +config_opts['plugin_conf']['root_cache_enable'] = False + +config_opts['dnf.conf'] = """ +[main] +keepcache=1 +system_cachedir=/var/cache/dnf +debuglevel=2 +reposdir=/dev/null +logfile=/var/log/yum.log +retries=20 +obsoletes=1 +gpgcheck=0 +assumeyes=1 +syslog_ident=mock +syslog_device= +install_weak_deps=0 +metadata_expire=0 +best=1 +protected_packages= + +# repos + +[offline] +name=offline repo +baseurl=file://{{ offline_local_repository }} +enabled=True +skip_if_unavailable=False +""" diff --git a/mock/etc/mock/logging.ini b/mock/etc/mock/logging.ini new file mode 100644 index 0000000..8186ead --- /dev/null +++ b/mock/etc/mock/logging.ini @@ -0,0 +1,84 @@ +[formatters] +keys: detailed,simple,unadorned,state + +[handlers] +keys: simple_console,detailed_console,unadorned_console,simple_console_warnings_only + +[loggers] +keys: root,build,state,mockbuild + +[formatter_state] +format: %(asctime)s - %(message)s + +[formatter_unadorned] +format: %(message)s + +[formatter_simple] +format: %(levelname)s: %(message)s + +;useful for debugging: +[formatter_detailed] +format: %(levelname)s %(filename)s:%(lineno)d: %(message)s + +[handler_unadorned_console] +class: StreamHandler +args: [] +formatter: unadorned +level: INFO + +[handler_simple_console] +class: StreamHandler +args: [] +formatter: simple +level: INFO + +[handler_simple_console_warnings_only] +class: StreamHandler +args: [] +formatter: simple +level: WARNING + +[handler_detailed_console] +class: StreamHandler +args: [] +formatter: detailed +level: WARNING + +; usually dont want to set a level for loggers +; this way all handlers get all messages, and messages can be filtered +; at the handler level +; +; all these loggers default to a console output handler +; +[logger_root] +level: NOTSET +handlers: simple_console + +; mockbuild logger normally has no output +; catches stuff like mockbuild.trace_decorator and mockbuild.util +; dont normally want to propagate to root logger, either +[logger_mockbuild] +level: NOTSET +handlers: +qualname: mockbuild +propagate: 1 + +[logger_state] +level: NOTSET +; unadorned_console only outputs INFO or above +handlers: unadorned_console +qualname: mockbuild.Root.state +propagate: 0 + +[logger_build] +level: NOTSET +handlers: simple_console_warnings_only +qualname: mockbuild.Root.build +propagate: 0 + +; the following is a list mock logger qualnames used within the code: +; +; qualname: mockbuild.util +; qualname: mockbuild.uid +; qualname: mockbuild.trace_decorator + diff --git a/mock/etc/pam/mock b/mock/etc/pam/mock new file mode 100644 index 0000000..4b566af --- /dev/null +++ b/mock/etc/pam/mock @@ -0,0 +1,13 @@ +#%PAM-1.0 +auth sufficient pam_rootok.so +auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet +# Uncomment the following line to implicitly trust users in the "wheel" group. +#auth sufficient pam_wheel.so trust use_uid +# Uncomment the following line to require a user to be in the "wheel" group. +#auth required pam_wheel.so use_uid +auth include system-auth +account sufficient pam_succeed_if.so user ingroup mock use_uid quiet +account include system-auth +password include system-auth +session include system-auth +session optional pam_xauth.so diff --git a/mock/etc/pki/README.txt b/mock/etc/pki/README.txt new file mode 100644 index 0000000..2914220 --- /dev/null +++ b/mock/etc/pki/README.txt @@ -0,0 +1,7 @@ +Any file named + RPM-GPG-KEY-* +is copied into chroot to directory /etc/pki/mock/. + +You can use it for your personal GPG keys. + +All distribution GPG keys are in package distribution-gpg-keys. diff --git a/mock/integration-tests/001-orphanskill-explicit.tst b/mock/integration-tests/001-orphanskill-explicit.tst new file mode 100644 index 0000000..fd0149a --- /dev/null +++ b/mock/integration-tests/001-orphanskill-explicit.tst @@ -0,0 +1,43 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# Test orphanskill feature (explicit) +# +header "Test orphanskill feature (explicit)" + +tmpdir=/var/tmp + +runcmd "$MOCKCMD --offline --init" +runcmd "$MOCKCMD --install gcc" +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --copyin integration-tests/daemontest.c $tmpdir" +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --chroot -- gcc -Wall -o $tmpdir/daemontest $tmpdir/daemontest.c" + +echo "#!/bin/sh +set -x +$tmpdir/daemontest +sleep 60" >> "$CHROOT$tmpdir/try" + +for isolation in simple nspawn; do + for bootstrap in --bootstrap-chroot --no-bootstrap-chroot; do + selector="--isolation=$isolation $bootstrap" + + # the following should launch about three processes in the chroot: bash, + # sleep, daemontest + runcmd "$MOCKCMD $selector --offline --disable-plugin=tmpfs --chroot -- bash $tmpdir/try" & + mockpid=$! + sleep 10 + + ! test -d /proc/$mockpid && die "Mock stopped too early ($selector)." + ! pgrep daemontest && die "Daemontest failed. daemontest should be running now but is not ($selector)." + + runcmd "$MOCKCMD $selector --offline --disable-plugin=tmpfs --orphanskill" + + wait "$mockpid" && "Unexpected success of mock ($selector)." + + pgrep daemontest && die "Daemontest FAILED. found a daemontest process running after exit ($selector)." + done +done + +exit 0 diff --git a/mock/integration-tests/002-copyin.tst b/mock/integration-tests/002-copyin.tst new file mode 100644 index 0000000..e20cb36 --- /dev/null +++ b/mock/integration-tests/002-copyin.tst @@ -0,0 +1,73 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# Test --copyin +# + +header "testing --copyin option" +runcmd "$MOCKCMD --offline --copyin DOESNOTEXIST /" +res=$? +if [ $res -ne 50 ]; then + echo "'mock --chroot' return code not properly passed back: $res" + exit 1 +fi + +runcmd "$MOCKCMD --offline --copyin ${TESTDIR}/test-B-1.1-0.src.rpm ${TESTDIR}/test-C-1.1-0.src.rpm /etc/fstab" +res=$? +if [ $res -ne 50 ]; then + echo "'mock --chroot' return code not properly passed back: $res" + exit 1 +fi + + +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --copyin ${TESTDIR}/test-C-1.1-0.src.rpm /" +res=$? +if [ $res -ne 0 ]; then + echo "mock returned fail when should have succeeded!" + exit 1 +fi +if [ ! -e $CHROOT/test-C-1.1-0.src.rpm ]; then + echo "--copyin FAILED. File $CHROOT/test-C-1.1-0.src.rpm not found." + exit 1 +fi + +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --copyin ${TESTDIR}/test-B-1.1-0.src.rpm /test-B-1.1-0.src.rpm" +res=$? +if [ $res -ne 0 ]; then + echo "mock returned fail when should have succeeded!" + exit 1 +fi +if [ ! -e $CHROOT/test-B-1.1-0.src.rpm ]; then + echo "--copyin FAILED. File $CHROOT/test-B-1.1-0.src.rpm not found." + exit 1 +fi + +TMPDIR=$(mktemp -d) +echo foo > ${TMPDIR}/bar +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --copyin ${TMPDIR} /foobar" +res=$? +if [ $res -ne 0 ]; then + echo "mock returned fail when should have succeeded!" + exit 1 +fi +if ! sudo ls -l $CHROOT/foobar/bar; then + echo "--copyin FAILED. File $CHROOT/foobar/bar not found." + exit 1 +fi + +mkdir -p ${TMPDIR}/TMPDIR +echo foo > ${TMPDIR}/TMPDIR/bar +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --copyin ${TMPDIR}/TMPDIR /" +res=$? +if [ $res -ne 0 ]; then + echo "mock returned fail when should have succeeded!" + exit 1 +fi +if ! sudo ls -l $CHROOT/TMPDIR/bar; then + echo "--copyin FAILED. File $CHROOT/TMPDIR/bar not found." + exit 1 +fi + + diff --git a/mock/integration-tests/01-shell.tst b/mock/integration-tests/01-shell.tst new file mode 100644 index 0000000..5377068 --- /dev/null +++ b/mock/integration-tests/01-shell.tst @@ -0,0 +1,14 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# test mock shell (interactive) and return code passing +# +header "testing interactive shell and return code" +echo exit 5 | runcmd "$MOCKCMD --old-chroot --offline --shell" +res=$? +if [ $res -ne 5 ]; then + echo "'mock --chroot' return code not properly passed back: $res" + exit 1 +fi diff --git a/mock/integration-tests/02-argpassing.tst b/mock/integration-tests/02-argpassing.tst new file mode 100644 index 0000000..f6de22e --- /dev/null +++ b/mock/integration-tests/02-argpassing.tst @@ -0,0 +1,14 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# Test that chroot with one arg is getting passed though a shell (via os.system()) +# +header "testing that args are passed correctly to a shell" +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --chroot 'touch /var/tmp/{foo,bar,baz}'" +if [ ! -f $CHROOT/var/tmp/foo ] || [ ! -f $CHROOT/var/tmp/bar ] || [ ! -f $CHROOT/var/tmp/baz ]; then + echo "'mock --chroot' with one argument is not being passed to os.system()" + exit 1 +fi + diff --git a/mock/integration-tests/03-chroot-one-arg.tst b/mock/integration-tests/03-chroot-one-arg.tst new file mode 100644 index 0000000..28c5488 --- /dev/null +++ b/mock/integration-tests/03-chroot-one-arg.tst @@ -0,0 +1,14 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# Test that chroot with more than one arg is not getting passed through a shell +# +header "Test that chroot with more than one arg is not getting passed through a shell" +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --chroot touch '/var/tmp/{quux,wibble}'" +if [ ! -f $CHROOT/var/tmp/\{quux,wibble\} ] || [ -f $CHROOT/var/tmp/quux ] || [ -f $CHROOT/var/tmp/wibble ]; then + echo "'mock --chroot' with more than one argument is being passed to os.system()" + exit 1 +fi + diff --git a/mock/integration-tests/04-offline-tmpfs.tst b/mock/integration-tests/04-offline-tmpfs.tst new file mode 100644 index 0000000..3a72af5 --- /dev/null +++ b/mock/integration-tests/04-offline-tmpfs.tst @@ -0,0 +1,14 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# Test offline build as well as tmpfs +# +header "Test offline build as well as tmpfs" +runcmd "$MOCKCMD --offline --enable-plugin=tmpfs --rebuild $MOCKSRPM" +if [ ! -e $outdir/mock-*.noarch.rpm ]; then + echo "rebuild test FAILED. could not find $outdir/mock-*.noarch.rpm" + exit 1 +fi + diff --git a/mock/integration-tests/05-orphanskill-std.tst b/mock/integration-tests/05-orphanskill-std.tst new file mode 100644 index 0000000..9896574 --- /dev/null +++ b/mock/integration-tests/05-orphanskill-std.tst @@ -0,0 +1,57 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# Test orphanskill feature (std) +# +header "Test orphanskill feature (std)" +if pgrep daemontest; then + echo "Exiting because there is already a daemontest running." + exit 1 +fi + +daemon_package=https://github.com/rpm-software-management/mock-test-data/raw/main/daemontest-1-0.src.rpm + +for isolation in nspawn simple; do + for bootstrap in --bootstrap-chroot --no-bootstrap-chroot; do + selector="--isolation=$isolation $bootstrap" + mock="$MOCKCMD_NO_RESULTDIR $selector" + + tmpdir=/var/tmp + + runcmd "$mock --offline --init" + runcmd "$mock --install gcc" + runcmd "$mock --offline --copyin integration-tests/daemontest.c $tmpdir" + runcmd "$mock --offline --chroot -- gcc -Wall -o $tmpdir/daemontest $tmpdir/daemontest.c" + + runcmd "$mock --offline --chroot -- $tmpdir/daemontest" + pgrep daemontest && die "Daemontest FAILED. found a daemontest process running after exit ($selector)." + + runcmd "$mock --chain $daemon_package" \ + || die "Can't build daemon package ($selector)." + + pgrep daemontest && die "Leftover daemontest process ($selector)." + + runcmd "$mock --chain $daemon_package --postinstall --config-opts=cleanup_on_success=False" \ + || die "Can't build && install the daemon package ($selector)." + + pgrep daemontest && die "Leftover daemontest process after --postinstall ($selector)." + + runcmd "$mock --shell 'set -x; /usr/bin/daemontest ; sleep 60 ;'" & + mockchild=$! + sleep 10 # give mock some time to start the daemontest process + + pgrep daemontest || die "The daemontest process doesn't exit ($selector)." + runcmd "$mock --orphanskill" + + wait "$mockchild" && "Unexpected success of mock ($selector)." + + pgrep daemontest && die "The daemontest should be killed ($selector)." + + # "uninstall" the $rpm from chroot + runcmd "$MOCKCMD --scrub chroot" || die "Can't scrub ($selector).". + done +done + +exit 0 diff --git a/mock/integration-tests/06-retcode.tst b/mock/integration-tests/06-retcode.tst new file mode 100644 index 0000000..2079e1f --- /dev/null +++ b/mock/integration-tests/06-retcode.tst @@ -0,0 +1,39 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# Test that chroot return code is properly passed up +# + +header "testing that chroot return code is passed back correctly" +runcmd "$MOCKCMD --offline --chroot -- bash -c 'exit 5'" +res=$? +if [ $res -ne 5 ]; then + echo "'mock --chroot' return code not properly passed back: $res" + exit 1 +fi + +header "testing exit code on unsupported command line" +runcmd "$MOCKCMD --offline --scm-enable --scm-option method=unsupported" +res=$? +if [ $res -ne 5 ]; then + echo "mock pass exit code $res instead of 5 where there is problem with command line options" + exit 1 +fi + +header "testing that error in root.log is passed back correctly" +runcmd "$MOCKCMD --offline --rebuild ${TESTDIR}/test-E-1.1-0.src.rpm" +res=$? +if [ $res -ne 30 ]; then + echo "mock pass exit code $res instead of 30 where there is problem in root.log" + exit 1 +fi + +header "testing error code when resultdir cannot be created" +runcmd "$MOCKCMD --offline --resultdir=/proc/doesnotwork --rebuild ${TESTDIR}/test-C-1.1-0.src.rpm" +res=$? +if [ $res -ne 70 ]; then + echo "mock pass exit code $res instead of 70 when resultdir cannot be created" + exit 1 +fi diff --git a/mock/integration-tests/07-init-clean.tst b/mock/integration-tests/07-init-clean.tst new file mode 100644 index 0000000..5ef714a --- /dev/null +++ b/mock/integration-tests/07-init-clean.tst @@ -0,0 +1,21 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# test init/clean +# +header "test init/clean" +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --clean" +if [ -e $CHROOT ]; then + echo "clean test FAILED. still found $CHROOT dir." + exit 1 +fi + +runcmd "$MOCKCMD --offline --disable-plugin=tmpfs --init" +runcmd "$MOCKCMD --install --disable-plugin=tmpfs ccache" +if [ ! -e $CHROOT/usr/bin/ccache ]; then + echo "init/clean test FAILED. ccache not found." + exit 1 +fi + diff --git a/mock/integration-tests/08-oldstyle-cmd.tst b/mock/integration-tests/08-oldstyle-cmd.tst new file mode 100644 index 0000000..fa796a1 --- /dev/null +++ b/mock/integration-tests/08-oldstyle-cmd.tst @@ -0,0 +1,16 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# test old-style cmdline options +# +header "test old-style cmdline options" +runcmd "$MOCKCMD --offline clean" +runcmd "$MOCKCMD --offline init" +runcmd "$MOCKCMD --disable-plugin=tmpfs install ccache" +if [ ! -e $CHROOT/usr/bin/ccache ]; then + echo "init/clean test FAILED. ccache not found." + exit 1 +fi + diff --git a/mock/integration-tests/09-mockchain-fail.tst b/mock/integration-tests/09-mockchain-fail.tst new file mode 100644 index 0000000..1ea139a --- /dev/null +++ b/mock/integration-tests/09-mockchain-fail.tst @@ -0,0 +1,13 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +header "test mockchain failure" +runcmd "$MOCKCHAIN --offline ${TESTDIR}/test-A-1.1-0.src.rpm ${TESTDIR}/test-B-1.1-0.src.rpm" +res=$? + +if [ $res -eq 0 ]; then + echo "mockchain returned success when it should have failed!" + exit 1 +fi +exit 0 diff --git a/mock/integration-tests/10-mockchain-partial.tst b/mock/integration-tests/10-mockchain-partial.tst new file mode 100644 index 0000000..6f0cf62 --- /dev/null +++ b/mock/integration-tests/10-mockchain-partial.tst @@ -0,0 +1,13 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +header "test mockchain partial failure" +runcmd "$MOCKCHAIN --offline -c ${TESTDIR}/*.src.rpm" +res=$? + +if [ $res -ne 4 ]; then + echo "mockchain did not report partial failure when it should!" + exit 1 +fi +exit 0 diff --git a/mock/integration-tests/11-mockchain-success.tst b/mock/integration-tests/11-mockchain-success.tst new file mode 100644 index 0000000..900ced1 --- /dev/null +++ b/mock/integration-tests/11-mockchain-success.tst @@ -0,0 +1,13 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +header "test mockchain success" +runcmd "$MOCKCHAIN -c ${TESTDIR}/test-C-1.1-0.src.rpm ${TESTDIR}/test-B-1.1-0.src.rpm ${TESTDIR}/test-A-1.1-0.src.rpm" +res=$? + +if [ $res -ne 0 ]; then + echo "mockchain returned fail when should have succeeded!" + exit 1 +fi +exit 0 diff --git a/mock/integration-tests/12-mockchain-recurse.tst b/mock/integration-tests/12-mockchain-recurse.tst new file mode 100644 index 0000000..ea88c47 --- /dev/null +++ b/mock/integration-tests/12-mockchain-recurse.tst @@ -0,0 +1,13 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +header "test mockchain recursive success" +runcmd "$MOCKCHAIN --recurse ${TESTDIR}/test-A-1.1-0.src.rpm ${TESTDIR}/test-B-1.1-0.src.rpm ${TESTDIR}/test-C-1.1-0.src.rpm" +res=$? + +if [ $res -ne 0 ]; then + echo "mockchain returned fail when should have succeeded!" + exit 1 +fi +exit 0 diff --git a/mock/integration-tests/13-mockchain-gen-br.tst b/mock/integration-tests/13-mockchain-gen-br.tst new file mode 100644 index 0000000..15531f3 --- /dev/null +++ b/mock/integration-tests/13-mockchain-gen-br.tst @@ -0,0 +1,43 @@ +#!/bin/sh + +if test -z "$TESTDIR"; then + TESTDIR=$(dirname "$(readlink -f "$0")") +fi + +. ${TESTDIR}/functions + +: "${MOCKCHAIN=mockchain}" + +header "online mockchain, tmpfs.keep_mounted=True, nosync and one package having generated BuildRequires" + +test "$(rpm -qa nosync | wc -l)" -eq 2 || die "nosync.x86_64 and nosync.i686 needs to be installed" + +confdir=$HOME/.config +mkdir -p "$confdir" +local_config=$confdir/mock.cfg + +# cleanup potentially mounted stuff we'll overmount by tmpfs +runcmd "$MOCKCHAIN --scrub=all" + +test -f "$local_config" && die "please remove $local_config first" + +cat > "$local_config" < "$local_config" </dev/null) +lines=0 +while read -r line; do + case $line in + "(none)") die "some packages are not signed" ;; + *) + if test ${#line} -lt 256; then + die "weird signature $line" + fi + ;; + esac + lines=$(( lines + 1 )) +done <<<"$rpmoutput" + +test $lines -eq 2 || die "two packages are expected to be signed" + +exit 0 diff --git a/mock/integration-tests/20-lvm-plugin.tst b/mock/integration-tests/20-lvm-plugin.tst new file mode 100644 index 0000000..1ab03ad --- /dev/null +++ b/mock/integration-tests/20-lvm-plugin.tst @@ -0,0 +1,98 @@ +#!/bin/bash + +storage=https://raw.githubusercontent.com/rpm-software-management/mock-test-data/main +srpm_version1=$storage/mock-test-bump-version-1-0.src.rpm +srpm_version2=$storage/mock-test-bump-version-2-0.src.rpm + +: "${MOCKCMD=mock}" +if test -z "$TESTDIR"; then + TESTDIR=$(dirname "$(readlink -f "$0")") +fi + +. ${TESTDIR}/functions + +header "testing that lvm plugin works" + +confdir=$HOME/.config +mkdir -p "$confdir" + +test -f "/test-lvm-disk" || die "Please run prepare-lvm.sh first" + +local_config=$confdir/mock.cfg + +test -f "$local_config" && die "please remove $local_config first" + +TMPDIR=$(mktemp -d) || die "can't create temporary directory" +REPODIR=$TMPDIR/local_repo +mkdir "$REPODIR" + +createrepo_c "$REPODIR" + +cat > "$local_config" <> "$local_config" + +# Build the updated package and recreate the repository +runcmd "$MOCKCMD --rebuild $srpm_version2 --resultdir $REPODIR" \ + || die "mock rebuild 2 (additional package in buildroot) failed" +createrepo_c "$REPODIR" +additional_snapshots_assert 0 + +# Now the package should be auto-updated by mock, and LVM snapshot re-created. +runcmd "$MOCKCMD --rebuild $srpm_version2 --resultdir $REPODIR" \ + || die "mock rebuild 3 (updated package in buildroot) failed" +additional_snapshots_assert 1 + +# The updated package is already there, so nothing is upgraded - and also no +# new LVM snapshot should be created +rm "$REPODIR/root.log" +runcmd "$MOCKCMD --rebuild $srpm_version2 --resultdir $REPODIR" \ + || die "mock rebuild 4 (updated package already in buildroot) failed" +additional_snapshots_assert 0 + +runcmd "$MOCKCMD --shell '/bin/true'" || die "mock shell failed" + +runcmd "$MOCKCMD --scrub=all" || die "mock scrub failed" + +# repeated run should succeed as well, rhbz#1805179 +runcmd "$MOCKCMD --scrub=all" || die "mock scrub failed" + +exit 0 diff --git a/mock/integration-tests/21-mockchain-image.tst b/mock/integration-tests/21-mockchain-image.tst new file mode 100644 index 0000000..63b2cd6 --- /dev/null +++ b/mock/integration-tests/21-mockchain-image.tst @@ -0,0 +1,18 @@ +#!/bin/sh + +if test -z "$TESTDIR"; then + TESTDIR=$(dirname "$(readlink -f "$0")") +fi + +. ${TESTDIR}/functions + +: "${MOCKCHAIN=mockchain}" +: "${MOCK=mock}" + +header "mockchain with --use-bootstap-image" + +test "$(rpm -qa podman | wc -l)" -eq 1 || die "podman package needs to be installed" + +runcmd "$MOCKCMD --scrub=chroot" +runcmd "$MOCKCMD --scrub=bootstrap" +runcmd "$MOCKCHAIN --use-bootstrap-image ${TESTDIR}/test-C-1.1-0.src.rpm ${TESTDIR}/test-B-1.1-0.src.rpm ${TESTDIR}/test-A-1.1-0.src.rpm" diff --git a/mock/integration-tests/22-rootdir.tst b/mock/integration-tests/22-rootdir.tst new file mode 100644 index 0000000..0bc9837 --- /dev/null +++ b/mock/integration-tests/22-rootdir.tst @@ -0,0 +1,23 @@ +#!/bin/sh + +if test -z "$TESTDIR"; then + TESTDIR=$(dirname "$(readlink -f "$0")") +fi + +. "${TESTDIR}/functions" +set -e + +# creating special files (e.g. /dev/null doesn't work below tmpfs) +WORKDIR=$(mktemp -d -p "$HOME") + +cleanup() { rm -rf "$WORKDIR"; } +trap cleanup EXIT + +header "mock with --rootdir option" + +for isolation in simple nspawn; do + rootdir=$WORKDIR/$isolation + runcmd "$MOCKCMD --isolation=$isolation --rootdir=$rootdir --scrub=chroot" + runcmd "$MOCKCMD --isolation=$isolation --rootdir=$rootdir --scrub=bootstrap" + runcmd "$MOCKCMD --isolation=$isolation --rootdir=$rootdir --bootstrap-chroot ${TESTDIR}/test-C-1.1-0.src.rpm" +done diff --git a/mock/integration-tests/23-local-mirrorlist.tst b/mock/integration-tests/23-local-mirrorlist.tst new file mode 100644 index 0000000..9cc45ff --- /dev/null +++ b/mock/integration-tests/23-local-mirrorlist.tst @@ -0,0 +1,45 @@ +#!/bin/sh + +if test -z "$TESTDIR"; then + TESTDIR=$(dirname "$(readlink -f "$0")") +fi + +. "${TESTDIR}/functions" +set -e + +: "${MOCKCMD=mock}" + +header "mock and local mirrorlist with expanded variables" + +TMPDIR=$(mktemp -d) +cleanup() { rm -rf "$TMPDIR"; } +trap cleanup EXIT + +mirrorlist=$TMPDIR/x86_64/mirrorlist +config=$TMPDIR/rawhide.cfg + +mkdir "$(dirname "$mirrorlist")" +cat > "$mirrorlist" < "$config" < "$local_config" <> "$local_config" + +# Build the updated package and recreate the repository +runcmd "$MOCKCMD --rebuild $srpm_version2 --resultdir $REPODIR" \ + || die "mock rebuild 2 (additional package in buildroot) failed" +createrepo_c "$REPODIR" +assert_changed_cache yes + +# Now the package should be auto-updated by mock, and root-cache tarball re-created. +runcmd "$MOCKCMD --rebuild $srpm_version2 --resultdir $REPODIR" \ + || die "mock rebuild 3 (updated package in buildroot) failed" +assert_changed_cache yes + +# The updated package is already there, so nothing is upgraded - and also no +# new new tarball crated +rm "$REPODIR/root.log" +runcmd "$MOCKCMD --rebuild $srpm_version2 --resultdir $REPODIR" \ + || die "mock rebuild 4 (updated package already in buildroot) failed" +assert_changed_cache no + +runcmd "$MOCKCMD --scrub=all" || die "mock scrub failed" + +exit 0 diff --git a/mock/integration-tests/26-external-deps.tst b/mock/integration-tests/26-external-deps.tst new file mode 100644 index 0000000..9cd1a30 --- /dev/null +++ b/mock/integration-tests/26-external-deps.tst @@ -0,0 +1,14 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# +# Test offline build as well as tmpfs +# +header "Test external deps" +runcmd "$MOCKCMD --offline --config-opts=external_buildrequires=True test-F-0-1.fc33.src.rpm" +if [ ! -e $outdir/mock-*.noarch.rpm ]; then + echo "rebuild test FAILED. could not find $outdir/mock-*.noarch.rpm" + exit 1 +fi + diff --git a/mock/integration-tests/27-nspawn.tst b/mock/integration-tests/27-nspawn.tst new file mode 100644 index 0000000..6988bd6 --- /dev/null +++ b/mock/integration-tests/27-nspawn.tst @@ -0,0 +1,13 @@ +#!/bin/sh + +. ${TESTDIR}/functions + +# Test that we use nspawn by default + +header "Test nspawn default" +output=$($MOCKCMD --shell 'echo $PPID') + +case $output in + 1) ;; + *) die "Looks like systemd-nspawn isn't used, unexpected output '$output'" ;; +esac diff --git a/mock/integration-tests/README.txt b/mock/integration-tests/README.txt new file mode 100644 index 0000000..27ac1a4 --- /dev/null +++ b/mock/integration-tests/README.txt @@ -0,0 +1,27 @@ +These 3 src.rpms are setup to build on almost any rpm-based system. +They have a simple chain of buildrequires: + +test-A BuildRequires test-B +test-B BuildRequires test-C + +So using a normal shell expansion the packages built like: + +mock --chain -r fedora-18-x86_64 *.src.rpm + +will fail to build b/c test-A will be built first and it won't have +its buildreqs satisified. + +Tests to run: + +test failure: +mock --chain -r fedora-18-x86_64 *.src.rpm + +test partial failure: +mock --chain -r fedora-18-x86_64 -c *.src.rpm + +test complete success: +mock --chain -r fedora-18-x86_64 -c test-C-1.1-0.src.rpm test-B-1.1-0.src.rpm test-A-1.1-0.src.rpm + +test success due to recursive rebuild: +mock --chain -r fedora-18-x86_64 --recurse *.src.rpm + diff --git a/mock/integration-tests/daemontest.c b/mock/integration-tests/daemontest.c new file mode 100644 index 0000000..e9023dd --- /dev/null +++ b/mock/integration-tests/daemontest.c @@ -0,0 +1,106 @@ +/* test program to test orhphanskill feature + * compile it (per below) and put it in /tmp/ of the chroot. + * Then: + * mock -r CFG chroot /tmp/daemontest + * + * Expected output: + * INFO: mock suid wrapper version 0.8.0 + * INFO: mock.py version 0.8.0 starting... + * State Changed: start + * State Changed: init + * WARNING: Leftover process 1331205 is being killed with signal 15: /builddir/a.out + * + */ + + +/* + * UNIX Daemon Server Programming Sample Program + * Levent Karakas May 2001 + * + * To compile: cc -o exampled examped.c + * To run: ./exampled + * To test daemon: ps -ef|grep exampled (or ps -aux on BSD systems) + * To test log: tail -f /tmp/exampled.log + * To test signal: kill -HUP `cat /tmp/exampled.lock` + * To terminate: kill `cat /tmp/exampled.lock` + * */ + +#include +#include +#include +#include +#include +#include +#include +#include + + +#define RUNNING_DIR "/tmp" +#define LOCK_FILE "exampled.lock" +#define LOG_FILE "exampled.log" + +void log_message(filename,message) +char *filename; +char *message; +{ +FILE *logfile; + logfile=fopen(filename,"a"); + if(!logfile) return; + fprintf(logfile,"%s\n",message); + fclose(logfile); +} + +void signal_handler(sig) +int sig; +{ + switch(sig) { + case SIGHUP: + log_message(LOG_FILE,"hangup signal catched"); + break; + case SIGTERM: + log_message(LOG_FILE,"terminate signal catched"); + exit(0); + break; + } +} + +void daemonize() +{ +int i,lfp; +char str[10]; + /* nspawn implies pid = 1, and to test --isolation=nspawn we need to drop + session leader status even there */ + /* if(getppid()==1) return; */ /* already a daemon */ + i=fork(); + if (i<0) exit(1); /* fork error */ + if (i>0) exit(0); /* parent exits */ + /* child (daemon) continues */ + setsid(); /* obtain a new process group */ + for (i=getdtablesize();i>=0;--i) close(i); /* close all descriptors */ + i=open("/dev/null",O_RDWR); dup(i); dup(i); /* handle standart I/O */ + umask(027); /* set newly created file permissions */ + chdir(RUNNING_DIR); /* change running directory */ + lfp=open(LOCK_FILE,O_RDWR|O_CREAT,0640); + if (lfp<0) exit(1); /* can not open */ + if (lockf(lfp,F_TLOCK,0)<0) exit(0); /* can not lock */ + /* first instance continues */ + sprintf(str,"%d\n",getpid()); + write(lfp,str,strlen(str)); /* record pid to lockfile */ + signal(SIGCHLD,SIG_IGN); /* ignore child */ + signal(SIGTSTP,SIG_IGN); /* ignore tty signals */ + signal(SIGTTOU,SIG_IGN); + signal(SIGTTIN,SIG_IGN); + signal(SIGHUP,signal_handler); /* catch hangup signal */ + signal(SIGTERM,signal_handler); /* catch kill signal */ +} + +int main() +{ + daemonize(); + // run for roughly 5 mins then exit. No need to stick around if unit test fails. + int i=0; + for( i=0; i<300; i++ ) sleep(1); + return 0; +} + +/* EOF */ diff --git a/mock/integration-tests/dropcache.py b/mock/integration-tests/dropcache.py new file mode 100755 index 0000000..81a0c57 --- /dev/null +++ b/mock/integration-tests/dropcache.py @@ -0,0 +1,13 @@ +#!/usr/bin/python3 + +import os +import sys + +if os.getuid() != 0: + print("must be root to drop caches!") + sys.exit(-1) + +print("******************* dropping caches") +with open('/proc/sys/vm/drop_caches', 'w') as f: + f.write("3") +sys.exit(0) diff --git a/mock/integration-tests/functions b/mock/integration-tests/functions new file mode 100644 index 0000000..7e32bf4 --- /dev/null +++ b/mock/integration-tests/functions @@ -0,0 +1,18 @@ +#!/bin/sh + +header() { + echo "" + echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" + echo $1 + echo "%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%" + echo "" +} + +runcmd() { + echo " => $1" + time sh -c "$1" + ret=$? + return $ret +} + +die () { echo >&2 "ERROR: $*" ; exit 1 ; } diff --git a/mock/integration-tests/overlayfs_layers_test.py b/mock/integration-tests/overlayfs_layers_test.py new file mode 100644 index 0000000..23d31a7 --- /dev/null +++ b/mock/integration-tests/overlayfs_layers_test.py @@ -0,0 +1,303 @@ +# -*- coding: utf-8 -*- +# This file is part of overlayfs plugin for mock +# Copyright (C) 2018 Zdeněk Žamberský ( https://github.com/zzambers ) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import os.path +import sys +import overlayfs + +# About test: +# This test calls methods of overlayfs plugin (snapshot/layers/refs related) +# and tests integrity of internal data structures. Test does not actually +# mount anything and can be ran as unpriviledged user. + +# How to run: +# Test accepts single argument with directory where to place test base_dir of +# overlayfs plugin (where to perform testing). Default is current directory +# ( if argument is omitted ). +# Test requires overlayfs.py to be present in the same directory as this test +# or corretly set up PYTHONPATH. + +####################### +# Dummy classes # +####################### + +# Dummy classes are used to satisfy plugin's constructor... + +class DummyPlugins(object): + + def add_hook(self, _name, _method): #pylint: disable=no-self-use + return + +class DummyConf(object): + + def __init__(self, base_dir): + self.base_dir = base_dir + + def get(self, name): + if name == "base_dir": + return self.base_dir + +class DummyBuildRoot(object): # pylint: disable=too-few-public-methods + + def __init__(self, rootDir, sharedRootName): + self.rootdir = rootDir + self.shared_root_name = sharedRootName + +#################### +# TEST class # +#################### + +class LayersTest(object): + + def __init__(self, baseDir, rootDir, configName): + plugins = DummyPlugins() + conf = DummyConf(baseDir) + buildRoot = DummyBuildRoot(rootDir, configName) + self.plugin = overlayfs.OverlayFsPlugin(plugins, conf, buildRoot) + + # assert methods used by test method + + @staticmethod + def assertFileExists(fileName): + if not os.path.exists(fileName): + errFormat = "Assertion error: file does not exist: {} !" + errMsg = errFormat.format(fileName) + raise Exception(errMsg) + + def assertFileHasContent(self, fileName, expected): + self.assertFileExists(fileName) + value = self.plugin.readFile(fileName) + if not value == expected: + fmt = "Assertion error: file {} expeceted content: {} actual content: {}" + errMsg = fmt.format(fileName, expected, value) + raise Exception(errMsg) + + def assertLayerRefcount(self, name, ntimes): + layerId = self.plugin.getLayerFromRef(name) + refCounterFile = self.plugin.getLayerRefCounterFile(layerId) + self.assertFileHasContent(refCounterFile, str(ntimes)) + + def assertSameLayer(self, name1, name2): + layer1Id = self.plugin.getLayerFromRef(name1) + layer2Id = self.plugin.getLayerFromRef(name2) + if not self.plugin.isSameLayer(layer1Id, layer2Id): + fmt = "Assertion error: {} and {} do not point to the same layer!" + errMsg = fmt.format(name1, name2) + raise Exception(errMsg) + + def assertNotSameLayer(self, name1, name2): + layer1Id = self.plugin.getLayerFromRef(name1) + layer2Id = self.plugin.getLayerFromRef(name2) + if self.plugin.isSameLayer(layer1Id, layer2Id): + fmt = "Assertion error: {} and {} do point to the same layer!" + errMsg = fmt.format(name1, name2) + raise Exception(errMsg) + + def assertRefExists(self, name): + if not self.plugin.refExists(name): + errMsg = "Assertion error: ref {} does not exist!".format(name) + raise Exception(errMsg) + + def assertRefNotExist(self, name): + if self.plugin.refExists(name): + errMsg = "Assertion error: ref {} exists!".format(name) + raise Exception(errMsg) + + def assertNLayers(self, expected): + n = len(os.listdir(self.plugin.getLayersDir())) + if not n == expected: + fmt = "Assertion error: expected {} layers, but got {} layers !" + errMsg = fmt.format(str(expected), str(n)) + raise Exception(errMsg) + + # tests that layers, refs work correctly, but does not actually mount + # anything + def runTest(self): + plugin = self.plugin + + plugin.basicInit() + + pluginBaseDir = plugin.getPluginInstanceDir() + layersDir = plugin.getLayersDir() + refsDir = plugin.getRefsDir() + self.assertFileExists(pluginBaseDir) + self.assertFileExists(layersDir) + self.assertFileExists(refsDir) + + plugin.initLayers() + baseLayerRef = plugin.getBaseLayerRef() + upperLayerRef = plugin.getUpperLayerRef() + currentLayerRef = plugin.getCurrentLayerRef() + self.assertRefExists(baseLayerRef) + self.assertRefExists(upperLayerRef) + self.assertRefExists(currentLayerRef) + + self.assertNLayers(1) + # .base == .upper + self.assertSameLayer(baseLayerRef, upperLayerRef) + # .base == .current + self.assertSameLayer(baseLayerRef, currentLayerRef) + # refs: .base, .upper, .current + self.assertLayerRefcount(baseLayerRef, 3) + + plugin.prepareLayersForMount() + self.assertNLayers(2) + # .base != .upper + self.assertNotSameLayer(baseLayerRef, upperLayerRef) + # .base == .current + self.assertSameLayer(baseLayerRef, currentLayerRef) + # refs: .base, .current + 1 layer + self.assertLayerRefcount(baseLayerRef, 3) + # refs: .upper + self.assertLayerRefcount(upperLayerRef, 1) + + layerARef = "a" + self.plugin.createSnapshot(layerARef) + self.assertNLayers(2) + # a == .upper + self.assertSameLayer(layerARef, upperLayerRef) + # a == .current + self.assertSameLayer(layerARef, currentLayerRef) + # refs: .base + 1 layer + self.assertLayerRefcount(baseLayerRef, 2) + # refs: a, .upper, .current + self.assertLayerRefcount(layerARef, 3) + + plugin.prepareLayersForMount() + self.assertNLayers(3) + # a != .upper + self.assertNotSameLayer(layerARef, upperLayerRef) + # a == .current + self.assertSameLayer(layerARef, currentLayerRef) + # refs: .base + 1 layer + self.assertLayerRefcount(baseLayerRef, 2) + # refs: a, .current + 1 layer + self.assertLayerRefcount(layerARef, 3) + # refs: .upper + self.assertLayerRefcount(upperLayerRef, 1) + + plugin.restoreSnapshot(layerARef) + self.assertNLayers(2) + # a == .upper + self.assertSameLayer(layerARef, upperLayerRef) + # a == .current + self.assertSameLayer(layerARef, currentLayerRef) + # refs: base + 1 layer + self.assertLayerRefcount(baseLayerRef, 2) + # refs: a, .current, .upper + self.assertLayerRefcount(layerARef, 3) + + plugin.prepareLayersForMount() + self.assertNLayers(3) + # a != .upper + self.assertNotSameLayer(layerARef, upperLayerRef) + # a == .current + self.assertSameLayer(layerARef, currentLayerRef) + # refs: base + 1 layer + self.assertLayerRefcount(baseLayerRef, 2) + # refs: a, .current + 1 layer + self.assertLayerRefcount(layerARef, 3) + # refs: .upper + self.assertLayerRefcount(upperLayerRef, 1) + + layerBRef = "b" + plugin.createSnapshot(layerBRef) + self.assertNLayers(3) + # b == .upper + self.assertSameLayer(layerBRef, upperLayerRef) + # b == .current + self.assertSameLayer(layerBRef, currentLayerRef) + # refs: base + 1 layer + self.assertLayerRefcount(baseLayerRef, 2) + # refs: a, + 1 layer + self.assertLayerRefcount(layerARef, 2) + # refs: b, .current, .upper + self.assertLayerRefcount(layerBRef, 3) + + plugin.prepareLayersForMount() + plugin.prepareLayersForMount() + self.assertNLayers(4) + # a != .upper + self.assertNotSameLayer(layerARef, upperLayerRef) + # b != .upper + self.assertNotSameLayer(layerBRef, upperLayerRef) + # b == .current + self.assertSameLayer(layerBRef, currentLayerRef) + # refs: .base + 1 layer + self.assertLayerRefcount(baseLayerRef, 2) + # refs: a, + 1 layer + self.assertLayerRefcount(layerARef, 2) + # refs: b, .current + 1 layer + self.assertLayerRefcount(layerBRef, 3) + + plugin.restoreSnapshot(baseLayerRef) + self.assertNLayers(3) + # a != .upper + self.assertNotSameLayer(layerARef, upperLayerRef) + # b != .upper + self.assertNotSameLayer(layerBRef, upperLayerRef) + # .base == .upper + self.assertSameLayer(baseLayerRef, upperLayerRef) + # .base == .current + self.assertSameLayer(baseLayerRef, currentLayerRef) + # refs: .base, .upper .current + 1 layer + self.assertLayerRefcount(baseLayerRef, 4) + # refs: a, + 1 layer + self.assertLayerRefcount(layerARef, 2) + # refs: b + self.assertLayerRefcount(layerBRef, 1) + + plugin.deleteSnapshot(layerARef) + self.assertNLayers(3) + self.assertRefNotExist(layerARef) + # b != .upper + self.assertNotSameLayer(layerBRef, upperLayerRef) + # .base == .upper + self.assertSameLayer(baseLayerRef, upperLayerRef) + # .base == .current + self.assertSameLayer(baseLayerRef, currentLayerRef) + # refs: .base, .upper, .current + 1 layer + self.assertLayerRefcount(baseLayerRef, 4) + # refs: b + self.assertLayerRefcount(layerBRef, 1) + + plugin.deleteSnapshot(layerBRef) + self.assertNLayers(1) + self.assertRefNotExist(layerBRef) + self.assertSameLayer(baseLayerRef, upperLayerRef) + # refs: base, .upper, .current + self.assertLayerRefcount(baseLayerRef, 3) + + +def main(): + args = sys.argv + if len(args) == 2: + currentDir = args[1] + else: + currentDir = os.getcwd() + + baseDir = os.path.join(currentDir, "overlayfs-base") + rootDir = os.path.join(currentDir, "root") + configName = "config-name" + + test = LayersTest(baseDir, rootDir, configName) + test.runTest() + + +if __name__ == "__main__": + main() diff --git a/mock/integration-tests/releasetests.sh b/mock/integration-tests/releasetests.sh new file mode 100755 index 0000000..ba29c20 --- /dev/null +++ b/mock/integration-tests/releasetests.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# vim:tw=0:ts=4:sw=4 + +# this is a test script to run everything through its paces before you do a +# release. The basic idea is: + +# 1) make distcheck to ensure that all autoconf stuff is setup properly +# 2) run some basic tests to test different mock options. +# 3) rebuild mock srpm using this version of mock under all distributed configs + +# This test will only run on a machine with full access to internet. +# might work with http_proxy= env var, but I havent tested that. +# +# This test script expects to be run on an x86_64 machine. It will *not* run +# properly on an i386 machine. +# + +set -e +set -x + +DIR=$(cd $(dirname $0); pwd) +TOP_SRCTREE=$DIR/../ +cd $TOP_SRCTREE + +make distclean ||: +./autogen.sh +./configure +make distcheck +make srpm +make check diff --git a/mock/integration-tests/runconfigs.sh b/mock/integration-tests/runconfigs.sh new file mode 100644 index 0000000..9ea4c0f --- /dev/null +++ b/mock/integration-tests/runconfigs.sh @@ -0,0 +1,89 @@ +#!/bin/sh + +if [ "$MOCKCMD" = "" ] +then + d=$(cd $(dirname $0); pwd) + . $d/testenvironment +fi + +. ${TESTDIR}/functions + +cd $TOPDIR + +if [ "$1" != "" ]; then + configs=$1 +else + configs=$(ls ../mock-core-configs/etc/mock | grep .cfg \ + | grep -v -e default -e custom -e chroot-aliases \ + | grep -E -v 'arm|ppc|s390|sparc|aarch') +fi + +cleanup() +{ + $MOCKCMD --clean + $MOCKCMD --scrub bootstrap + exit 1 +} +trap cleanup INT HUP QUIT TERM + +fails=0 + +# +# Test build all configs we ship. +# +header "testing all supported configurations" +for i in $configs; do + srpm=$SIMPLESRPM + case $i in + # Keep building the SIMPLESRPM + # - oraclelinux+epel can't work with mock.spec, as %rhel is not defined + # - fedora-eln doesn't provide all the EPEL packages needed by mock.spec + # - navy doesn't provide all the needed packages: + # https://github.com/rpm-software-management/mock/pull/1550 + oraclelinux+epel-7*) ;; + fedora-eln*) ;; + navy-*) ;; + # For EPEL/Fedora try to build Mock. + fedora*|*+epel*-8*) + # we support building mock there, so test it instead + srpm=$MOCKSRPM + ;; + # Skip tests for those chroots. + # - amazonlinux - see #522 + amazonlinux*) continue;; + esac + + # For branched Fedoras, try also updates-testing. + target_config=$(basename "$(readlink -f "../mock-core-configs/etc/mock/$i")") + enablerepo= + case $target_config in + fedora-eln*|fedora-rawhide*|fedora*i*86*) ;; + fedora*) + enablerepo=" --enablerepo updates-testing " ;; + esac + + name=$(basename $i .cfg) + header "testing config $name.cfg with tmpfs plugin" + runcmd "$MOCKCMD -r $name --enable-plugin=tmpfs --rebuild $srpm $enablerepo" + if [ $? != 0 ]; then + echo "FAILED: $i (tmpfs)" + fails=$(($fails+1)) + else + echo "PASSED: $i (tmpfs)" + fi + sudo python ${TESTDIR}/dropcache.py + header "testing config $name.cfg *without* tmpfs plugin" + runcmd "$MOCKCMD -r $name --disable-plugin=tmpfs --rebuild $srpm $enablerepo" + if [ $? != 0 ]; then + echo "FAILED: $i" + fails=$(($fails+1)) + else + echo "PASSED: $i" + fi + + runcmd "$MOCKCMD -r $name --scrub=all" || echo >&2 "Can not even --scrub $name!" +done + +msg=$(printf "%d configuration failures\n" $fails) +header "$msg" +exit $fails diff --git a/mock/integration-tests/runregressions.sh b/mock/integration-tests/runregressions.sh new file mode 100644 index 0000000..37d83c8 --- /dev/null +++ b/mock/integration-tests/runregressions.sh @@ -0,0 +1,43 @@ +#!/bin/sh + + +if [ "$MOCKCMD" = "" ] +then + d=$(cd $(dirname $0); pwd) + . $d/testenvironment +fi + +. ${TESTDIR}/functions + +cd $TOPDIR + +trap '$MOCKCMD --clean; exit 1' INT HUP QUIT TERM + +$MOCKCMD --init + +fails=0 +# +# run regression tests +# +header "running regression tests" +for i in ${TESTDIR}/*.tst; do + sh $i + if [ $? != 0 ]; then + fails=$(($fails + 1)) + echo "* FAILED: $i" + else + echo "* PASSED: $i" + fi + echo "****************************************************" +done + +msg=$(printf "%d regression failures\n" $fails) +header "$msg" + +# +# clean up +# +header "clean up from first round of tests" +runcmd "$MOCKCMD --offline --clean" + +exit $fails diff --git a/mock/integration-tests/runtests.sh b/mock/integration-tests/runtests.sh new file mode 100755 index 0000000..e02a560 --- /dev/null +++ b/mock/integration-tests/runtests.sh @@ -0,0 +1,74 @@ +#!/bin/sh -x +# vim:tw=0:ts=4:sw=4 + +# this is a test script to run everything through its paces before you do a +# release. The basic idea is: + +# 1) make distcheck to ensure that all autoconf stuff is setup properly +# 2) run some basic tests to test different mock options. +# 3) rebuild mock srpm using this version of mock under all distributed configs + +# This test will only run on a machine with full access to internet. +# might work with http_proxy= env var, but I havent tested that. +# +# This test script expects to be run on an x86_64 machine. It will *not* run +# properly on an i386 machine. +# + +#VERBOSE= +VERBOSE=--verbose +export VERBOSE + +d=$(cd $(dirname $0); pwd) +. $d/testenvironment +. ${TESTDIR}/functions + +trap '$MOCKCMD --clean; exit 1' INT HUP QUIT TERM + +# clear out root cache so we get at least run without root cache present +#sudo rm -rf /var/lib/mock/cache/${testConfig}/root_cache + +# +# pre-populate yum cache for the rest of the commands below +# + +# With TMT, we have the deps pre-installed by ansible. +if test -z "$TMT_VERSION"; then + MOCKSRPM=$( + cd "$TOPDIR" || exit 1 + tito build --srpm --offline | grep Wrote | grep src.rpm | awk '{ print $2 }' + ) + test -e "$MOCKSRPM" || exit 1 +else + set -- /tmp/mock-test-srpms/mock-*.src.rpm + MOCKSRPM=$1 +fi + +runcmd "$MOCKCMD --init" +header "installing dependencies for $MOCKSRPM" +runcmd "$MOCKCMD --disable-plugin=tmpfs --installdeps $MOCKSRPM" +if [ ! -e $CHROOT/usr/include/python* ]; then +echo "installdeps test FAILED. could not find /usr/include/python*" +exit 1 +fi + +header "running regression tests" +sh ${TESTDIR}/runregressions.sh +fails=$? + +msg=$(printf "%s regression failures\n" $fails) +header "$msg" + +# +# clean up +# +header "clean up from first round of tests" +runcmd "$MOCKCMD --offline --clean" + +header "running configuration tests" +sh ${TESTDIR}/runconfigs.sh +cfgfails=$? + +msg=$(printf "%d total failures\n" $(($fails+$cfgfails))) +header "$msg" +exit $fails diff --git a/mock/integration-tests/setup-box b/mock/integration-tests/setup-box new file mode 100755 index 0000000..8e56a4c --- /dev/null +++ b/mock/integration-tests/setup-box @@ -0,0 +1,54 @@ +#! /bin/sh + +directory=$(dirname "$(readlink -f "$0")")/setup-playbook +cd "$directory" || exit 1 + +echo >&2 "Running in directory: $directory" + +inventory=$directory/inventory +test -f "$inventory" || { + cat >&2 < + + [testing_mock_machines:vars] + mock_rhn_user= + mock_rhn_pass= + +... and then re-run this script. + +If you have no IP_ADDRESS for testing, you may, e.g., create a new machine in +EC2 with group_vars/all.yaml file like: + + infra_subnet: subnet-0995f6a466849f4c3 + aws: + ssh_key: "praiskup" + image: ami-004f552bba0e5f64f + instance_type: t2.large + root_volume_size: 60 + security_group: mock-testing + +by running + + $ ansible-playbook spawn-test-machine-ec2.yaml + PLAY [Start new machine] ***************** + + TASK [create the testing mock vm in ec2] * + changed: [localhost] + + TASK [print ipv4] ************************ + ... + "Public IP: 18.234.135.130" + ... + +EOF + exit 1 +} + +ansible-playbook -i inventory setup-playbook.yml "$@" diff --git a/mock/integration-tests/setup-playbook/.gitignore b/mock/integration-tests/setup-playbook/.gitignore new file mode 100644 index 0000000..e845c18 --- /dev/null +++ b/mock/integration-tests/setup-playbook/.gitignore @@ -0,0 +1 @@ +inventory diff --git a/mock/integration-tests/setup-playbook/README.md b/mock/integration-tests/setup-playbook/README.md new file mode 100644 index 0000000..725a1b9 --- /dev/null +++ b/mock/integration-tests/setup-playbook/README.md @@ -0,0 +1,54 @@ +Starting EC2 machine +-------------------- + +1. init git submodules + + git submodule update --init --recursive + +2. setup AWS vars like this (change fields appropriately): + + cat > group_vars/all.yaml < + aws_secret_access_key= + +4. start the machine + + ansible-playbook spawn-test-machine-ec2.yaml + + PLAY [Start new machine] ******************* + + TASK [create the testing mock vm in ec2] *** + changed: [localhost] + + TASK [print ipv4] ************************** + ok: [localhost] => { + "msg": [ + "Instance ID: i-02f769285490cbb64", + "Network ID: eni-0298fa7a391ecc42e", + "Unusable Public IP: 107.20.103.13" + ] + } + + PLAY RECAP ******************************** + localhost : ok=2 changed=1 unreachable=0 ... diff --git a/mock/integration-tests/setup-playbook/files/install-copr-packages b/mock/integration-tests/setup-playbook/files/install-copr-packages new file mode 100755 index 0000000..9001e61 --- /dev/null +++ b/mock/integration-tests/setup-playbook/files/install-copr-packages @@ -0,0 +1,106 @@ +#! /bin/bash + +# Install RPM packages (and dependencies) identified by the (a) package names, +# (b) upstream commmit and (c) Copr directory. The short variant of the +# upstream commit must be part of the packages' Release in NVR. +# +# ARG1: coprowner/projectname:dir +# ARG2: upstream-commit +# ARGN: packagename1 packagename2 ... + +# TODO: DNF5/YUM compat? +DNF=/usr/bin/dnf-3 +REPOQUERY=( "$DNF" repoquery ) +DOWNLOAD=( "$DNF" download ) + +info() { echo >&2 "INFO: $*" ; } +error() { echo >&2 "ERROR: $*" ; false; } +die() { echo >&2 "FATAL: $*" ; exit 1 ; } + +copr_dir=$1 ; shift +commit=$1 ; shift +copr_uri=https://download.copr.fedorainfracloud.org/results + +copr_chroot() ( + # mimic: https://github.com/rpm-software-management/dnf5/blob/c6edcd75260accf7070f261e5b406fcf1f5db328/dnf5-plugins/copr_plugin/copr_config.cpp#L71-L80 + . /etc/os-release + name=$ID + version=$VERSION_ID + arch=$(rpm --eval %_arch) + if test "$name" = fedora; then + if test "$REDHAT_SUPPORT_PRODUCT_VERSION" = rawhide; then + version=rawhide + fi + fi + echo "$name-$version-$arch" +) + +repo=$copr_uri/$copr_dir/$(copr_chroot) + +repoid=pull-request-proposed-packages + +info "using [$repoid] repo: $repo" + +export clean_cache=true + + +repos=( "--repofrompath=$repoid,$repo" ) + +_repoquery() { + local opts=( "${repos[@]}" --disablerepo='*' --enablerepo "$repoid" ) + if ${clean_cache-true}; then + opts+=( "--setopt=$repoid.metadata_expire=1" ) + fi + local cmd=( "${REPOQUERY[@]}" "${opts[@]}" "$@" ) + info "Executing: ${cmd[*]}" + "${cmd[@]}" 2>/dev/null +} + +find_nvr() { + # ARGS: $1=pkg $2=commit + # RETURN: $find_nvr_result + # STATUS: true if found + local _pkgname=$1 _commit=${2:0:7} _found=false + while read -r name version release; do + test -z "$name" && continue + test "$name" = "$_pkgname" || continue + case $release in + *$_commit*) + find_nvr_result=$name-$version-$release + $_found && error "second occurence of $name-$version-$release" + _found=true + ;; + *) + continue + ;; + esac + done < <( _repoquery --qf='%{NAME} %{VERSION} %{RELEASE}\n' ) + $_found || error "$_pkgname with $commit in release not found" +} + +nvrs=() +SECONDS=0 +TIMEOUT=${TIMEOUT-1200} # 20 minutes by default +for pkg; do + while true; do + if find_nvr "$pkg" "$commit"; then + nvrs+=( "$find_nvr_result" ) + clean_cache=false + break + fi + test "$SECONDS" -gt "$TIMEOUT" && die "The timeout ${TIMEOUT}s left" + clean_cache=true + sleep 5 + done +done + +if test -n "$SRPM_DOWNLOAD_DIR"; then + mkdir -p "$SRPM_DOWNLOAD_DIR" + cmd=( "${DOWNLOAD[@]}" "${repos[@]}" '--disablerepo=*' --enablerepo "$repoid" + "${nvrs[@]}" --source --downloaddir "$SRPM_DOWNLOAD_DIR" ) +else + cmd=( "$DNF" -y install "${nvrs[@]}" "${repos[@]}" --nogpgcheck ) +fi + +info "Running: ${cmd[*]}" +"${cmd[@]}" diff --git a/mock/integration-tests/setup-playbook/files/install-mock-packages-built-by-packit b/mock/integration-tests/setup-playbook/files/install-mock-packages-built-by-packit new file mode 100755 index 0000000..77016fc --- /dev/null +++ b/mock/integration-tests/setup-playbook/files/install-mock-packages-built-by-packit @@ -0,0 +1,23 @@ +#! /bin/bash -x + +# Install specific Mock RPM packages (and dependencies) from a Copr repository +# created by Packit PR or PUSH workflow. Pick the precise package version built +# from the latest commit. + +_d=$(dirname "$(readlink -f "$0")") + +if test -n "$PACKIT_PR_ID"; then + PROJECT=packit/rpm-software-management-mock-$PACKIT_PR_ID + COMMIT=$PACKIT_COMMIT_SHA +elif test -n "$PACKIT_COPR_PROJECT"; then + PROJECT=$PACKIT_COPR_PROJECT + COMMIT=$PACKIT_COMMIT_SHA +elif test -n "$TEST_GIT_MAIN"; then + PROJECT=@mock/mock + COMMIT=$(curl -H "Accept: application/vnd.github.VERSION.sha" https://api.github.com/repos/rpm-software-management/mock/commits/main) +else + echo >&2 "Can't decide where to install packages from" + exit 1 +fi + +"$_d/install-copr-packages" "$PROJECT" "$COMMIT" "$@" diff --git a/mock/integration-tests/setup-playbook/play-tf.yml b/mock/integration-tests/setup-playbook/play-tf.yml new file mode 100644 index 0000000..a482a5c --- /dev/null +++ b/mock/integration-tests/setup-playbook/play-tf.yml @@ -0,0 +1,25 @@ +--- +- name: Prepare the testing machine for running Mock tests + hosts: all + user: "root" + vars: + mock_test_username: mockbuild + mock_test_workdir: /home/mock/mock-testing + mock_gpg_dir: "/home/mockbuild/gpg" + mock_gpg_wrapper: "/home/mockbuild/gpg-mock" + mock_test_rpmmacros: /home/mockbuild/.rpmmacros + mock_lvm_volume: /test-lvm-disk + mock_clone: /home/mockbuild/mock + no_subscription_management: true + + tasks: + - include_tasks: tasks/main.yml + + - name: upload the "install copr package" script + copy: + src: "{{ item }}" + dest: "/usr/bin/{{ item }}" + mode: '0755' + loop: + - install-copr-packages + - install-mock-packages-built-by-packit diff --git a/mock/integration-tests/setup-playbook/setup-playbook.yml b/mock/integration-tests/setup-playbook/setup-playbook.yml new file mode 100644 index 0000000..40ba79e --- /dev/null +++ b/mock/integration-tests/setup-playbook/setup-playbook.yml @@ -0,0 +1,23 @@ +--- +- name: prepare for root connection + hosts: testing_mock_machines + gather_facts: false + roles: + - roles/fix-root-ssh + +- name: remote tasks + hosts: testing_mock_machines + user: "root" + + vars: + mock_test_username: mockbuild + mock_test_workdir: /home/mock/mock-testing + mock_gpg_dir: "/home/mockbuild/gpg" + mock_gpg_wrapper: "/home/mockbuild/gpg-mock" + mock_test_rpmmacros: /home/mockbuild/.rpmmacros + mock_lvm_volume: /test-lvm-disk + mock_clone: /home/mockbuild/mock + + tasks: + - include_tasks: tasks/main.yml + tags: always diff --git a/mock/integration-tests/setup-playbook/spawn-test-machine-ec2.yaml b/mock/integration-tests/setup-playbook/spawn-test-machine-ec2.yaml new file mode 100644 index 0000000..81e5cf6 --- /dev/null +++ b/mock/integration-tests/setup-playbook/spawn-test-machine-ec2.yaml @@ -0,0 +1,40 @@ +--- +- name: "Start new machine" + hosts: localhost + connection: local + gather_facts: false + tasks: + - name: "create the testing mock vm in ec2" + amazon.aws.ec2_instance: + name: "testing-mock" + state: running + profile: "{{ aws.profile }}" + region: us-east-1 + key_name: "{{ aws.ssh_key }}" + count: 1 + image_id: "{{ aws.image }}" + instance_type: "{{ aws.instance_type }}" + detailed_monitoring: true + network: + assign_public_ip: true + vpc_subnet_id: "{{ aws.infra_subnet }}" + security_group: "{{ aws.security_group }}" + termination_protection: false + wait: true + tags: + FedoraGroup: copr + CoprPurpose: testing-mock + volumes: + - ebs: + volume_size: "{{ aws.root_volume_size }}" + encrypted: true + delete_on_termination: true + device_name: /dev/sda1 + register: instances_started + + - name: print ipv4 + debug: + msg: + - "Instance ID: {{ instances_started.instances[0].instance_id }}" + - "Network ID: {{ instances_started.instances[0].network_interfaces[0].network_interface_id }}" + - "Unusable Public IP: {{ instances_started.instances[0].public_ip_address }}" diff --git a/mock/integration-tests/setup-playbook/tasks/main.yml b/mock/integration-tests/setup-playbook/tasks/main.yml new file mode 100644 index 0000000..998ec55 --- /dev/null +++ b/mock/integration-tests/setup-playbook/tasks/main.yml @@ -0,0 +1,148 @@ +--- +- name: install also RPMs from the stable copr repository + community.general.copr: + name: '@mock/mock-stable' + +- name: up2date mock dependencies + community.general.copr: + name: '@mock/mock-deps' + +- name: install mock and other needed packages + package: name={{ packages }} state=latest + vars: + packages: + - git + - make + - mock + - mock-lvm + - nosync.x86_64 + - nosync.i686 + - podman + - python3-behave + - python3-hamcrest + - rpm-sign + - subscription-manager + - tito + - tmux + tags: packages + +- name: update all packages + package: name='*' state=latest + tags: packages + +- name: create the testing user + user: + name: "{{ mock_test_username }}" + groups: + - mock + - wheel + +- name: create gpg directory + file: + state: directory + path: "{{ item }}" + owner: "{{ mock_test_username }}" + mode: 0700 + with_items: + - "{{ mock_gpg_dir }}" + - "{{ mock_test_workdir }}" + +- name: install gpg-create rule + copy: + content: | + %echo Generating a basic OpenPGP key + %no-protection + Key-Type: RSA + Key-Length: 4096 + Name-Real: John Doe + Name-Email: jdoe@foo.com + Expire-Date: 0 + Passphrase: redhat + %commit + %echo done + dest: "{{ mock_test_workdir }}/gpg-batch" + +- name: install the gpg wrapper + template: + src: gpg-mock.j2 + dest: "{{ mock_gpg_wrapper }}" + mode: 0700 + owner: "{{ mock_test_username }}" + group: "{{ mock_test_username }}" + register: gpg_script_generated + tags: doit + +- name: generate the GPG key + shell: > + runuser -u {{ mock_test_username }} -- + {{ mock_gpg_wrapper }} --batch + --generate-key {{ mock_test_workdir }}/gpg-batch + when: gpg_script_generated.changed + +- name: install rpm macros file + copy: + content: | + # GENERATED AUTOMATICALLY by $0 + # from man(8) rpmsign + %_gpg_name John Doe + %__gpg {{ mock_gpg_wrapper }} + dest: "{{ mock_test_rpmmacros }}" + owner: "{{ mock_test_username }}" + group: "{{ mock_test_username }}" + mode: 0644 + +- name: check volume exists + stat: + path: "{{ mock_lvm_volume }}" + register: volume_stat + +- name: create 8GB LVM volume + shell: dd if=/dev/zero of={{ mock_lvm_volume }} bs=1M count=8000 + when: not volume_stat.stat.exists + +- name: check if lvs exists + shell: + cmd: vgs | grep mock + changed_when: lvs_check.rc != 0 + failed_when: false + register: lvs_check + +- name: create the volume group + shell: | + device=`losetup -f {{ mock_lvm_volume }} --show` + pvcreate -y -ff $device + vgcreate mock $device + when: lvs_check.changed + +- name: Red Hat subscription + redhat_subscription: + state: present + username: "{{ mock_rhn_user }}" + password: "{{ mock_rhn_pass }}" + pool: "Red Hat Enterprise Linux for Virtual Datacenters, Standard" + when: + - no_subscription_management is not defined or no_subscription_management is false + +- name: Clone Mock repo + git: + repo: https://github.com/rpm-software-management/mock.git + dest: "{{ mock_clone }}" + version: main + become: true + become_user: "{{ mock_test_username }}" + +# TODO: drop sudo usage from our testsuite +- name: don't require sudo password from the testing user + lineinfile: + path: /etc/sudoers + line: "{{ mock_test_username }} ALL=(ALL) NOPASSWD: ALL" + +- name: install mock configuration for the testing user + copy: + content: | + # make sure we don't break testsuite for a random connection error + config_opts['package_manager_max_attempts'] = 4 + config_opts['package_manager_attempt_delay'] = 10 + dest: "/etc/mock/site-defaults.cfg" + mode: 0644 + tags: mock_config diff --git a/mock/integration-tests/setup-playbook/templates/gpg-mock.j2 b/mock/integration-tests/setup-playbook/templates/gpg-mock.j2 new file mode 100644 index 0000000..cbf59ac --- /dev/null +++ b/mock/integration-tests/setup-playbook/templates/gpg-mock.j2 @@ -0,0 +1,12 @@ +#! /bin/bash + +echo -n >&2 "running: $0 " + +for arg +do + echo -n >&2 "$(printf %q "$arg") " +done +echo + + +exec /usr/bin/gpg --homedir "{{ mock_gpg_dir }}" "$@" diff --git a/mock/integration-tests/test-A-1.1-0.src.rpm b/mock/integration-tests/test-A-1.1-0.src.rpm new file mode 100644 index 0000000..e38485e Binary files /dev/null and b/mock/integration-tests/test-A-1.1-0.src.rpm differ diff --git a/mock/integration-tests/test-B-1.1-0.src.rpm b/mock/integration-tests/test-B-1.1-0.src.rpm new file mode 100644 index 0000000..245c345 Binary files /dev/null and b/mock/integration-tests/test-B-1.1-0.src.rpm differ diff --git a/mock/integration-tests/test-C-1.1-0.src.rpm b/mock/integration-tests/test-C-1.1-0.src.rpm new file mode 100644 index 0000000..f988900 Binary files /dev/null and b/mock/integration-tests/test-C-1.1-0.src.rpm differ diff --git a/mock/integration-tests/test-D-1.1-0.src.rpm b/mock/integration-tests/test-D-1.1-0.src.rpm new file mode 100644 index 0000000..e20bf11 Binary files /dev/null and b/mock/integration-tests/test-D-1.1-0.src.rpm differ diff --git a/mock/integration-tests/test-E-1.1-0.src.rpm b/mock/integration-tests/test-E-1.1-0.src.rpm new file mode 100644 index 0000000..1512bb6 Binary files /dev/null and b/mock/integration-tests/test-E-1.1-0.src.rpm differ diff --git a/mock/integration-tests/test-F-0-1.fc33.src.rpm b/mock/integration-tests/test-F-0-1.fc33.src.rpm new file mode 100644 index 0000000..7a4ef08 Binary files /dev/null and b/mock/integration-tests/test-F-0-1.fc33.src.rpm differ diff --git a/mock/integration-tests/testenvironment b/mock/integration-tests/testenvironment new file mode 100644 index 0000000..c98e1f7 --- /dev/null +++ b/mock/integration-tests/testenvironment @@ -0,0 +1,21 @@ +TESTDIR=$(cd $(dirname $0); pwd; cd ..) +TOPDIR=$(dirname $TESTDIR) +SIMPLESRPM=$TESTDIR/test-C-1.1-0.src.rpm + +#VERBOSE= +VERBOSE=--verbose + +# +# most tests below will use this mock command line +# +testConfig=fedora-42-x86_64 +uniqueext="$$-$RANDOM" +outdir=${TOPDIR}/mock-unit-test +cfgdir=${TOPDIR}/../mock-core-configs/etc/mock +#MOCKCMD="sudo ./py/mock.py $VERBOSE --resultdir=$outdir --uniqueext=$uniqueext --configdir=$cfgdir -r $testConfig --plugin-option=root_cache:age_check=False $MOCK_EXTRA_ARGS" +MOCKCMD_NO_RESULTDIR="mock $VERBOSE --uniqueext=$uniqueext --configdir=$cfgdir -r $testConfig $MOCK_EXTRA_ARGS" +MOCKCMD="$MOCKCMD_NO_RESULTDIR --resultdir=$outdir" +MOCKCHAIN="mock $VERBOSE --chain --configdir=$cfgdir -r $testConfig $MOCK_EXTRA_ARGS" +CHROOT=/var/lib/mock/${testConfig}-$uniqueext/root + +export MOCKSRPM TOPDIR TESTDIR testConfig uniqueext outdir MOCKCMD MOCKCMD_NO_RESULTDIR CHROOT VERBOSE MOCKCHAIN SIMPLESRPM diff --git a/mock/integration-tests/verify_repos.sh b/mock/integration-tests/verify_repos.sh new file mode 100644 index 0000000..9a5b84a --- /dev/null +++ b/mock/integration-tests/verify_repos.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +cfgs=$(grep 'mirrorlist' ../etc/mock/*.cfg | sed -e 's/^.*mirrorlist=//') +tmpfile=$(mktemp) +for c in $cfgs; do + wget -q -O $tmpfile $c + lines=$(wc -l $tmpfile | awk '{print $1}') + if [ $lines = 1 ]; then + echo "!!! $c is not a valid URL!!!" + fi +done diff --git a/mock/mock.conf b/mock/mock.conf new file mode 100644 index 0000000..73912e3 --- /dev/null +++ b/mock/mock.conf @@ -0,0 +1,2 @@ +#Type Name ID GECOS Home directory Shell +g mock 135 - diff --git a/mock/mockchain b/mock/mockchain new file mode 100755 index 0000000..6d94286 --- /dev/null +++ b/mock/mockchain @@ -0,0 +1,28 @@ +#! /bin/bash + +mock_command=(mock --chain) +args=() + +while test $# -gt 0; do + case $1 in + --mock-option) + shift + mock_command+=( "$1" ) + ;; + --mock-option=*) + option=${1//--mock-option=} + mock_command+=( "$option" ) + ;; + *) + args+=( "$1" ) + ;; + esac + shift +done + +mock_command+=( "${args[@]}" ) + +echo >&2 +echo >&2 "## mockchain is deprecated, use mock --chain instead ##" +echo >&2 +exec "${mock_command[@]}" diff --git a/mock/precompile-bash-completion b/mock/precompile-bash-completion new file mode 100755 index 0000000..c34b55a --- /dev/null +++ b/mock/precompile-bash-completion @@ -0,0 +1,25 @@ +#! /bin/bash + +mock_options=$( + # shellcheck source=/dev/null + . /usr/share/bash-completion/bash_completion + _parse_help py/mock.py +) + +# Some trivial assert to check it actually works +case $mock_options in +*--chroot*) ;; +*) exit 1 ;; +esac + +while IFS= read -r line; do + case $line in + *PRECOMPILED_PARSED_MOCK_HELP*) + echo "echo '" + echo "$mock_options" + echo "'" + echo "return" + ;; + *) echo "$line" ;; + esac +done < ./etc/bash_completion.d/mock > "${1-mock-preprocessed}" diff --git a/mock/py/mock.py b/mock/py/mock.py new file mode 100755 index 0000000..5e2cc19 --- /dev/null +++ b/mock/py/mock.py @@ -0,0 +1,1181 @@ +#!/usr/bin/python3 -tt +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: + +# Originally written by Seth Vidal +# Sections taken from Mach by Thomas Vander Stichele +# Major reorganization and adaptation by Michael Brown +# Copyright (C) 2007 Michael E Brown +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + +# pylint: disable=pointless-string-statement,wrong-import-position +""" + mock [options] {--init|--clean|--scrub=[all,chroot,cache,root-cache,c-cache,yum-cache,dnf-cache,lvm,overlayfs]} + mock [options] [--rebuild] /path/to/srpm(s) + mock [options] [--chain] /path/to/srpm(s) + mock [options] --buildsrpm {--spec /path/to/spec --sources /path/to/src| + --scm-enable [--scm-option key=value]} + mock [options] {--shell|--chroot} + mock [options] --installdeps {SRPM|RPM} + mock [options] --install PACKAGE + mock [options] --copyin path [..path] destination + mock [options] --copyout path [..path] destination + mock [options] --scm-enable [--scm-option key=value] + mock [options] --dnf-cmd arguments + mock [options] --yum-cmd arguments +""" + +from __future__ import print_function + +# library imports +import argparse +import configparser +import errno +import glob +import grp +import logging +import logging.config + +from pprint import pformat +import os +import os.path +import signal +import shutil +import shlex +import sys +import time +import copy + +# pylint: disable=import-error +from functools import partial + +# our imports +from mockbuild import config +from mockbuild import util +from mockbuild.constants import MOCKCONFDIR, VERSION +from mockbuild.file_downloader import FileDownloader +from mockbuild.mounts import BindMountPoint, FileSystemMountPoint +import mockbuild.backend +from mockbuild.backend import Commands +from mockbuild.buildroot import Buildroot +import mockbuild.exception +from mockbuild.plugin import Plugins +import mockbuild.rebuild +from mockbuild.state import State +from mockbuild.trace_decorator import traceLog +import mockbuild.uid +from mockbuild.scrub_all import scrub_all_chroots + +# set up basic logging until config file can be read +FORMAT = "%(levelname)s: %(message)s" +logging.basicConfig(format=FORMAT, level=logging.WARNING) +log = logging.getLogger() + + +signal_names = {1: "SIGHUP", + 13: "SIGPIPE", + 15: "SIGTERM" + } + +# This line is replaced by spec file %install section. +_MOCK_NVR = None + + +class RepoCallback(argparse.Action): + """ Parse --enablerepo/--disablerepo options """ + def __call__(self, parser, namespace, value, option_string=None): + if not hasattr(namespace, self.dest): + setattr(namespace, self.dest, []) + options = getattr(namespace, self.dest) + options.extend((option_string, value)) + + +def command_parse(): + """return options and args from parsing the command line""" + plugins = config.PLUGIN_LIST + parser = argparse.ArgumentParser(usage=__doc__) + + parser.add_argument('--version', action='version', version=VERSION) + + # hack from optparse=>argparse migration time, use add_argument if possible + parser.add_option = parser.add_argument + + # modes (basic commands) + parser.add_option("--rebuild", action="store_const", const="rebuild", + dest="mode", default='__default__', + help="rebuild the specified SRPM(s)") + parser.add_option("--calculate-build-dependencies", action="store_const", + const="calculatedeps", dest="mode", + help="Resolve and install static and dynamic build dependencies") + parser.add_option("--chain", action="store_const", const="chain", + dest="mode", + help="build multiple RPMs in chain loop") + parser.add_option("--buildsrpm", action="store_const", const="buildsrpm", + dest="mode", + help="Build a SRPM from spec (--spec ...) and sources" + "(--sources ...) or from SCM") + parser.add_option("--debug-config", action="store_const", const="debugconfig", + dest="mode", + help="Prints all options in config_opts") + parser.add_option("--debug-config-expanded", action="store_const", const="debugconfigexpand", + dest="mode", + help="Prints all options in config_opts with jinja template values already expanded") + parser.add_option("--shell", action="store_const", + const="shell", dest="mode", + help="run the specified command interactively within the chroot." + " Default command: /bin/sh") + parser.add_option("--chroot", action="store_const", + const="chroot", dest="mode", + help="run the specified command noninteractively within the chroot.") + parser.add_option("--clean", action="store_const", const="clean", + dest="mode", + help="completely remove the specified chroot") + scrub_choices = ('chroot', 'cache', 'root-cache', 'c-cache', 'yum-cache', + 'dnf-cache', 'lvm', 'overlayfs', 'bootstrap', 'all') + scrub_metavar = "[all|chroot|cache|root-cache|c-cache|yum-cache|dnf-cache]" + parser.add_option("--scrub", action='append', choices=scrub_choices, default=[], + metavar=scrub_metavar, + help="completely remove the specified chroot " + "or cache dir or all of the chroot and cache") + parser.add_option( + "--scrub-all-chroots", action="store_const", dest="mode", + const="scrub-all-chroots", help=( + "Run mock --scrub=all for all chroots that appear to have been " + "used previously (see manual page for more info)." + ), + + ) + parser.add_option("--init", action="store_const", const="init", dest="mode", + help="initialize the chroot, do not build anything") + parser.add_option("--installdeps", action="store_const", const="installdeps", + dest="mode", + help="install build dependencies for a specified SRPM or SPEC file") + parser.add_option("-i", "--install", action="store_const", const="install", + dest="mode", + help="install packages using package manager") + parser.add_option("--list-chroots", action="store_const", const="listchroots", + dest="mode", + help="List all chroot's configs") + parser.add_option("--update", action="store_const", const="update", + dest="mode", + help="update installed packages using package manager") + parser.add_option("--remove", action="store_const", const="remove", + dest="mode", + help="remove packages using package manager") + parser.add_option("--orphanskill", action="store_const", const="orphanskill", + dest="mode", + help="Kill all processes using specified buildroot.") + + parser.add_option("--copyin", action="store_const", const="copyin", + dest="mode", + help="Copy file(s) into the specified chroot") + + parser.add_option("--copyout", action="store_const", const="copyout", + dest="mode", + help="Copy file(s) from the specified chroot") + + parser.add_option("--pm-cmd", action="store_const", const="pm-cmd", + dest="mode", + help="Execute package management command (with yum or dnf)") + + parser.add_option("--yum-cmd", action="store_const", const="yum-cmd", + dest="mode", + help="Execute package management command with yum") + + parser.add_option("--dnf-cmd", action="store_const", const="dnf-cmd", + dest="mode", + help="Execute package management command with dnf") + + parser.add_option("--snapshot", action="store_const", const="snapshot", + dest="mode", + help="Create a new LVM/overlayfs snapshot with given name") + + parser.add_option("--remove-snapshot", action="store_const", const="remove_snapshot", + dest="mode", + help="Remove LVM/overlayfs snapshot with given name") + + parser.add_option("--rollback-to", action="store_const", const="rollback-to", + dest="mode", + help="Rollback to given snapshot") + + parser.add_option("--umount", action="store_const", const="umount", + dest="mode", help="Umount the buildroot if it's " + "mounted from separate device (LVM/overlayfs)") + parser.add_option("--mount", action="store_const", const="mount", + dest="mode", help="Mount the buildroot if it's " + "mounted from separate device (LVM/overlayfs)") + # chain + parser.add_option('--localrepo', default=None, + help=("local path for the local repo, defaults to making " + "its own (--chain mode only)")) + parser.add_option('-c', '--continue', default=False, action='store_true', + dest='cont', + help="if a pkg fails to build, continue to the next one") + parser.add_option( + '-a', '--addrepo', default=[], action='append', dest='repos', + metavar="REPO", help=( + "Add a repo baseurl to the DNF/YUM configuration for both the " + "build chroot and the bootstrap chroot. This option can be " + "specified multiple times, allowing you to reference multiple " + "repositories in addition to the default repository set.")) + parser.add_option('--recurse', default=False, action='store_true', + help="if more than one pkg and it fails to build, try to build the rest and come back to it") + parser.add_option('--tmp_prefix', default=None, dest='tmp_prefix', + help="tmp dir prefix - will default to username-pid if not specified") + # options + parser.add_option("-r", "--root", action="store", type=str, dest="chroot", + help="chroot config file name or path. Taken as a path if it ends " + "in .cfg, otherwise looked up in the configdir. default: %%default", + metavar="CONFIG", + default='default') + + parser.add_option("--offline", action="store_false", dest="online", + default=True, + help="activate 'offline' mode.") + + parser.add_option("-n", "--no-clean", action="store_false", dest="clean", + help="do not clean chroot before building", default=True) + parser.add_option("--cleanup-after", action="store_true", + dest="cleanup_after", default=None, + help="Clean chroot after building. Use with --resultdir." + " Only active for 'rebuild'.") + parser.add_option("-N", "--no-cleanup-after", action="store_false", + dest="cleanup_after", default=None, + help="Don't clean chroot after building. If automatic" + " cleanup is enabled, use this to disable.", ) + parser.add_option("--cache-alterations", action="store_true", + dest="cache_alterations", default=False, + help="Rebuild the root cache after making alterations to the chroot" + " (i.e. --install). Only useful when using tmpfs plugin.") + parser.add_option("--nocheck", action="store_false", dest="check", + default=True, help="pass --nocheck to rpmbuild to skip 'make check' tests") + parser.add_option("--arch", action="store", dest="arch", + default=None, help="Sets kernel personality().") + parser.add_option("--forcearch", action="store", dest="forcearch", + default=None, help="Force architecture to DNF (pass --forcearch to DNF).") + parser.add_option("--target", action="store", dest="rpmbuild_arch", + default=None, help="passed to rpmbuild as --target") + parser.add_option("-D", "--define", action="append", dest="rpmmacros", + default=[], type=str, metavar="'MACRO EXPR'", + help="define an rpm macro (may be used more than once)") + parser.add_option("--macro-file", action="store", type=str, dest="macrofile", + default=[], help="Use pre-defined rpm macro file") + parser.add_option("--with", action="append", dest="rpmwith", + default=[], type=str, metavar="option", + help="enable configure option for build (may be used more than once)") + parser.add_option("--without", action="append", dest="rpmwithout", + default=[], type=str, metavar="option", + help="disable configure option for build (may be used more than once)") + parser.add_option("--resultdir", action="store", type=str, + default=None, help="path for resulting files to be put") + parser.add_option("--rootdir", action="store", type=str, + default=None, help="Path for where the chroot should be built") + parser.add_option("--uniqueext", action="store", type=str, + default=None, + help="Arbitrary, unique extension to append to buildroot" + " directory name") + parser.add_option("--configdir", action="store", dest="configdir", + default=None, + help="Change where config files are found") + parser.add_option("--config-opts", action="append", dest="cli_config_opts", + default=[], help="Override configuration option.") + parser.add_option("--rpmbuild_timeout", action="store", + dest="rpmbuild_timeout", type=int, default=None, + help="Fail build if rpmbuild takes longer than 'timeout'" + " seconds ") + parser.add_option("--unpriv", action="store_true", default=False, + help="Drop privileges before running command when using --chroot") + parser.add_option("--cwd", action="store", default=None, + metavar="DIR", + help="Change to the specified directory (relative to the chroot)" + " before running command when using --chroot") + + parser.add_option("--spec", action="store", + help="Specifies spec file to use to build an SRPM") + parser.add_option("--sources", action="store", + help="Specifies sources (either a single file or a directory of files)" + "to use to build an SRPM (used only with --buildsrpm)") + parser.add_option("--symlink-dereference", action="store_true", dest="symlink_dereference", + default=False, help="Follow symlinks in sources (used only with --buildsrpm)") + + parser.add_option("--short-circuit", + choices=['prep', 'install', 'build', 'binary'], + help="Pass short-circuit option to rpmbuild to skip already " + "complete stages. Warning: produced packages are unusable. " + "Implies --no-clean. Valid options: build, install, binary") + parser.add_option("--rpmbuild-opts", action="store", + help="Pass additional options to rpmbuild") + parser.add_option("--enablerepo", action=RepoCallback, type=str, + dest="enable_disable_repos", default=[], + help="Pass enablerepo option to yum/dnf", metavar='[repo]') + parser.add_option("--disablerepo", action=RepoCallback, type=str, + dest="enable_disable_repos", default=[], + help="Pass disablerepo option to yum/dnf", metavar='[repo]') + parser.add_option("--old-chroot", action="store_true", dest="old_chroot", + default=False, + help="Obsoleted. Use --isolation=simple") + parser.add_option("--new-chroot", action="store_true", dest="new_chroot", + default=False, + help="Obsoleted. Use --isolation=nspawn") + parser.add_option("--isolation", action="store", dest="isolation", + help="what level of isolation to use. Valid option: simple, nspawn") + parser.add_option("--enable-network", action="store_true", dest="enable_network", + default=False, + help="enable networking.") + parser.add_option("--postinstall", action="store_true", dest="post_install", + default=False, help="Try to install built packages in " + "the same buildroot right after build") + + # verbosity + parser.add_option("-v", "--verbose", action="store_const", const=2, + dest="verbose", default=1, help="verbose build") + parser.add_option("-q", "--quiet", action="store_const", const=0, + dest="verbose", help="quiet build") + parser.add_option("--trace", action="store_true", default=False, + dest="trace", help="Enable internal mock tracing output.") + + # plugins + parser.add_option("--enable-plugin", action="append", + dest="enabled_plugins", type=str, default=[], + help="Enable plugin. Currently-available plugins: %s" + % repr(plugins)) + parser.add_option("--disable-plugin", action="append", + dest="disabled_plugins", type=str, default=[], + help="Disable plugin. Currently-available plugins: %s" + % repr(plugins)) + parser.add_option("--plugin-option", action="append", dest="plugin_opts", + default=[], type=str, + metavar="PLUGIN:KEY=VALUE", + help="define an plugin option (may be used more than once)") + + parser.add_option("-p", "--print-root-path", help="print path to chroot root", + dest="printrootpath", action="store_true", + default=False) + + parser.add_option("-l", "--list-snapshots", + help="list LVM/overlayfs snapshots associated with buildroot", + dest="list_snapshots", action="store_true", + default=False) + + # SCM options + parser.add_option("--scm-enable", help="build from SCM repository", + dest="scm", action="store_true", + default=None) + parser.add_option("--scm-option", action="append", dest="scm_opts", + default=[], type=str, + help="define an SCM option (may be used more than once)") + + # Package management options + parser.add_option("--yum", help="use yum as package manager", + dest="pkg_manager", action="store_const", const="yum") + parser.add_option("--dnf", help="use dnf as package manager", + dest="pkg_manager", action="store_const", const="dnf") + + # Bootstrap options + parser.add_option('--bootstrap-chroot', dest='bootstrapchroot', action='store_true', + help="build in two stages, using chroot rpm for creating the build chroot", + default=None) + parser.add_option('--no-bootstrap-chroot', dest='bootstrapchroot', action='store_false', + help="build in a single stage, using system rpm for creating the build chroot", + default=None) + + parser.add_option('--use-bootstrap-image', dest='usebootstrapimage', action='store_true', + help="create bootstrap chroot from container image (turns " + "--bootstrap-chroot on)", default=None) + parser.add_option('--no-bootstrap-image', dest='usebootstrapimage', action='store_false', + help="don't create bootstrap chroot from container image", default=None) + + parser.add_option('--buildroot-image', + help=( + "Use an OCI image (or a local file containing an OCI " + "image as a tarball) as the base for the buildroot. " + "The image must contain a compatible distribution " + "(e.g., fedora:41 for fedora-41-x86_64)")) + + parser.add_option('--additional-package', action='append', default=[], + type=str, dest="additional_packages", + help=("Additional package to install into the buildroot before " + "the build is done. Can be specified multiple times.")) + parser.add_option("--hermetic-build", nargs=2, + metavar=("LOCKFILE", "REPO_DIRECTORY"), + help="Perform a hermetic (fully offline) build") + + (options, args) = parser.parse_known_args() + + if options.mode == '__default__': + # handle old-style commands + if len(args) and args[0] in ('chroot', 'shell', 'rebuild', 'install', + 'installdeps', 'remove', 'init', 'clean'): + options.mode = args[0] + args = args[1:] + else: + options.mode = 'rebuild' + + if options.hermetic_build and options.chroot != 'default': + raise mockbuild.exception.BadCmdline( + "The --hermetic-build mode uses a special chroot configuration, " + "you can not select the chroot configuration with the " + "-r/--root option.") + + if options.hermetic_build and options.mode != "rebuild": + raise mockbuild.exception.BadCmdline("--rebuild mode needed with --hermetic-build") + + options.calculatedeps = None + if options.mode == "calculatedeps": + options.mode = "rebuild" + options.calculatedeps = True + + # Optparse.parse_args() eats '--' argument, while argparse doesn't. Do it manually. + if args and args[0] == '--': + args = args[1:] + + if options.scrub: + options.mode = 'clean' + + # explicitly disallow multiple targets in --target argument + if options.rpmbuild_arch: + if options.rpmbuild_arch.find(',') != -1: + raise mockbuild.exception.BadCmdline("--target option accepts only " + "one arch. Invalid: %s" % options.rpmbuild_arch) + + if options.mode == 'buildsrpm' and not (options.spec): + if not options.scm: + raise mockbuild.exception.BadCmdline("Must specify both --spec and " + "--sources with --buildsrpm") + + if options.recurse: + # --recurse implies --continue + options.cont = True + + if options.localrepo and options.mode != 'chain': + raise mockbuild.exception.BadCmdline( + "The --localrepo option works only with --chain") + + if options.recurse and options.mode != 'chain': + raise mockbuild.exception.BadCmdline( + "Only --chain mode supports --recurse build algorithm") + + if options.cont and options.mode != 'chain': + raise mockbuild.exception.BadCmdline( + "Only --chain mode supports --continue build algorithm") + + if options.spec: + options.spec = os.path.expanduser(options.spec) + if options.sources: + options.sources = os.path.expanduser(options.sources) + + if options.additional_packages and options.mode != 'rebuild': + raise mockbuild.exception.BadCmdline( + "The --additional-package option requires the --rebuild mode") + + return (options, args) + + +def handle_signals(buildroot, number, frame): + + log.info("\nReceived signal %s activating orphansKill", signal_names[number]) + util.orphansKill(buildroot.make_chroot_path()) + sys.exit(128 + number) + + +@traceLog() +def setup_logging(config_path, config_opts, options): + log_ini = os.path.join(config_path, config_opts["log_config_file"]) + + try: + if not os.path.exists(log_ini): + if os.path.normpath('/etc/mock') != os.path.normpath(config_path): + log.warning("Could not find required logging config file: %s. Using default...", + log_ini) + log_ini = os.path.join("/etc/mock", config_opts["log_config_file"]) + if not os.path.exists(log_ini): + raise IOError("Could not find log config file %s" % log_ini) + else: + raise IOError("Could not find log config file %s" % log_ini) + except IOError as exc: + log.error(exc) + sys.exit(50) + + try: + log_cfg = configparser.ConfigParser() + logging.config.fileConfig(log_ini) + log_cfg.read(log_ini) + except (IOError, OSError, configparser.NoSectionError) as exc: + log.error("Log config file(%s) not correctly configured: %s", log_ini, exc) + sys.exit(50) + + try: + # set up logging format strings + config_opts['build_log_fmt_str'] = log_cfg.get("formatter_%s" % config_opts['build_log_fmt_name'], + "format", raw=1) + config_opts['root_log_fmt_str'] = log_cfg.get("formatter_%s" % config_opts['root_log_fmt_name'], + "format", raw=1) + config_opts['state_log_fmt_str'] = log_cfg.get("formatter_%s" % config_opts['state_log_fmt_name'], + "format", raw=1) + except configparser.NoSectionError as exc: + log.error("Log config file (%s) missing required section: %s", log_ini, exc) + sys.exit(50) + + # set logging verbosity + if options.verbose == 0: + log.handlers[0].setLevel(logging.WARNING) + tmplog = logging.getLogger("mockbuild.Root.state") + if tmplog.handlers: + tmplog.handlers[0].setLevel(logging.WARNING) + elif options.verbose == 1: + log.handlers[0].setLevel(logging.INFO) + elif options.verbose == 2: + log.handlers[0].setLevel(logging.DEBUG) + logging.getLogger("mockbuild.Root.build").propagate = 1 + logging.getLogger("mockbuild").propagate = 1 + + # enable tracing if requested + logging.getLogger("trace").propagate = 0 + if options.trace: + logging.getLogger("trace").propagate = 1 + + logging.getLogger("mockbuild").mock_stderr_line_prefix = "" + if config_opts['stderr_line_prefix'] != "": + logging.getLogger("mockbuild").mock_stderr_line_prefix = config_opts['stderr_line_prefix'] + + +@traceLog() +def check_arch_combination(target_arch, config_opts): + try: + legal = config_opts['legal_host_arches'] + except KeyError: + return + host_arch = config_opts['host_arch'] + if (host_arch not in legal) and not config_opts['forcearch']: + log.info("Unable to build arch %s natively on arch %s. Setting forcearch to use software emulation.", + target_arch, host_arch) + config_opts['forcearch'] = target_arch + + config.multiply_platform_multiplier(config_opts) + + if not config_opts['forcearch']: + return + + # Check below that we can do cross-architecture builds. + + option = f"--forcearch={config_opts['forcearch']}" + binary_pattern = config_opts["qemu_user_static_mapping"].get(config_opts["forcearch"]) + if not binary_pattern: + # Probably a missing configuration. + log.warning( + "Mock will likely fail, %s is enabled " + "while Mock is unable to detect the corresponding " + "/usr/bin/qemu-*-static binary", + option, + ) + time.sleep(5) + return + + binary = f'/usr/bin/qemu-{binary_pattern}-static' + if os.path.exists(binary): + return + + # qemu-user-static is required, but seems to be missing + if not util.is_host_rh_family(): + # On non-RH systems we are not sure where the qemu--static + # binaries reside - therefore we can't even check whether they are + # installed. Therefore we don't raise an exception; we at least + # notify the user verbosely, cross our fingers, and continue. + log.warning("Mock with %s will likely fail with missing %s", + option, binary) + time.sleep(5) + return + + raise mockbuild.exception.InvalidArchitecture( + f'The {option} feature requires the {binary} ' + 'file to be installed (typically qemu-user-static* package)') + + +@traceLog() +def do_debugconfig(config_opts, expand=False): + jinja_expand = config_opts['__jinja_expand'] + defaults = config.setup_default_config_opts() + defaults['__jinja_expand'] = expand + config_opts['__jinja_expand'] = expand + for key in sorted(config_opts): + if key == '__jinja_expand': + value = jinja_expand + else: + value = config_opts[key] + if (key in defaults) and (key in config_opts) and (config_opts[key] != defaults[key]) or \ + (key not in defaults): + print("config_opts['{}'] = {}".format(key, pformat(value))) + config_opts['__jinja_expand'] = jinja_expand + + +@traceLog() +def do_listchroots(config_path, uidManager): + uidManager.run_in_subprocess_without_privileges( + config.list_configs, config_path, + ) + + +@traceLog() +def rootcheck(): + "verify mock was started correctly (either by sudo or consolehelper)" + # if we're root due to sudo or consolehelper, we're ok + # if not raise an exception and bail + if os.getuid() == 0 and not (os.environ.get("SUDO_UID") or os.environ.get("USERHELPER_UID")): + raise RuntimeError("mock will not run from the root account (needs an unprivileged uid so it can drop privs)") + + +@traceLog() +def groupcheck(unprivGid, tgtGid): + "verify that the user running mock is part of the correct group" + # verify that we're in the correct group (so all our uid/gid manipulations work) + inmockgrp = False + members = [] + for gid in os.getgroups() + [unprivGid]: + name = grp.getgrgid(gid).gr_name + if gid == tgtGid: + inmockgrp = True + break + members.append(name) + if not inmockgrp: + name = grp.getgrgid(tgtGid).gr_name + raise RuntimeError("Must be member of '%s' group to run mock! (%s)" % + (name, ", ".join(members))) + + +@traceLog() +def unshare_namespace(config_opts): + base_unshare_flags = util.CLONE_NEWNS + # IPC ns is unshared later + extended_unshare_flags = base_unshare_flags | util.CLONE_NEWUTS + try: + util.unshare(extended_unshare_flags) + except mockbuild.exception.UnshareFailed as e: + log.debug("unshare(%d) failed, falling back to unshare(%d)", + extended_unshare_flags, base_unshare_flags) + try: + util.unshare(base_unshare_flags) + except mockbuild.exception.UnshareFailed as e2: + log.error("Namespace unshare failed.") + if util.mock_host_environment_type() == "docker" \ + and not ('docker_unshare_warning' in config_opts + and config_opts['docker_unshare_warning']): + log.error("It seems we are running inside of Docker. Let skip unsharing.") + log.error("You should *not* run anything but Mock in this container. You have been warned!") + time.sleep(5) + else: + sys.exit(e2.resultcode) + + +@traceLog() +def main(): + "Main executable entry point." + + # initial sanity check for correct invocation method + rootcheck() + + # drop unprivileged to parse args, etc. + # uidManager saves current real uid/gid which are unprivileged (callers) + # due to suid helper, our current effective uid is 0 + # also supports being run by sudo + # + # setuid wrapper has real uid = unpriv, effective uid = 0 + # sudo sets real/effective = 0, and sets env vars + # setuid wrapper clears environment, so there wont be any conflict between these two + uidManager = mockbuild.uid.setup_uid_manager() + + # go unpriv only when root to make --help etc work for non-mock users + if os.geteuid() == 0: + uidManager.dropPrivsTemp() + + (options, args) = command_parse() + if options.printrootpath or options.list_snapshots: + options.verbose = 0 + + if options.mode == "scrub-all-chroots": + return scrub_all_chroots() + + # config path -- can be overridden on cmdline + config_path = MOCKCONFDIR + if options.configdir: + config_path = options.configdir + + if options.hermetic_build: + options.chroot = "hermetic-build" + + config_opts = uidManager.run_in_subprocess_without_privileges( + config.load_config, config_path, options.chroot) + + # cmdline options override config options + config.set_config_opts_per_cmdline(config_opts, options, args) + + sys.setrecursionlimit(config_opts["recursion_limit"]) + + util.subscription_redhat_init(config_opts, uidManager) + + # allow a different mock group to be specified + uidManager.fix_different_chrootgid(config_opts) + + # verify that our unprivileged uid is in the mock group + groupcheck(uidManager.unprivGid, config_opts['chrootgid']) + + # configure logging + setup_logging(config_path, config_opts, options) + + # verify that we're not trying to build an arch that we can't + check_arch_combination(config_opts['rpmbuild_arch'], config_opts) + + # security cleanup (don't need/want this in the chroot) + if 'SSH_AUTH_SOCK' in os.environ: + del os.environ['SSH_AUTH_SOCK'] + + # elevate privs + uidManager.become_user_without_push(0, 0) + + # do whatever we're here to do + py_version = '{0}.{1}.{2}'.format(*sys.version_info[:3]) + log.info("mock.py version %s starting (python version = %s%s), args: %s", + VERSION, py_version, + "" if not _MOCK_NVR else ", NVR = " + _MOCK_NVR, + " ".join(shlex.quote(x) for x in sys.argv)) + state = State() + plugins = Plugins(config_opts, state) + + # When scrubbing all, we also want to scrub a bootstrap chroot + if options.scrub: + config_opts['use_bootstrap'] = True + + # outer buildroot to bootstrap the installation - based on main config with some differences + bootstrap_buildroot = None + if config_opts['use_bootstrap']: + # first take a copy of the config so we can make some modifications + bootstrap_buildroot_config = config_opts.copy() + # copy plugins configuration so we get a separate deep copy + bootstrap_buildroot_config['plugin_conf'] = \ + copy.deepcopy(config_opts['plugin_conf']) # pylint: disable=no-member + # add '-bootstrap' to the end of the root name + bootstrap_buildroot_config['root'] = bootstrap_buildroot_config['root'] + '-bootstrap' + # don't share root cache tarball + bootstrap_buildroot_config['plugin_conf']['root_cache_opts']['dir'] = \ + "{{cache_topdir}}/" + bootstrap_buildroot_config['root'] + "/root_cache/" + # we don't want to affect the bootstrap.config['nspawn_args'] array, deep copy + bootstrap_buildroot_config['nspawn_args'] = config_opts.get('nspawn_args', []).copy() + + # allow bootstrap buildroot to access the network for getting packages + bootstrap_buildroot_config['rpmbuild_networking'] = True + bootstrap_buildroot_config['use_host_resolv'] = True + util.setup_host_resolv(bootstrap_buildroot_config) + + # disable updating bootstrap chroot + bootstrap_buildroot_config['update_before_build'] = False + + # Enforce host-native repo architecture for bootstrap chroot (unless + # bootstrap_forcearch=True, which should never be the case). This + # decision affects condPersonality() for DNF calls! + host_arch = config_opts["host_arch"] + if config_opts["use_bootstrap_image"]: + # with bootstrap image, bootstrap is always native + bootstrap_buildroot_config['repo_arch'] = config_opts['repo_arch_map'].get(host_arch, host_arch) + elif host_arch not in config_opts.get("legal_host_arches", []) \ + and not config_opts.get('bootstrap_forcearch'): + # target chroot uses --forcearch, but bootstrap is native + bootstrap_buildroot_config['repo_arch'] = config_opts['repo_arch_map'].get(host_arch, host_arch) + # else: + # We keep the 'repo_arch' copied from target chroot (config_opts['repo_arch']): + # - for the 'multilib' cases (we don't want to use x86_64 repos for i586 chroots) + # - 'bootstrap_forcearch' is set + + # disable forcearch in bootstrap, per https://github.com/rpm-software-management/mock/issues/1110 + bootstrap_buildroot_config['forcearch'] = None + + bootstrap_buildroot_state = State(bootstrap=True) + bootstrap_plugins = Plugins(bootstrap_buildroot_config, bootstrap_buildroot_state) + bootstrap_buildroot = Buildroot(bootstrap_buildroot_config, + uidManager, bootstrap_buildroot_state, bootstrap_plugins, + is_bootstrap=True) + + # override configs for bootstrap_* + for option in list(bootstrap_buildroot.config.keys()): + # options that are not related to bootstrap chroot config_opts + dont_copy = ["bootstrap_image"] + prefix = "bootstrap_" + if option.startswith(prefix) and option not in dont_copy: + bootstrap_option = option[len(prefix):] + bootstrap_buildroot.config[bootstrap_option] = bootstrap_buildroot.config[option] + del bootstrap_buildroot.config[option] + + if config_opts['redhat_subscription_required']: + key_dir = '/etc/pki/entitlement' + chroot_dir = bootstrap_buildroot.make_chroot_path(key_dir) + mount_point = BindMountPoint(srcpath=key_dir, bindpath=chroot_dir) + bootstrap_buildroot.mounts.add(mount_point) + + # this changes config_opts['nspawn_args'], so do it after initializing + # bootstrap chroot to not inherit the changes there + util.setup_host_resolv(config_opts) + + buildroot = Buildroot(config_opts, uidManager, state, plugins, bootstrap_buildroot) + + for baseurl in options.repos: + util.add_local_repo(config_opts, baseurl, + bootstrap=bootstrap_buildroot) + + if bootstrap_buildroot is not None: + # add the extra bind mount to the outer chroot + inner_mount = bootstrap_buildroot.make_chroot_path(buildroot.make_chroot_path()) + + # Hide re-mounted chroot from host by private tmpfs. + buildroot.mounts.bootstrap_mounts.append( + FileSystemMountPoint(filetype='tmpfs', + device='hide_root_in_bootstrap', + path=inner_mount, + options="private,mode=0755")) + buildroot.mounts.bootstrap_mounts.append( + BindMountPoint(buildroot.make_chroot_path(), inner_mount, + recursive=True, options="private").treat_as_chroot()) + + signal.signal(signal.SIGTERM, partial(handle_signals, buildroot)) + signal.signal(signal.SIGPIPE, partial(handle_signals, buildroot)) + signal.signal(signal.SIGHUP, partial(handle_signals, buildroot)) + + # postprocess option arguments for bootstrap + if options.mode in ['installdeps', 'install']: + args = [buildroot.wrap_host_file(arg) for arg in args] + + log.info("Signal handler active") + commands = Commands(config_opts, uidManager, plugins, state, buildroot, bootstrap_buildroot) + + # TODO: The printrootpath and list_snapshots logic escape the finalization + # like 'bootstrap.finalize()' or 'state.alldone()'. Move it into + # run_command(). + state.start("run") + + if options.printrootpath: + print(buildroot.make_chroot_path('')) + sys.exit(0) + + if options.list_snapshots: + plugins.call_hooks('list_snapshots', required=True) + if bootstrap_buildroot is not None: + bootstrap_buildroot.plugins.call_hooks('list_snapshots', required=True) + sys.exit(0) + + # dump configuration to log + log.debug("mock final configuration:") + for k, v in list(config_opts.items()): + log.debug(" %s: %s", k, v) + + os.umask(0o02) + os.environ["HOME"] = buildroot.homedir + + # New namespace starting from here + unshare_namespace(config_opts) + + if config_opts['hostname']: + util.sethostname(config_opts['hostname']) + + # set personality (ie. setarch) + util.condPersonality(config_opts['target_arch']) + + result = 0 + try: + result = run_command(options, args, config_opts, commands, buildroot) + # finish state.log if no exception was raised in run_command() + state.finish("run") + state.alldone() + finally: + buildroot.finalize() + if bootstrap_buildroot is not None: + bootstrap_buildroot.finalize() + return result + + +@traceLog() +def run_command(options, args, config_opts, commands, buildroot): + result = 0 + # TODO separate this + # Fetch and prepare sources from SCM + if config_opts['scm']: + try: + # pylint: disable=import-outside-toplevel + from mockbuild import scm + except ImportError as e: + raise mockbuild.exception.BadCmdline( + "Mock SCM module not installed: %s. You should install package mock-scm." % e) + scmWorker = scm.scmWorker(log, config_opts, config_opts['macros']) + with buildroot.uid_manager: + scmWorker.get_sources() + (options.sources, options.spec) = scmWorker.prepare_sources() + + if options.mode == 'init': + if config_opts['clean']: + commands.clean() + commands.init() + + elif options.mode == 'clean': + if len(options.scrub) == 0: + commands.clean() + else: + commands.scrub(options.scrub) + + elif options.mode == 'chain': + if len(args) == 0: + log.critical("You must specify an SRPM file with --chain") + return 50 + commands.init(do_log=True) + result = commands.chain(args, options, buildroot) + + elif options.mode == 'shell': + if len(args): + cmd = args + else: + cmd = None + commands.init(do_log=False) + return commands.shell(options, cmd) + + elif options.mode == 'chroot': + if len(args) == 0: + log.critical("You must specify a command to run with --chroot") + return 50 + commands.init(do_log=True) + return commands.chroot(args, options) + + elif options.mode == 'installdeps': + if len(args) == 0: + log.critical("You must specify an SRPM file with --installdeps") + return 50 + commands.init() + rpms = [] + for file in args: + if os.path.splitext(file)[1] == ".spec": + commands.installSpecDeps(file) + else: + rpms.append(file) + if rpms: + util.checkSrpmHeaders(rpms, plainRpmOk=1) + commands.installSrpmDeps(*rpms) + + elif options.mode == 'install': + if len(args) == 0: + log.critical("You must specify a package list to install.") + return 50 + + commands.init() + commands.install_external(args) + buildroot.install(*args) + + elif options.mode == 'update': + commands.init() + buildroot.pkg_manager.execute('update', *args) + + elif options.mode == 'remove': + if len(args) == 0: + log.critical("You must specify a package list to remove.") + return 50 + commands.init() + buildroot.remove(*args) + + elif options.mode == 'rebuild': + if options.hermetic_build: + # No caches with hermetic builds! Bootstrap is extracted from + # given tarball, buildroot installed from pre-fetched RPMs. + commands.scrub(["all"]) + + if config_opts['scm'] or (options.spec and options.sources): + srpm = mockbuild.rebuild.do_buildsrpm(config_opts, commands, buildroot, options, args) + if srpm: + args.append(srpm) + if config_opts['scm']: + scmWorker.clean() + options.spec = None + options.sources = None + else: + config_opts['clean'] = False + + try: + srpms = [] + for srpm_location in args: + with buildroot.uid_manager: + srpm = FileDownloader.get(srpm_location) + if not srpm: + raise mockbuild.exception.BadCmdline( + "Invalid {} source RPM".format(srpm_location)) + srpms.append(srpm) + mockbuild.rebuild.do_rebuild(config_opts, commands, buildroot, options, srpms) + finally: + FileDownloader.cleanup() + + elif options.mode == 'buildsrpm': + mockbuild.rebuild.do_buildsrpm(config_opts, commands, buildroot, options, args) + + elif options.mode == 'debugconfig': + do_debugconfig(config_opts) + + elif options.mode == 'debugconfigexpand': + do_debugconfig(config_opts, True) + + elif options.mode == 'listchroots': + do_listchroots(config_opts["config_path"], buildroot.uid_manager) + + elif options.mode == 'orphanskill': + util.orphansKill(buildroot.make_chroot_path()) + + elif options.mode == 'copyin': + commands.init() + if len(args) < 2: + log.critical("Must have source and destinations for copyin") + return 50 + dest = buildroot.make_chroot_path(args[-1]) + if len(args) > 2 and not os.path.isdir(dest): + log.critical("multiple source files and %s is not a directory!", dest) + return 50 + args = args[:-1] + for src in args: + if not os.path.lexists(src): + log.critical("No such file or directory: %s", src) + return 50 + log.info("copying %s to %s", src, dest) + if os.path.isdir(src): + dest2 = dest + if os.path.exists(dest2): + path_suffix = os.path.split(src)[1] + dest2 = os.path.join(dest2, path_suffix) + if os.path.exists(dest2): + log.critical("Destination %s already exists!", dest2) + return 50 + shutil.copytree(src, dest2) + else: + shutil.copy(src, dest) + buildroot.chown_home_dir() + + elif options.mode == 'copyout': + commands.init() + with buildroot.uid_manager: + if len(args) < 2: + log.critical("Must have source and destinations for copyout") + return 50 + dest = args[-1] + sources = [] + for arg in args[:-1]: + with util.env_var_override("HOME", buildroot.homedir): + arg = os.path.expanduser(arg) + matches = glob.glob(buildroot.make_chroot_path(arg)) + if not matches: + log.critical("%s not found", arg) + return 50 + sources += matches + if len(sources) > 1 and not os.path.isdir(dest): + log.critical("multiple source files and %s is not a directory!", dest) + return 50 + for src in sources: + log.info("copying %s to %s", src, dest) + if os.path.isdir(src): + shutil.copytree(src, dest, symlinks=True) + else: + if os.path.islink(src): + linkto = os.readlink(src) + os.symlink(linkto, dest) + else: + shutil.copy(src, dest) + + elif options.mode in ('pm-cmd', 'yum-cmd', 'dnf-cmd'): + log.info('Running %s %s', options.mode, ' '.join(args)) + commands.init() + buildroot.pkg_manager.execute(*args) + elif options.mode == 'snapshot': + if len(args) < 1: + log.critical("Requires a snapshot name") + return 50 + buildroot.plugins.call_hooks('make_snapshot', args[0], required=True) + if buildroot.bootstrap_buildroot is not None: + buildroot.bootstrap_buildroot.plugins.call_hooks('make_snapshot', args[0], required=True) + elif options.mode == 'rollback-to': + if len(args) < 1: + log.critical("Requires a snapshot name") + return 50 + buildroot.plugins.call_hooks('rollback_to', args[0], required=True) + if buildroot.bootstrap_buildroot is not None: + buildroot.bootstrap_buildroot.plugins.call_hooks('rollback_to', args[0], required=True) + elif options.mode == 'remove_snapshot': + if len(args) < 1: + log.critical("Requires a snapshot name") + return 50 + buildroot.plugins.call_hooks('remove_snapshot', args[0], required=True) + if buildroot.bootstrap_buildroot is not None: + buildroot.bootstrap_buildroot.plugins.call_hooks('remove_snapshot', args[0], required=True) + elif options.mode == 'umount': + buildroot.plugins.call_hooks('umount_root') + if buildroot.bootstrap_buildroot is not None: + buildroot.bootstrap_buildroot.plugins.call_hooks('umount_root') + elif options.mode == 'mount': + buildroot.plugins.call_hooks('mount_root') + if buildroot.bootstrap_buildroot is not None: + buildroot.bootstrap_buildroot.plugins.call_hooks('mount_root') + + buildroot.nuke_rpm_db() + return result + + +if __name__ == '__main__': + # TODO: this was documented as "fix for python 2.4 logging module bug:" + # TODO: ...but it is apparently still needed; without it there are various + # TODO: exceptions from trace_decorator like: + # TODO: TypeError: not enough arguments for format string + logging.raiseExceptions = 0 + + exitStatus = 0 + + + try: + exitStatus = main() + + except (SystemExit,): + raise # pylint: disable=try-except-raise + + except (OSError,) as e: + if e.errno == errno.EPERM: + print() + log.error("%s", e) + print() + log.error("The most common cause for this error is trying to run " + "/usr/libexec/mock/mock as an unprivileged user.") + log.error("You should not run /usr/libexec/mock/mock directly.") + print() + exitStatus = 2 + else: + raise + + except (KeyboardInterrupt,): + exitStatus = 7 + log.error("Exiting on user interrupt, -C") + + except (mockbuild.exception.ResultDirNotAccessible,) as exc: + exitStatus = exc.resultcode + log.error(str(exc)) + + except (mockbuild.exception.BadCmdline, mockbuild.exception.BuildRootLocked) as exc: + exitStatus = exc.resultcode + log.error(str(exc)) + + except (mockbuild.exception.Error) as exc: + exitStatus = exc.resultcode + log.error(str(exc)) + + except (Exception,) as exc: # pylint: disable=broad-except + exitStatus = 1 + log.exception(exc) + + logging.shutdown() + sys.exit(exitStatus) diff --git a/mock/py/mockbuild/__init__.py b/mock/py/mockbuild/__init__.py new file mode 100644 index 0000000..e8150a0 --- /dev/null +++ b/mock/py/mockbuild/__init__.py @@ -0,0 +1 @@ +"""All of the mock utility classes.""" diff --git a/mock/py/mockbuild/backend.py b/mock/py/mockbuild/backend.py new file mode 100644 index 0000000..63b063c --- /dev/null +++ b/mock/py/mockbuild/backend.py @@ -0,0 +1,848 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Originally written by Seth Vidal +# Sections taken from Mach by Thomas Vander Stichele +# Major reorganization and adaptation by Michael Brown +# Copyright (C) 2007 Michael E Brown + +import glob +import os +import shutil +import sys +import tempfile +import getpass + +# 3rd party imports +import rpm +from mockbuild.mounts import BindMountPoint + +from . import file_util +from . import text +from . import util +from .external import ExternalDeps +from .file_downloader import FileDownloader +from .exception import PkgError, Error, RootError, BuildError +from .trace_decorator import getLog, traceLog +from .rebuild import do_rebuild + + +class Commands(object): + """Executes mock commands in the buildroot""" + + @traceLog() + def __init__(self, config, uid_manager, plugins, state, buildroot, bootstrap_buildroot): + self.uid_manager = uid_manager + self.buildroot = buildroot + self.bootstrap_buildroot = bootstrap_buildroot + self.state = state + self.plugins = plugins + self.config = config + self.external = ExternalDeps(buildroot, bootstrap_buildroot, uid_manager) + + self.rpmbuild_arch = config['rpmbuild_arch'] + self.clean_the_chroot = config['clean'] + + self.build_results = [] + + # config options + self.configs = config['config_paths'] + self.config_name = config['chroot_name'] + self.buildroot.chrootuid = config['chrootuid'] + self.buildroot.chrootgid = config['chrootgid'] + self.use_host_resolv = config['use_host_resolv'] + self.chroot_file_contents = config['files'] + self.nspawn_args = config['nspawn_args'] + self.more_buildreqs = config['more_buildreqs'] + self.cache_alterations = config['cache_alterations'] + + self.backup = config['backup_on_clean'] + self.backup_base_dir = config['backup_base_dir'] + + # do we allow interactive root shells? + self.no_root_shells = config['no_root_shells'] + + self.private_network = not config['rpmbuild_networking'] + self.rpmbuild_noclean_option = None + + @traceLog() + def backup_results(self): + srcdir = os.path.join(self.buildroot.basedir, "result") + if not os.path.exists(srcdir): + return + dstdir = os.path.join(self.backup_base_dir, self.config['root']) + file_util.mkdirIfAbsent(dstdir) + rpms = glob.glob(os.path.join(srcdir, "*rpm")) + if len(rpms) == 0: + return + self.state.state_log.info("backup_results: saving with cp %s %s", " ".join(rpms), dstdir) + util.run(cmd="cp %s %s" % (" ".join(rpms), dstdir)) + + @traceLog() + def clean(self): + """clean out chroot with extreme prejudice :)""" + if self.backup: + self.backup_results() + self.state.start("clean chroot") + self.buildroot.delete() + self.state.finish("clean chroot") + + @traceLog() + def scrub(self, scrub_opts): + """clean out chroot and/or cache dirs with extreme prejudice :)""" + statestr = "scrub %s" % scrub_opts + self.state.start(statestr) + try: + try: + self.plugins.call_hooks('clean') + if self.bootstrap_buildroot is not None: + self.bootstrap_buildroot.plugins.call_hooks('clean') + + for scrub in scrub_opts: + self.plugins.call_hooks('scrub', scrub) + if self.bootstrap_buildroot is not None: + self.bootstrap_buildroot.plugins.call_hooks('scrub', scrub) + + if scrub == 'all': + self.buildroot.root_log.info("scrubbing everything for %s", self.config_name) + self.buildroot.delete() + file_util.rmtree(self.buildroot.cachedir, selinux=self.buildroot.selinux) + if self.bootstrap_buildroot is not None: + self.bootstrap_buildroot.delete() + file_util.rmtree(self.bootstrap_buildroot.cachedir, + selinux=self.bootstrap_buildroot.selinux) + elif scrub == 'chroot': + self.buildroot.root_log.info("scrubbing chroot for %s", self.config_name) + self.buildroot.delete() + elif scrub == 'cache': + self.buildroot.root_log.info("scrubbing cache for %s", self.config_name) + file_util.rmtree(self.buildroot.cachedir, selinux=self.buildroot.selinux) + elif scrub == 'c-cache': + self.buildroot.root_log.info("scrubbing c-cache for %s", self.config_name) + file_util.rmtree(os.path.join(self.buildroot.cachedir, 'ccache'), + selinux=self.buildroot.selinux) + elif scrub == 'root-cache': + self.buildroot.root_log.info("scrubbing root-cache for %s", self.config_name) + file_util.rmtree(os.path.join(self.buildroot.cachedir, 'root_cache'), + selinux=self.buildroot.selinux) + elif scrub in ['yum-cache', 'dnf-cache']: + self.buildroot.root_log.info("scrubbing yum-cache and dnf-cache for %s", self.config_name) + file_util.rmtree(os.path.join(self.buildroot.cachedir, 'yum_cache'), + selinux=self.buildroot.selinux) + file_util.rmtree(os.path.join(self.buildroot.cachedir, 'dnf_cache'), + selinux=self.buildroot.selinux) + elif scrub == 'bootstrap' and self.bootstrap_buildroot is not None: + self.buildroot.root_log.info("scrubbing bootstrap for %s", self.config_name) + self.bootstrap_buildroot.delete() + file_util.rmtree(self.bootstrap_buildroot.cachedir, selinux=self.bootstrap_buildroot.selinux) + + except IOError as e: + getLog().warning("parts of chroot do not exist: %s", e) + raise + finally: + self.state.finish(statestr) + + @traceLog() + def make_chroot_path(self, *args): + '''For safety reasons, self._rootdir should not be used directly. Instead + use this handy helper function anytime you want to reference a path in + relation to the chroot.''' + return self.buildroot.make_chroot_path(*args) + + @traceLog() + def init(self, do_log=True, **kwargs): + try: + if do_log: + self.buildroot.resetLogging() + + if self.bootstrap_buildroot is not None: + file_util.mkdirIfAbsent(self.buildroot.make_chroot_path()) + self.bootstrap_buildroot.initialize(**kwargs) + self.buildroot.initialize(**kwargs) + if not self.buildroot.chroot_was_initialized: + self._show_installed_packages() + except (KeyboardInterrupt, Exception): + self.plugins.call_hooks('initfailed') + # intentionally we do not call bootstrap hook here - it does not have sense + raise + + @traceLog() + def getPreconfiguredDeps(self, srpms): + """ + First check that some plugin didn't request installation of additional + packages into buildroot. + + Second, introspect the given array of SRPMs and check whether user did + not want to install additional packages for them + (config['more_buildreqs']). + + Return the list of additional requirements that should be installed. + """ + deps = list(self.buildroot.preexisting_deps) + + if not self.more_buildreqs: + # no need to analyze the src.rpm headers for NVR match + return deps + + # Check whether the N/NV/NVR isn't configured to have additional + # explicit BuildRequires. + for hdr in util.yieldSrpmHeaders(srpms, plainRpmOk=1): + # get text buildreqs + deps.extend(util.getAddtlReqs(hdr, self.more_buildreqs)) + + return deps + + @traceLog() + def installSrpmDeps(self, *srpms): + """Figure out deps from srpm. Call package manager to install them""" + try: + self.uid_manager.becomeUser(0, 0) + + deps = self.getPreconfiguredDeps(srpms) + if deps: + self.buildroot.pkg_manager.install(*deps, check=True) + + # install actual build dependencies + self.buildroot.pkg_manager.builddep(*srpms, check=True) + finally: + self.uid_manager.restorePrivs() + + @traceLog() + def installSpecDeps(self, spec_file): + try: + # pylint: disable=no-member + spec_file = util.host_file(spec_file) + spec = rpm.ds(rpm.spec(spec_file).sourceHeader, "requires") + self.uid_manager.becomeUser(0, 0) + for i in range(len(spec)): # pylint: disable=consider-using-enumerate + requirement_name = spec[i][2:] + self.buildroot.pkg_manager.install(requirement_name, check=True) + + finally: + self.uid_manager.restorePrivs() + + @traceLog() + def _show_installed_packages(self): + '''report the installed packages in the chroot to the root log''' + pkgs = sorted(self.buildroot.all_chroot_packages()) + self.buildroot.root_log.info("Installed packages:") + self.buildroot.root_log.info('\n'.join(pkgs)) + + @traceLog() + def install_external(self, requires): + """ requires is list of packages to be install. + This function extract any external:* and install them. + """ + external_deps = self.external.extract_external_deps(requires) + if external_deps: + if not self.config.get('use_bootstrap'): + raise Error('ExternalBuildRequires requires `use_bootstrap` to be set on.') + if self.config.get('external_buildrequires'): + self.external.install_external_deps(external_deps) + else: + raise Error('ExternalBuildRequires are found but support is disabled.' + ' See "external_buildrequires" in config_opts.') + # + # UNPRIVILEGED: + # Everything in this function runs as the build user + # -> except hooks. :) + # + @traceLog() + def build(self, srpm, timeout, check=True, spec=None): + """build an srpm into binary rpms, capture log""" + + # tell caching we are building + self.plugins.call_hooks('earlyprebuild') + # intentionally we do not call bootstrap hook here - it does not have sense + + baserpm = os.path.basename(srpm) + + buildstate = "build phase for %s" % baserpm + self.state.start(buildstate) + # remove rpm db files to prevent version mismatch problems + # note: moved to do this before the user change below! + self.buildroot.nuke_rpm_db() + dropped_privs = False + buildsetup_finished = False + try: + if not util.USE_NSPAWN: + self.uid_manager.becomeUser(self.buildroot.chrootuid, self.buildroot.chrootgid) + dropped_privs = True + buildsetup = "build setup for %s" % baserpm + self.state.start(buildsetup) + + srpm = self.copy_srpm_into_chroot(srpm) + self.install_srpm(srpm) + + if spec and not self.config['scm']: + # scm sets options.spec, but we want to get spec from SRPM when using scm + spec_path = self.copy_spec_into_chroot(spec) + else: + spec = self.get_specfile_name(srpm) + spec_path = os.path.join(self.buildroot.builddir, 'SPECS', spec) + + rebuilt_srpm = self.rebuild_installed_srpm(spec_path, timeout) + + # Check if we will have dynamic BuildRequires, but do not allow it + hdr = next(util.yieldSrpmHeaders((rebuilt_srpm,))) + # pylint: disable=no-member + requires = {text._to_text(req) for req in hdr[rpm.RPMTAG_REQUIRES]} + dynamic_buildreqs = 'rpmlib(DynamicBuildRequires)' in requires + + if dynamic_buildreqs and not self.config.get('dynamic_buildrequires'): + raise Error('DynamicBuildRequires are found but support is disabled.' + ' See "dynamic_buildrequires" in config_opts.') + + self.install_external(requires) + # install the (static) BuildRequires + self.installSrpmDeps(rebuilt_srpm) + self.state.finish(buildsetup) + buildsetup_finished = True + + rpmbuildstate = "rpmbuild %s" % baserpm + + # tell caching we are building + self.plugins.call_hooks('prebuild') + # intentionally we do not call bootstrap hook here - it does not have sense + + try: + self.state.start(rpmbuildstate) + results = self.rebuild_package(spec_path, timeout, check, dynamic_buildreqs) + finally: + self.state.finish(rpmbuildstate) + + # In the nspawn case, we retained root until here, but we + # need to ensure our output files are owned by the caller's uid. + # So drop them now. + if not dropped_privs: + self.uid_manager.becomeUser(self.buildroot.chrootuid, self.buildroot.chrootgid) + dropped_privs = True + if results: + self.build_results.extend(self.copy_build_results(results)) + elif self.config.get('short_circuit'): + self.buildroot.root_log.info("Short circuit builds don't produce RPMs") + else: + raise PkgError('No build results found') + self.state.result = 'success' + + finally: + if not buildsetup_finished: + self.state.finish(buildsetup) + self.state.finish(buildstate) + if dropped_privs: + self.uid_manager.restorePrivs() + if self.state.result != 'success': + self.state.result = 'fail' + # tell caching we are done building + self.plugins.call_hooks('postbuild') + # intentionally we do not call bootstrap hook here - it does not have sense + + + @traceLog() + def shell(self, options, cmd=None): + log = getLog() + log.debug("shell: calling preshell hooks") + self.plugins.call_hooks("preshell") + # intentionally we do not call bootstrap hook here - it does not have sense + if options.unpriv or self.no_root_shells: + uid = self.buildroot.chrootuid + gid = self.buildroot.chrootgid + else: + uid = 0 + gid = 0 + cwd = options.cwd + if not cwd: + # Hack! We don't set the cwd here because we know that we work + # with old systemd-nspawn without --chdir option. Still, users + # might use --chdir explicitly and such situation would still + # result in failure. rhbz#1976702 + if not util.USE_NSPAWN or util.check_nspawn_has_chdir_option(): + cwd = self.config['chroothome'] + + try: + self.state.start("shell") + ret = util.doshell(chrootPath=self.buildroot.make_chroot_path(), + environ=self.buildroot.env, uid=uid, gid=gid, + cwd=cwd, + nspawn_args=self.config.get("nspawn_args", []), + unshare_net=self.private_network, + cmd=cmd) + finally: + log.debug("shell: unmounting all filesystems") + self.state.finish("shell") + + log.debug("shell: calling postshell hooks") + self.plugins.call_hooks('postshell') + # intentionally we do not call bootstrap hook here - it does not have sense + return ret + + @traceLog() + def chroot(self, args, options): + log = getLog() + shell = False + if len(args) == 1: + args = [args[0]] + shell = True + log.info("Running in chroot: %s", args) + self.plugins.call_hooks("prechroot") + # intentionally we do not call bootstrap hook here - it does not have sense + chrootstate = "chroot %s" % args + self.state.start(chrootstate) + result=0 + try: + if options.unpriv: + result = self.buildroot.doChroot(args, shell=shell, printOutput=True, + uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, + user=self.buildroot.chrootuser, cwd=options.cwd, + raiseExc=False, + unshare_net=self.private_network)[1] + else: + result = self.buildroot.doChroot(args, shell=shell, cwd=options.cwd, + unshare_net=self.private_network, + printOutput=True, raiseExc=False)[1] + finally: + self.state.finish(chrootstate) + self.plugins.call_hooks("postchroot") + # intentionally we do not call bootstrap hook here - it does not have sense + return result + + @traceLog() + def chain(self, args, options, buildroot): + log = getLog() + if not options.tmp_prefix: + try: + options.tmp_prefix = getpass.getuser() + except Exception: # pylint: disable=broad-except + log.error("Could not find login name for tmp dir prefix add --tmp_prefix") + sys.exit(1) + pid = os.getpid() + self.config['uniqueext'] = '{0}-{1}'.format(options.tmp_prefix, pid) + + # create a tempdir for our local info + if options.localrepo: + local_tmp_dir = os.path.abspath(options.localrepo) + else: + pre = 'mock-chain-{0}-'.format(self.config['uniqueext']) + with self.uid_manager: + local_tmp_dir = tempfile.mkdtemp(prefix=pre, dir='/var/tmp') + + with self.uid_manager: + self.config['local_repo_dir'] = os.path.normpath( + local_tmp_dir + '/results/' + self.config['chroot_name'] + '/') + file_util.mkdirIfAbsent(self.config['local_repo_dir']) + + local_baseurl = "file://{0}".format(self.config['local_repo_dir']) + log.info("results dir: %s", self.config['local_repo_dir']) + # modify with localrepo + util.add_local_repo(self.config, local_baseurl, 'local_build_repo', + bootstrap=buildroot.bootstrap_buildroot) + + with self.uid_manager: + util.createrepo(self.config, self.config['local_repo_dir']) + + built_pkgs = [] + skipped_pkgs = [] + try_again = True + to_be_built = args + return_code = 0 + num_of_tries = 0 + while try_again: + num_of_tries += 1 + failed = [] + for pkg in to_be_built: + if failed and not options.cont: + log.error("Stopping the --chain build because --continue " + "isn't specified and the package '%s' failed " + "to build", failed[0]) + break + + if not pkg.endswith('.rpm'): + log.error("%s doesn't appear to be an rpm - skipping", pkg) + failed.append(pkg) + continue + + with self.uid_manager: + pkg_location = pkg + pkg = FileDownloader.get(pkg_location) + + if not pkg: + failed.append(pkg_location) + continue + + log.info("Start chain build: %s", pkg_location) + build_ret_code = 0 + try: + s_pkg = os.path.basename(pkg) + pdn = s_pkg.replace('.src.rpm', '') + resultdir = os.path.join(self.config['local_repo_dir'], pdn) + self.buildroot.resultdir = resultdir + self.buildroot.resetLogging(force=True) + success_file = os.path.join(resultdir, 'success') + build_ret_code = 0 + try: + if os.path.exists(success_file): + build_ret_code = 2 + else: + do_rebuild(self.config, self, buildroot, options, [pkg]) + except Error as err: + log.error(str(err)) + build_ret_code = 1 + except (RootError,) as e: + log.warning(e.msg) + failed.append(pkg) + log.info("End chain build: %s", pkg_location) + + with self.uid_manager: + if build_ret_code == 1: + failed.append(pkg) + log.info("Error building %s.", os.path.basename(pkg)) + if options.recurse: + log.info("Will try to build again (if some other package will succeed).") + else: + log.info("See logs/results in %s", self.config['local_repo_dir']) + file_util.touch(os.path.join(resultdir, 'fail')) + elif build_ret_code == 0: + log.info("Success building %s", os.path.basename(pkg)) + built_pkgs.append(pkg) + file_util.touch(success_file) + # createrepo with the new pkgs + util.createrepo(self.config, self.config['local_repo_dir']) + elif build_ret_code == 2: + log.info("Skipping already built pkg %s", os.path.basename(pkg)) + skipped_pkgs.append(pkg) + + if failed and options.recurse: + if len(failed) != len(to_be_built): + to_be_built = failed + try_again = True + log.info('Some package succeeded, some failed.') + log.info('Trying to rebuild %s failed pkgs, because --recurse is set.', len(failed)) + else: + log.info("Tried %s times - following pkgs could not be successfully built:", num_of_tries) + for pkg in failed: + msg = FileDownloader.original_name(pkg) + log.info(msg) + try_again = False + return_code = 4 + else: + try_again = False + if failed: + return_code = 4 + + FileDownloader.cleanup() + + log.info("Results out to: %s", self.config['local_repo_dir']) + if skipped_pkgs: + log.info("Packages skipped: %s", len(skipped_pkgs)) + for pkg in skipped_pkgs: + log.info(pkg) + log.info("Packages built: %s", len(built_pkgs)) + if built_pkgs: + if failed: + if len(built_pkgs): + log.info("Some packages successfully built in this order:") + else: + log.info("Packages successfully built in this order:") + for pkg in built_pkgs: + log.info(pkg) + return return_code + + # + # UNPRIVILEGED: + # Everything in this function runs as the build user + # -> except hooks. :) + # + @traceLog() + def buildsrpm(self, spec, sources, timeout, follow_links): + """build an srpm, capture log""" + + # tell caching we are building + self.plugins.call_hooks('earlyprebuild') + # intentionally we do not call bootstrap hook here - it does not have sense + + try: + self.uid_manager.becomeUser(self.buildroot.chrootuid, self.buildroot.chrootgid) + self.state.start("buildsrpm") + host_chroot_sources = None + + # copy spec/sources + shutil.copy(spec, self.buildroot.make_chroot_path(self.buildroot.builddir, "SPECS")) + + if sources: + # Resolve any symlinks + sources = os.path.realpath(sources) + if os.path.isdir(sources): + host_chroot_sources = self.buildroot.make_chroot_path( + self.buildroot.builddir, "SOURCES") + file_util.rmtree(host_chroot_sources) + + shutil.copytree(sources, + host_chroot_sources, + symlinks=(not follow_links)) + else: + host_chroot_sources = self.buildroot.make_chroot_path( + self.buildroot.builddir, "SOURCES", os.path.basename(sources)) + shutil.copy(sources, host_chroot_sources) + + spec = self.buildroot.make_chroot_path(self.buildroot.builddir, "SPECS", os.path.basename(spec)) + + self.plugins.call_hooks('pre_srpm_build', + spec, + host_chroot_sources) + + # get rid of rootdir prefix + chrootspec = spec.replace(self.buildroot.make_chroot_path(), '') + + self.state.start("rpmbuild -bs") + try: + rebuilt_srpm = self.rebuild_installed_srpm(chrootspec, timeout) + finally: + self.state.finish("rpmbuild -bs") + + srpm_basename = os.path.basename(rebuilt_srpm) + + self.buildroot.root_log.debug("Copying package to result dir") + shutil.copy2(rebuilt_srpm, self.buildroot.resultdir) + + return os.path.join(self.buildroot.resultdir, srpm_basename) + + finally: + self.uid_manager.restorePrivs() + + # tell caching we are done building + self.plugins.call_hooks('postbuild') + # intentionally we do not call bootstrap hook here - it does not have sense + self.state.finish("buildsrpm") + + # + # UNPRIVILEGED: + # Everything in this function runs as the build user + # + @traceLog() + def copy_srpm_into_chroot(self, srpm_path): + srpmFilename = os.path.basename(srpm_path) + dest = self.buildroot.make_chroot_path(self.buildroot.builddir, 'originals') + shutil.copyfile(srpm_path, os.path.join(dest, srpmFilename)) + return os.path.join(self.buildroot.builddir, 'originals', srpmFilename) + + @traceLog() + def copy_spec_into_chroot(self, spec_path): + specFilename = os.path.basename(spec_path) + dest = self.buildroot.make_chroot_path(self.buildroot.builddir, 'originals') + shutil.copy2(spec_path, os.path.join(dest, specFilename)) + return os.path.join(self.buildroot.builddir, 'originals', specFilename) + + @traceLog() + def get_specfile_name(self, srpm_path): + files = self.buildroot.doChroot([self.config['rpm_command'], "-qpl", srpm_path], + shell=False, uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, + unshare_net=self.private_network, + user=self.buildroot.chrootuser, + returnOutput=True + )[0] + specs = [item.rstrip() for item in files.split('\n') if item.rstrip().endswith('.spec')] + if len(specs) < 1: + raise PkgError( + "No specfile found in srpm: " + os.path.basename(srpm_path)) + return specs[0] + + @traceLog() + def install_srpm(self, srpm_path): + command = [self.config['rpm_command'], "-Uvh", "--nodeps", srpm_path] + output, return_code = self.buildroot.doChroot( + command, shell=False, uid=self.buildroot.chrootuid, + gid=self.buildroot.chrootgid, user=self.buildroot.chrootuser, + unshare_net=self.private_network, returnOutput=True, + returnStderr=True, raiseExc=False) + if return_code: + raise PkgError("Source RPM is not installable:\n{0}".format(output)) + + + @property + def _rpmbuild_noclean_option(self): + """ + Detect and cache if rpmbuild in buildroot supports the --noclean + option. Return "--noclean" string if supported, otherwise return an + empty string. + + TODO: Remove this method once nobody is building for RHEL 6. + """ + if self.config["cleanup_on_success"]: + return "" + + if self.rpmbuild_noclean_option is not None: + return self.rpmbuild_noclean_option + + self.rpmbuild_noclean_option = "" + _, status = self.buildroot.doChroot( + "case $(rpmbuild --help) in *--noclean*) exit 0; esac; exit 1", + shell=True, raiseExc=False + ) + if not status: + self.rpmbuild_noclean_option = "--noclean" + return self.rpmbuild_noclean_option + + + @traceLog() + def rebuild_installed_srpm(self, spec_path, timeout): + command = ['{command} -bs {0} --target {1} --nodeps {2}'.format( + self._rpmbuild_noclean_option, self.rpmbuild_arch, spec_path, + command=self.config['rpmbuild_command'])] + command = ["bash", "--login", "-c"] + command + self.buildroot.doChroot( + command, + shell=False, logger=self.buildroot.build_log, timeout=timeout, + uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, + user=self.buildroot.chrootuser, + unshare_net=self.private_network, + printOutput=self.config['print_main_output'] + ) + results = glob.glob("%s/%s/SRPMS/*src.rpm" % (self.make_chroot_path(), + self.buildroot.builddir)) + if len(results) != 1: + raise PkgError("Expected to find single rebuilt srpm, found %d in %s" + % (len(results), "%s/%s/SRPMS/*src.rpm" % (self.make_chroot_path(), + self.buildroot.builddir))) + return results[0] + + @traceLog() + def rebuild_package(self, spec_path, timeout, check, dynamic_buildrequires): + # --nodeps because rpm in the root may not be able to read rpmdb + # created by rpm that created it (outside of chroot) + check_opt = [] + calculatedeps = self.config["calculatedeps"] + if not check: + # this is because EL5/6 does not know --nocheck + # when EL5/6 targets are not supported, replace it with --nocheck + check_opt += ["--define", "'__spec_check_template exit 0; '"] + + mode = ['-bb'] + sc = self.config.get('short_circuit') + if sc: + mode[0] = {'prep': '-bp', + 'install': '-bi', + 'build': '-bc', + 'binary': '-bb'}[sc] + mode += ['--short-circuit'] + additional_opts = [self.config.get('rpmbuild_opts', '')] + if additional_opts == ['']: + additional_opts = [] + + def get_command(mode, checkdeps=False): + nodeps_opt = [] if checkdeps else ['--nodeps'] + command = [self.config['rpmbuild_command']] + mode + \ + [self._rpmbuild_noclean_option] + \ + ['--target', self.rpmbuild_arch] + nodeps_opt + \ + check_opt + [spec_path] + additional_opts + command = ["bash", "--login", "-c"] + [' '.join(command)] + return command + + bd_out = self.make_chroot_path(self.buildroot.builddir) + dynamic_buildrequires = dynamic_buildrequires and self.config.get('dynamic_buildrequires') + if dynamic_buildrequires: + max_loops = int(self.config.get('dynamic_buildrequires_max_loops')) + success = False + br_mode = ['-br'] + while not success and max_loops > 0: + # run rpmbuild+installSrpmDeps until + # * it fails + # * installSrpmDeps does nothing + # * or we run out of dynamic_buildrequires_max_loops tries + packages_before = self.buildroot.all_chroot_packages() + command = get_command(br_mode) + (output, returncode) = \ + self.buildroot.doChroot(command, + shell=False, logger=self.buildroot.build_log, timeout=timeout, + uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, + user=self.buildroot.chrootuser, + unshare_net=self.private_network, raiseExc=False, + printOutput=self.config['print_main_output']) + if returncode > 0 and returncode != 11: + # we treat exit status 11 as success, as well as exit + # status 0, see issue#434 + raise BuildError("Command failed: \n # %s\n%s" % (command, output)) + max_loops -= 1 + self.buildroot.build_log.info("Dynamic buildrequires detected") + self.buildroot.build_log.info("Going to install missing buildrequires. See root.log for details.") + self.buildroot.root_log.info("Going to install missing dynamic buildrequires") + buildreqs = glob.glob(bd_out + '/SRPMS/*.buildreqs.nosrc.rpm') + self.installSrpmDeps(*buildreqs) + packages_after = self.buildroot.all_chroot_packages() + if packages_after == packages_before: + success = True + for f_buildreqs in buildreqs: + if not (success and calculatedeps): + # we want to keep the nosrc.rpm file + os.remove(f_buildreqs) + # The first rpmbuild -br already did %prep, so we don't need waste time + if '--noprep' not in br_mode: + br_mode += ['--noprep'] + if not sc: + # We want to (re-)write src.rpm with dynamic BuildRequires, + # but with short-circuit it doesn't matter + mode = ['-ba'] + mode += ['--noprep'] + + self.plugins.call_hooks('postdeps') + + # When we used dynamic buildrequires, the rpmbuild call will + # execute the %generate_buildrequires section once again + # in order to actually add the BuildRequires to the SRPM metadata. + # Since the output of the %generate_buildrequires section isn't + # controlled by mock, it is possible (albeit unlikely) that the list + # of generated BuildRequires will differ now, as it is not guaranteed + # to remain stable. + # By instructing rpmbuild to check dependencies, we ensure a failure + # if a new unsatisfied dependency is generated. + # Unfortunately, we can only do this when using a bootstrap chroot, + # because the rpm in the chroot might not understand the rpmdb otherwise. + # See https://github.com/rpm-software-management/mock/issues/1246 + + if not calculatedeps: + checkdeps = dynamic_buildrequires and self.bootstrap_buildroot is not None + self.buildroot.doChroot(get_command(mode, checkdeps=checkdeps), + shell=False, logger=self.buildroot.build_log, timeout=timeout, + uid=self.buildroot.chrootuid, gid=self.buildroot.chrootgid, + user=self.buildroot.chrootuser, + unshare_net=self.private_network, + printOutput=self.config['print_main_output']) + results = glob.glob(bd_out + '/RPMS/*.rpm') + results += glob.glob(bd_out + '/SRPMS/*.rpm') + self.buildroot.final_rpm_list = [os.path.basename(result) for result in results] + return results + + @traceLog() + def copy_build_results(self, results): + self.buildroot.root_log.debug("Copying packages to result dir") + ret = [] + try: + for item in results: + shutil.copy2(item, self.buildroot.resultdir) + ret.append(os.path.join(self.buildroot.resultdir, os.path.split(item)[1])) + except OSError as err: + raise Error(f"Can not copy {item} into resultdir {self.buildroot.resultdir}: {err}") from err + return ret + + @traceLog() + def install_build_results(self, results): + self.buildroot.root_log.info("Installing built packages") + + # Mount resultdir into bootstrap, so we can later install the build + # results from there using bootstrap package manager. + mount_context = util.nullcontext() + + if self.bootstrap_buildroot: + resultdir = self.buildroot.resultdir + bootstrap_resultdir = self.bootstrap_buildroot.make_chroot_path(resultdir) + results_bindmount = BindMountPoint(resultdir, bootstrap_resultdir, + options="private") + mount_context = results_bindmount.having_mounted() + + with self.uid_manager.elevated_privileges(): + with mount_context: + pkgs = [pkg for pkg in results if not pkg.endswith("src.rpm")] + try: + self.buildroot.install(*pkgs) + except: + self.buildroot.root_log.error("Installation of built packages failed") + raise diff --git a/mock/py/mockbuild/buildroot.py b/mock/py/mockbuild/buildroot.py new file mode 100644 index 0000000..bf1445f --- /dev/null +++ b/mock/py/mockbuild/buildroot.py @@ -0,0 +1,1135 @@ +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +from contextlib import contextmanager +import errno +import fcntl +import functools +import glob +import grp +import logging +import os +import pwd +import shutil +import stat +import tempfile +import uuid + +from . import file_util +from . import mounts +from . import text +from . import uid +from . import util +from .exception import (BuildRootLocked, Error, ResultDirNotAccessible, + BadCmdline, BootstrapError, RootError) +from .package_manager import package_manager +from .trace_decorator import getLog, traceLog +from .podman import Podman, PodmanError +from .shadow_utils import ShadowUtils + + +# pylint: disable=too-many-lines + + +def noop_in_bootstrap(f): + # pylint: disable=inconsistent-return-statements + def wrapper(self, *args, **kwargs): + if self.is_bootstrap: + getLog().debug("method {} skipped in bootstrap".format(f.__name__)) + return + return f(self, *args, **kwargs) + return wrapper + + +def call_just_once(f): + """ Assure that method returns None, and is called just once """ + @functools.lru_cache(maxsize=None) + @functools.wraps(f) + def wrapper(*args, **kwargs): + if f(*args, **kwargs) is not None: + raise RuntimeError(f"Method '{f.__name__}' can not return values") + return wrapper + + +class Buildroot(object): + # pylint: disable=too-many-public-methods,too-many-instance-attributes + @traceLog() + def __init__(self, config, uid_manager, state, plugins, bootstrap_buildroot=None, is_bootstrap=False): + self.config = config + self.uid_manager = uid_manager + self.shadow_utils = ShadowUtils(self) + self.state = state + self.plugins = plugins + self.bootstrap_buildroot = bootstrap_buildroot + self.is_bootstrap = is_bootstrap + self.shared_root_name = config['root'] + if 'unique-ext' in config: + config['root'] = "%s-%s" % (config['root'], config['unique-ext']) + self.root_name = config['root'] + self.mockdir = config['basedir'] + self.basedir = os.path.join(config['basedir'], config['root']) + self.rootdir = config['rootdir'] + + # don't mixup bootstrap && normal chroot root dirs + if is_bootstrap: + self.rootdir = os.path.join(self.basedir, 'root') + + self.resultdir = text.compat_expand_string(config['resultdir'], config) + + # In bootstrap buildroot, resultdir _should_ be basically unused (nobody + # looks there anyways). But it is actually used on many fronts -- e.g. + # by plugins, bootstrap logging, etc. + # + # By default, bootstrap buildroot has different resultdir from the + # normal buildroot, as it is set to `{{basedir}}/{{root}}/result'`. But + # things start to be much more complicated if user sets `--resuldir + # /static/dir` (or `config_opts["resultdir'`) and both buildroots start + # to fight against each other. + # + # Let's work-around this, and always set predictable resultdir for + # bootstrap. We intentionally pick a directory which we can create (and + # write into) under non-privileged user. + if is_bootstrap: + self.resultdir = os.path.join(self.basedir, 'results') + + self.homedir = config['chroothome'] + self.cache_topdir = config['cache_topdir'] + self.cachedir = os.path.join(self.cache_topdir, self.shared_root_name) + self.builddir = os.path.join(self.homedir, 'build') + self._lock_file = None + self.selinux = (not self.config['plugin_conf']['selinux_enable'] + and util.selinuxEnabled()) + + self.chrootuid = config['chrootuid'] + self.chrootuser = config['chrootuser'] + self.chrootgid = config['chrootgid'] + self.chrootgroup = config['chrootgroup'] + self.env = config['environment'] + self.env['HOME'] = self.homedir + proxy_env = util.get_proxy_environment(config) + self.env.update(proxy_env) + os.environ.update(proxy_env) + + image = 'bootstrap' if is_bootstrap else 'buildroot' + self.use_chroot_image = self.config[f"use_{image}_image"] + self.chroot_image = self.config[f"{image}_image"] + self.image_skip_pull = self.config[f"{image}_image_skip_pull"] + self.image_assert_digest = self.config.get(f"{image}_image_assert_digest", None) + self.image_keep_getting = self.config[f"{image}_image_keep_getting"] + + self.pkg_manager = None + self.mounts = mounts.Mounts(self) + + self.root_log = getLog("mockbuild") + self.build_log = getLog("mockbuild.Root.build") + self.logging_initialized = False + self.chroot_was_initialized = False + + self._homedir_bindmounts = {} + additional_packages = [self.wrap_host_file(f) for f in + config["additional_packages"] or []] + + if is_bootstrap: + self.preexisting_deps = [] + else: + self.preexisting_deps = additional_packages + + self.plugins.init_plugins(self) + self.tmpdir = None + self.nosync_path = None + self.final_rpm_list = None + + self._setup_nspawn_btrfs_device() + self._setup_nspawn_devicemapper_device() + self._setup_nspawn_fuse_device() + self._setup_nspawn_loop_devices() + + + def set_package_manager(self, fallback=None): + """ + (Re)Set the 'self.pkg_manager' object. This might be a preliminary + choice for bootstrap chroot if 'self.uses_bootstrap_image' because we + the image might contain a different package manager by default. + Typically when 'dnf5' is desired, but 'dnf' only is baked into the + image. So there are 5 situations when this method may be called: + + 1. self is bootstrap and we install by package manager from host, and + we try to to use any of the package managers on host (fallback) + 2. self is bootstrap generated from a container image, first we try to + use any available package manager in such image (fallback) to install + the desired one + 3. self is bootstrap, generated from image, but we already have the + required package manager installed (fallback=False) + 4. self is buildroot, and bootstrapping is off - we use the best + matching package manager on host (fallback) but we do warn + 5. self is buildroot, but we install from a pre-prepared bootstrap + chroot where we already have the desired package manager (there's no + fallback/guessing) + """ + + if fallback is None: + # True for situation 1, 2, 4. False for 5. We do not enter this if + # condition for situation 3. + fallback = self.is_bootstrap or not self.bootstrap_buildroot + + old_name = None if not self.pkg_manager else self.pkg_manager.name + self.pkg_manager = package_manager(self, self.bootstrap_buildroot, fallback) + + fallback = "fallback" if fallback else "direct choice" + name = self.pkg_manager.name + if not old_name: + getLog().info("Package manager %s detected and used (%s)", name, + fallback) + elif old_name != name: + getLog().info("Switching package manager from %s to the %s (%s)", + old_name, name, fallback) + + @traceLog() + def make_chroot_path(self, *paths): + new_path = self.rootdir + for path in paths: + if path.startswith('/'): + path = path[1:] + new_path = os.path.join(new_path, path) + return new_path + + @traceLog() + def initialize(self, prebuild=False): + """ + Initialize the buildroot to a point where it's possible to execute + commands in chroot. If it was already initialized, just lock the shared + lock. + """ + try: + self._lock_buildroot(exclusive=True) + self._init(prebuild=prebuild) + except BuildRootLocked: + self._init_locked() + finally: + self._lock_buildroot(exclusive=False) + + @traceLog() + def chroot_is_initialized(self): + return os.path.exists(self.make_chroot_path('.initialized')) + + @call_just_once + def _setup_basedir(self): + file_util.mkdirIfAbsent(self.basedir) + mockgid = grp.getgrnam('mock').gr_gid + os.chown(self.basedir, os.getuid(), mockgid) + os.chmod(self.basedir, 0o775) + + @traceLog() + def create_resultdir(self): + """ + (re)create self.resultdir directory with appropriate permissions + """ + self._setup_basedir() + with self.uid_manager: + try: + file_util.mkdirIfAbsent(self.resultdir) + except Error as err: + raise ResultDirNotAccessible( + ResultDirNotAccessible.__doc__ % self.resultdir + ) from err + + + @traceLog() + def _init_locked(self): + """ + Things to do if other mock process initialized the Buidlroot. Stuff + which would otherwise happen in self._init(). + """ + # Detect what package manager to use. + self.set_package_manager() + + @traceLog() + def _load_from_container_image(self): + if not self.use_chroot_image or self.chroot_was_initialized: + return + + if util.mock_host_environment_type() == "docker": + getLog().info( + "It seems that you run Mock in a Docker container. Mock " + "though uses container tooling itself (namely Podman) for " + "downloading container image. This might require you to " + "run Mock in 'docker run --privileged'.") + + class _FallbackException(Exception): + pass + + @contextmanager + def _fallback(message): + try: + yield + except PodmanError as exc: + if not self.config["image_fallback"]: + err = BootstrapError if self.is_bootstrap else RootError + raise err from exc + raise _FallbackException( + f"{message}, falling back to chroot installation: {exc}" + ) from exc + + try: + with _fallback("Can't work with Podman"): + podman = Podman(self, self.chroot_image) + + with _fallback("Can't initialize from container image"): + if not self.image_skip_pull: + podman.retry_image_pull(self.image_keep_getting) + else: + podman.read_image_id() + getLog().info("Using local image %s (%s)", + self.chroot_image, podman.image_id) + podman.tag_image() + + digest_expected = self.config.get("image_assert_digest") + if digest_expected: + getLog().info("Checking image digest: %s", + digest_expected) + digest = podman.get_oci_digest() + if digest != digest_expected: + raise BootstrapError( + f"Expected digest for image {podman.image} is" + f"{digest_expected}, but {digest} found.") + + if self.is_bootstrap and not podman.check_native_image_architecture(): + raise BootstrapError("Container image architecture check failed") + + podman.cp(self.make_chroot_path(), self.config["tar_binary"]) + podman.untag() + file_util.unlink_if_exists(os.path.join(self.make_chroot_path(), + "etc/rpm/macros.image-language-conf")) + except _FallbackException as exc: + getLog().warning("%s", exc) + self.use_chroot_image = False + + + @traceLog() + def _init(self, prebuild): + self.state.start("chroot init") + self._setup_basedir() + file_util.mkdirIfAbsent(self.make_chroot_path()) + self.plugins.call_hooks('mount_root') + # intentionally we do not call bootstrap hook here - it does not have sense + self._setup_nosync() + self.chroot_was_initialized = self.chroot_is_initialized() + self.create_resultdir() + getLog().info("calling preinit hooks") + self.plugins.call_hooks('preinit') + # intentionally we do not call bootstrap hook here - it does not have sense + self.chroot_was_initialized = self.chroot_is_initialized() + self._load_from_container_image() + self._setup_dirs() + + # /dev is later overwritten by systemd-nspawn, but we need this for + # initial installation when chroot is empty + self._setup_devices() + + self._setup_files() + + # write out config details + self.root_log.debug('rootdir = %s', self.make_chroot_path()) + self.root_log.debug('resultdir = %s', self.resultdir) + + self.set_package_manager() + + # this creates some managed mounts + self.pkg_manager.initialize() + + self.mounts.mountall_managed() + + self._setup_resolver_config() + self._setup_katello_ca() + + if prebuild: + self.pkg_manager.log_package_management_packages() + + if not self.chroot_was_initialized: + self._setup_dbus_uuid() + self._init_aux_files() + if not util.USE_NSPAWN: + self._setup_timezone() + self._init_pkg_management() + self._setup_files_postinstall() + self._setup_build_dirs() + elif prebuild: + if 'age_check' in self.config['plugin_conf']['root_cache_opts'] and \ + not self.config['plugin_conf']['root_cache_opts']['age_check']: + self._init_pkg_management() + if (self.config['online'] and self.config['update_before_build'] + and self.config['clean']): + update_state = "{0} update".format(self.pkg_manager.name) + self.state.start(update_state) + packages_before = self.all_chroot_packages() + self.pkg_manager.update() + packages_after = self.all_chroot_packages() + if packages_before != packages_after: + new_packages = "\n".join(packages_after - packages_before) + self.root_log.info("Calling postupdate hooks because there " + "are new/updated packages:\n%s", + new_packages) + self.plugins.call_hooks('postupdate') + self.state.finish(update_state) + + # cleanup a potential mess after the previous build + self._cleanup_homedir() + self._setup_build_dirs() + + # (re)create users to ensure the uid/gid are up to date with config + # after doing 'dnf update', 'dnf builddep', etc. + self._make_users() + + # Change owner of homdir tree if the root of it not owned + # by the current user + if prebuild: + self.chown_home_dir() + else: + self.chown_home_dir(recursive=False) + + # mark the buildroot as initialized + file_util.touch(self.make_chroot_path('.initialized')) + + # done with init + self.plugins.call_hooks('postinit') + # intentionally we do not call bootstrap hook here - it does not have sense + + self.mounts.mountall_user() + + self.state.finish("chroot init") + + def doOutChroot(self, command, *args, **kwargs): + """ + Execute the command in bootstrap chroot (when bootstrap is enabled) or + on host. Return (output, exit_status) tuple. + """ + + # the chrootPath would imply running chroot within containers, as well + # as on host (where we would have to setup nspawn_args, which is not + # implemented). + assert "chrootPath" not in kwargs + + if self.bootstrap_buildroot: + with self.mounts.buildroot_in_bootstrap_mounted(): + return self.bootstrap_buildroot.doChroot( + command, *args, **kwargs) + + return util.do_with_status(command, *args, **kwargs) + + def doChroot(self, command, nosync=False, *args, **kargs): + """Execute given command in root. Returns (output, child.returncode)""" + self.nuke_rpm_db() + env = dict(self.env) + if nosync and self.nosync_path: + env['LD_PRELOAD'] = self.nosync_path + if util.USE_NSPAWN: + if 'uid' not in kargs: + kargs['uid'] = uid.getresuid()[1] + if 'gid' not in kargs: + kargs['gid'] = uid.getresgid()[1] + self.uid_manager.becomeUser(0, 0) + + kargs.setdefault("nspawn_args", []) + kargs["nspawn_args"].extend(self.config.get("nspawn_args", [])) + + try: + result = util.do_with_status(command, chrootPath=self.make_chroot_path(), + env=env, *args, **kargs) + finally: + if util.USE_NSPAWN: + self.uid_manager.restorePrivs() + return result + + def doChrootPlugin(self, command, cwd=None): + """ + Execute command (specified as array, not a shell command string) in this + buildroot in `cwd`, as a non-privileged user. Used by plugins. + """ + private_network = not self.config.get('rpmbuild_networking', False) + self.doChroot( + command, + shell=False, + cwd=cwd, + logger=self.build_log, + uid=self.chrootuid, + gid=self.chrootgid, + user=self.chrootuser, + unshare_net=private_network, + printOutput=self.config.get('print_main_output', True) + ) + + def all_chroot_packages(self): + """package set, result of rpm -qa in the buildroot""" + self.nuke_rpm_db() + command = [self.config['rpm_command'], "-qa", + "--root", self.make_chroot_path()] + out, _ = self.doOutChroot(command, returnOutput=True, printOutput=False, + shell=False) + return set(out.splitlines()) + + @traceLog() + def _copy_config(self, filename, symlink=False, warn=True): + orig_conf_file = os.path.join('/etc', filename) + conf_file = self.make_chroot_path(orig_conf_file) + + try: + os.remove(conf_file) + except FileNotFoundError: + pass + + # for /etc sub-directories + file_util.mkdirIfAbsent(os.path.dirname(conf_file)) + + if os.path.exists(orig_conf_file): + if symlink and os.path.islink(orig_conf_file): + linkto = os.readlink(orig_conf_file) + os.symlink(linkto, conf_file) + else: + shutil.copy2(orig_conf_file, conf_file) + elif warn: + self.root_log.warning("File %s not present. It is not copied into the chroot.", orig_conf_file) + + @traceLog() + def _setup_resolver_config(self): + if self.config['use_host_resolv'] and self.config['rpmbuild_networking']: + self._copy_config('resolv.conf') + self._copy_config('hosts') + + @traceLog() + def _setup_katello_ca(self): + if not all([self.is_bootstrap, + self.config["redhat_subscription_required"]]): + return + self._copy_config('rhsm/ca/katello-server-ca.pem', warn=False) + + @traceLog() + def _setup_dbus_uuid(self): + machine_uuid = uuid.uuid4().hex + dbus_uuid_path = self.make_chroot_path('etc', 'machine-id') + symlink_path = self.make_chroot_path('var', 'lib', 'dbus', 'machine-id') + with open(dbus_uuid_path, 'w') as uuid_file: + uuid_file.write(machine_uuid) + uuid_file.write('\n') + if not os.path.exists(symlink_path): + os.symlink("../../../etc/machine-id", symlink_path) + + @traceLog() + def _setup_timezone(self): + self._copy_config('localtime', symlink=True) + + @staticmethod + def _module_commands_from_config(config): + commands = [] + for config_command in config: + action, raw_modules = config_command + if raw_modules: + modules = str(raw_modules).strip() + modules = [m.strip() for m in modules.split(",")] + else: + modules = [] + cmd = ["module", action] + modules + commands.append(cmd) + return commands + + @traceLog() + def _module_setup(self): + if 'module_enable' in self.config and self.config['module_enable']: + cmd = ['module', 'enable'] + self.config['module_enable'] + self.pkg_manager.init_install_output += self.pkg_manager.execute(*cmd, returnOutput=1) + + if 'module_install' in self.config and self.config['module_install']: + cmd = ['module', 'install'] + self.config['module_install'] + self.pkg_manager.init_install_output += self.pkg_manager.execute(*cmd, returnOutput=1) + + module_config = self.config["module_setup_commands"] or [] + for cmd in self._module_commands_from_config(module_config): + self.pkg_manager.init_install_output += self.pkg_manager.execute(*cmd, returnOutput=1) + + @traceLog() + def _init_pkg_management(self): + # The desired package manager. + pm = self.config['package_manager'] + + if self.bootstrap_image_is_ready: + return + + if self.is_bootstrap: + update_state = f'installing {pm} tooling' + cmd = self.config.get(f"{pm}_install_command") + else: + update_state = f'installing minimal buildroot with {pm}' + cmd = self.config['chroot_setup_cmd'] + + self.state.start(update_state) + + self._module_setup() + + if cmd: + if isinstance(cmd, str): + cmd = cmd.split() + self.pkg_manager.init_install_output += self.pkg_manager.execute(*cmd, returnOutput=1) + + if self.uses_bootstrap_image: + # TODO: Call _module_setup() again? Depending on the + # implementation, we might want to reset the modules after switching + # to the desired manager. But DNF5 doesn't support modules, yet: + # https://github.com/rpm-software-management/dnf5/issues/146 + self.set_package_manager(fallback=False) + + if 'chroot_additional_packages' in self.config and self.config['chroot_additional_packages']: + cmd = self.config['chroot_additional_packages'] + if isinstance(cmd, str): + cmd = cmd.split() + cmd = ['install'] + cmd + self.pkg_manager.init_install_output += self.pkg_manager.execute(*cmd, returnOutput=1) + + self.state.finish(update_state) + + @noop_in_bootstrap + def _cleanup_homedir(self): + # Resolving the 'mockbuilder' user + excluded = [self.make_chroot_path(self.homedir, path) + for path in self.config['exclude_from_homedir_cleanup']] + \ + self.mounts.get_mountpoints() + file_util.rmtree(self.make_chroot_path(self.homedir), + selinux=self.selinux, exclude=excluded) + + @traceLog() + @noop_in_bootstrap + def _make_users(self): + # The mockbuild user + self.shadow_utils.delete_user(self.chrootuser, can_fail=True) + self.shadow_utils.delete_group(self.chrootgroup, can_fail=True) + self.shadow_utils.create_group(self.chrootgroup, gid=self.chrootgid) + self.shadow_utils.create_user( + self.chrootuser, uid=self.chrootuid, gid=self.chrootgid, + home=self.homedir, + ) + self._enable_chrootuser_account() + + # Other users + for user in self.config.get("copy_host_users", []): + self.shadow_utils.copy_from_host(user) + + + @traceLog() + def _enable_chrootuser_account(self): + passwd = self.make_chroot_path('/etc/passwd') + with open(passwd) as f: + lines = f.readlines() + disabled = False + newlines = [] + for l in lines: + parts = l.strip().split(':') + if parts[0] == self.chrootuser and parts[1].startswith('!!'): + disabled = True + parts[1] = parts[1][2:] + newlines.append(':'.join(parts)) + if disabled: + with open(passwd, "w") as f: + for l in newlines: + f.write(l + '\n') + + @traceLog() + def resetLogging(self, force=False): + """ + Pre-create a result directory for log files, initialize the log files + there, and identify Mock by dumping there it's version + """ + assert not self.is_bootstrap + + # ensure we dont attach the handlers multiple times. + if self.logging_initialized and not force: + return + self.logging_initialized = True + + self.create_resultdir() + + with self.uid_manager: + # attach logs to log files. + # This happens in addition to anything that + # is set up in the config file... ie. logs go everywhere + for (log, filename, fmt_str) in ( + (self.state.state_log, "state.log", self.config['state_log_fmt_str']), + (self.build_log, "build.log", self.config['build_log_fmt_str']), + (self.root_log, "root.log", self.config['root_log_fmt_str'])): + # release used FileHandlers if re-initializing to not leak FDs + if force: + for handler in log.handlers[:]: + handler.close() + log.removeHandler(handler) + fullPath = os.path.join(self.resultdir, filename) + fh = logging.FileHandler(fullPath, "a+") + formatter = logging.Formatter(fmt_str) + fh.setFormatter(formatter) + fh.setLevel(logging.NOTSET) + log.addHandler(fh) + log.info("Mock Version: %s", self.config['version']) + + @traceLog() + def _init_aux_files(self): + chroot_file_contents = self.config['files'] + for key in chroot_file_contents: + p = self.make_chroot_path(key) + if not os.path.exists(p): + file_util.mkdirIfAbsent(os.path.dirname(p)) + with open(p, 'w+') as fo: + fo.write(chroot_file_contents[key]) + + @traceLog() + def nuke_rpm_db(self): + """remove rpm DB lock files from the chroot""" + + dbfiles = glob.glob(self.make_chroot_path('var/lib/rpm/__db*')) + if not dbfiles: + return + self.root_log.debug("removing %d rpm db files", len(dbfiles)) + # become root + self.uid_manager.becomeUser(0, 0) + try: + for tmp in dbfiles: + self.root_log.debug("nuke_rpm_db: removing %s", tmp) + try: + os.unlink(tmp) + except OSError as e: + getLog().error("%s", e) + raise + finally: + self.uid_manager.restorePrivs() + + @traceLog() + def _open_lock(self): + file_util.mkdirIfAbsent(self.basedir) + self._lock_file = open(os.path.join(self.basedir, "buildroot.lock"), "a+") + + @traceLog() + def _lock_buildroot(self, exclusive): + lock_type = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH + if not self._lock_file: + self._open_lock() + try: + fcntl.lockf(self._lock_file.fileno(), lock_type | fcntl.LOCK_NB) + except IOError: + raise BuildRootLocked("Build root is locked by another process.") + + @traceLog() + def _unlock_buildroot(self): + if self._lock_file: + self._lock_file.close() + self._lock_file = None + + @traceLog() + def _setup_dirs(self): + self.root_log.debug('create skeleton dirs') + dirs = ['var/lib/rpm', + 'var/lib/yum', + 'var/lib/dbus', + 'var/log', + 'var/cache/dnf', + 'var/cache/yum', + 'etc/rpm', + 'tmp', + 'tmp/ccache', + 'var/tmp', + 'etc/dnf', + 'etc/dnf/vars', + 'etc/yum.repos.d', + 'etc/yum', + 'proc', + 'sys'] + dirs += self.config['extra_chroot_dirs'] + for item in dirs: + file_util.mkdirIfAbsent(self.make_chroot_path(item)) + + @traceLog() + def chown_home_dir(self, recursive=True): + """ set ownership of homedir and subdirectories to mockbuild user """ + self.uid_manager.changeOwner(self.make_chroot_path(self.homedir), + recursive=recursive) + + @traceLog() + def prepare_installation_time_homedir(self): + """ Create a fake home directory with an appropriate .rpmmacros. """ + + rpm_config_home = os.path.join(self.rootdir, "installation-homedir") + file_util.mkdirIfAbsent(rpm_config_home) + + # Since /proc and /sys are mounted special filesystems when RPM is running + # to install the buildroot, it doesn't make sense for RPM to try and + # set the permissions on them - and that might fail with permission errors. + with open(os.path.join(rpm_config_home, ".rpmmacros"), "w") as macro_fd: + macro_fd.write("%_netsharedpath /proc:/sys\n") + + # To make the DNF (and wrapped RPM) use the appropriate .rpmmacros file + # we need to set the $HOME environment variable to non-standard + # directory. But note that we don't just set HOME=/rpmconfig because + # `rpm --rootdir ` reads the macros file from caller's + # filesystem, so we set HOME=/var/lib/mock///rpmconfig. + return rpm_config_home + + @traceLog() + def _prepare_rpm_macros(self): + """ + Install the /builddir/.rpmmacros file used by /bin/rpmbuild at build + time. + """ + macro_dir = self.make_chroot_path(self.homedir) + file_util.mkdirIfAbsent(macro_dir) + macrofile_out = os.path.join(macro_dir, ".rpmmacros") + with open(macrofile_out, 'w+') as rpmmacros: + + # user specific from rpm macro file definitions first + if 'macrofile' in self.config: + with open(self.config['macrofile'], 'r') as macro_conf: + rpmmacros.write("%s\n\n" % macro_conf.read()) + + for key, value in list(self.config['macros'].items()): + rpmmacros.write("%s %s\n" % (key, value)) + + @traceLog() + @noop_in_bootstrap + def _setup_build_dirs(self): + build_dirs = ['RPMS', 'SPECS', 'SRPMS', 'SOURCES', 'BUILD', 'BUILDROOT', + 'originals'] + file_util.mkdirIfAbsent(self.make_chroot_path(self.builddir)) + with self.uid_manager: + self.uid_manager.changeOwner(self.make_chroot_path(self.builddir)) + for item in build_dirs: + path = self.make_chroot_path(self.builddir, item) + file_util.mkdirIfAbsent(path) + self.uid_manager.changeOwner(path) + self._prepare_rpm_macros() + + @traceLog() + def _setup_nspawn_btrfs_device(self): + if not util.USE_NSPAWN or self.is_bootstrap: + return + if os.path.exists('/dev/btrfs-control'): + self.config['nspawn_args'].append('--bind=/dev/btrfs-control') + + @traceLog() + def _setup_nspawn_devicemapper_device(self): + if not util.USE_NSPAWN or self.is_bootstrap: + return + if os.path.exists('/dev/mapper/control'): + self.config['nspawn_args'].append('--bind=/dev/mapper/control') + + @traceLog() + def _setup_nspawn_fuse_device(self): + if not util.USE_NSPAWN or self.is_bootstrap: + return + if os.path.exists('/dev/fuse'): + self.config['nspawn_args'].append('--bind=/dev/fuse') + + @traceLog() + def _setup_nspawn_loop_devices(self): + if not util.USE_NSPAWN or self.is_bootstrap: + return + + self.config['nspawn_args'].append('--bind=/dev/loop-control') + # for nspawn we create the loop devices directly on host + for i in range(self.config['dev_loop_count']): + loop_file = '/dev/loop{}'.format(i) + try: + os.mknod(loop_file, stat.S_IFBLK | 0o666, os.makedev(7, i)) + except OSError as e: + if e.errno != errno.EEXIST: + raise + self.config['nspawn_args'].append('--bind={0}'.format(loop_file)) + + @traceLog() + def _setup_devices(self): + if self.config['internal_dev_setup']: + file_util.rmtree(self.make_chroot_path("dev"), selinux=self.selinux, exclude=self.mounts.get_mountpoints()) + file_util.mkdirIfAbsent(self.make_chroot_path("dev", "pts")) + file_util.mkdirIfAbsent(self.make_chroot_path("dev", "shm")) + file_util.mkdirIfAbsent(self.make_chroot_path("dev", "mapper")) + prevMask = os.umask(0000) + devFiles = [ + (stat.S_IFCHR | 0o666, os.makedev(1, 3), "dev/null"), + (stat.S_IFCHR | 0o666, os.makedev(1, 7), "dev/full"), + (stat.S_IFCHR | 0o666, os.makedev(1, 5), "dev/zero"), + (stat.S_IFCHR | 0o666, os.makedev(1, 8), "dev/random"), + (stat.S_IFCHR | 0o444, os.makedev(1, 9), "dev/urandom"), + (stat.S_IFCHR | 0o666, os.makedev(5, 0), "dev/tty"), + (stat.S_IFCHR | 0o600, os.makedev(5, 1), "dev/console"), + (stat.S_IFCHR | 0o666, os.makedev(5, 2), "dev/ptmx"), + (stat.S_IFCHR | 0o666, os.makedev(10, 229), "dev/fuse"), + (stat.S_IFCHR | 0o660, os.makedev(10, 234), "dev/btrfs-control"), + (stat.S_IFCHR | 0o600, os.makedev(10, 236), "dev/mapper/control"), + (stat.S_IFCHR | 0o666, os.makedev(10, 237), "dev/loop-control"), + (stat.S_IFCHR | 0o600, os.makedev(10, 57), "dev/prandom"), + (stat.S_IFCHR | 0o600, os.makedev(10, 183), "dev/hwrng"), + ] + for i in range(self.config['dev_loop_count']): + devFiles.append((stat.S_IFBLK | 0o666, os.makedev(7, i), "dev/loop{loop_number}".format(loop_number=i))) + kver = os.uname()[2] + self.root_log.debug("kernel version == %s", kver) + for i in devFiles: + src_path = "/" + i[2] + chroot_path = self.make_chroot_path(i[2]) + + if util.cmpKernelVer(kver, '2.6.18') >= 0 and src_path == '/dev/ptmx': + continue + + if os.path.ismount(chroot_path): + # repeated call of bootstrap._init() in chain() in container + # where we can not mknod so we bindmount instead + self.root_log.debug("file %s is already mounted", chroot_path) + continue + + # create node, but only if it exist on host too + # except for loop devices, which only show up on the host after they are first used + if os.path.exists(src_path) or "loop" in src_path: + try: + os.mknod(chroot_path, i[0], i[1]) + except OSError as e: + # If mknod gives us a permission error, fall back to a different + # strategy of using a bind mount from root to host. This won't + # work for the loop devices, so just skip them in this case. + if e.errno == errno.EPERM: + if os.path.exists(src_path): + self.mounts.add_device_bindmount(src_path) + continue + raise + + # Further adjustments if we created a new node instead of bind-mounting + # an existing one: + + # set context. (only necessary if host running selinux enabled.) + # fails gracefully if chcon not installed. + if self.selinux: + util.do(["chcon", "--reference=" + src_path, chroot_path], + raiseExc=0, shell=False, env=self.env) + + if src_path in ('/dev/tty', '/dev/ptmx'): + os.chown(chroot_path, pwd.getpwnam('root')[2], grp.getgrnam('tty')[2]) + + os.symlink("/proc/self/fd/0", self.make_chroot_path("dev/stdin")) + os.symlink("/proc/self/fd/1", self.make_chroot_path("dev/stdout")) + os.symlink("/proc/self/fd/2", self.make_chroot_path("dev/stderr")) + + if os.path.isfile(self.make_chroot_path('etc', 'mtab')) or \ + os.path.islink(self.make_chroot_path('etc', 'mtab')): + os.remove(self.make_chroot_path('etc', 'mtab')) + os.symlink("../proc/self/mounts", self.make_chroot_path('etc', 'mtab')) + + # symlink /dev/fd in the chroot for everything except RHEL4 + if util.cmpKernelVer(kver, '2.6.9') > 0: + os.symlink("/proc/self/fd", self.make_chroot_path("dev/fd")) + + os.umask(prevMask) + + os.symlink("pts/ptmx", self.make_chroot_path('/dev/ptmx')) + + @traceLog() + def _setup_files(self): + # self.root_log.debug('touch required files') + for item in [self.make_chroot_path('etc', 'fstab'), + self.make_chroot_path('etc', 'yum', 'yum.conf'), + self.make_chroot_path('etc', 'dnf', 'dnf.conf'), + self.make_chroot_path('var', 'log', 'yum.log')]: + file_util.touch(item) + short_yum_confpath = self.make_chroot_path('etc', 'yum.conf') + if not os.path.exists(short_yum_confpath): + os.symlink("yum/yum.conf", short_yum_confpath) + + @traceLog() + def _setup_files_postinstall(self): + for item in [self.make_chroot_path('etc', 'os-release')]: + file_util.touch(item) + + @traceLog() + def _setup_nosync(self): + if not self.config['nosync']: + return + + multilib = ('x86_64', 's390x') + # ld_preload need to be same as in bootstrap because we call DNF in + # bootstrap, but it will load nosync from the final chroot + if self.bootstrap_buildroot is not None: + self.tmpdir = self.bootstrap_buildroot.tmpdir + if not os.path.isdir(self.tmpdir): + os.mkdir(self.tmpdir, 0o700) + else: + self.tmpdir = tempfile.mkdtemp(prefix="tmp.mock.", dir='/var/tmp') + os.chmod(self.tmpdir, 0o777) + tmp_libdir = os.path.join(self.tmpdir, '$LIB') + mock_libdir = self.make_chroot_path(tmp_libdir) + nosync_unresolved = '/usr/$LIB/nosync/nosync.so' + + def copy_nosync(lib64=False): + def resolve(path): + return path.replace('$LIB', 'lib64' if lib64 else 'lib') + nosync = resolve(nosync_unresolved) + if not os.path.exists(nosync): + return False + for dst_unresolved in (tmp_libdir, mock_libdir): + dst = resolve(dst_unresolved) + file_util.mkdirIfAbsent(dst) + shutil.copy2(nosync, dst) + return True + + target_arch = self.config['target_arch'] + copied_lib = copy_nosync() + copied_lib64 = copy_nosync(lib64=True) + if not copied_lib and not copied_lib64: + self.root_log.warning("nosync is enabled but the library " + "wasn't found on the system") + return + if all([target_arch in multilib, + not self.config['nosync_force'], + copied_lib != copied_lib64]): + self.root_log.warning("For multilib systems, both architectures" + " of nosync library need to be installed") + return + self.nosync_path = os.path.join(tmp_libdir, 'nosync.so') + + @traceLog() + def finalize(self): + """ + Remove temporary files. If this is the last process working with the + buildroot (exclusive lock can be acquired) also kill orphan processes, + unmount mounts and call postumount hooks. + """ + if self.tmpdir: + for d in self.tmpdir, self.make_chroot_path(self.tmpdir): + if os.path.exists(d): + shutil.rmtree(d) + if os.path.exists(self.make_chroot_path()): + try: + self._lock_buildroot(exclusive=True) + util.orphansKill(self.make_chroot_path()) + self.mounts.umountall() + self.plugins.call_hooks('postumount') + # intentionally we do not call bootstrap hook here - it does not have sense + except BuildRootLocked: + pass + finally: + self._unlock_buildroot() + + @traceLog() + def wrap_host_file(self, filename): + """ + If the bootstrap chroot feature is enabled, and the FILENAME represents + a filename that exists on host, bind-mount it into the bootstrap + chroot automatically and return its modified filename (relatively to + bootstrap chroot). But on some places, we still need to access the + host's file so we use BindMountedFile() wrapper. + """ + bootstrap = self.bootstrap_buildroot + if not bootstrap: + return filename + + # optimized of + # not os.path.exists(filename) or not filename.lower().endswith('.rpm') + if not (os.path.exists(filename) and filename.lower().endswith('.rpm')): + # probably just '--install pkgname' + # or '--install /usr/bin/time' + return filename + + basename = os.path.basename(filename) + if basename in self._homedir_bindmounts: + raise BadCmdline(f"File '{basename}' can not be bind-mounted to " + "bootstrap chroot twice") + self._homedir_bindmounts[basename] = 1 + + host_filename = os.path.abspath(filename) + chroot_filename = os.path.join( + bootstrap.homedir, basename, + ) + bind_path = bootstrap.make_chroot_path(chroot_filename) + bootstrap.mounts.add_user_mount(mounts.BindMountPoint( + srcpath=filename, + bindpath=bind_path, + )) + + return util.BindMountedFile(chroot_filename, host_filename) + + @traceLog() + def delete(self): + """ + Deletes the buildroot contents. + """ + if os.path.exists(self.basedir): + p = self.make_chroot_path() + self._lock_buildroot(exclusive=True) + util.orphansKill(p) + self.mounts.umountall() + self.plugins.call_hooks('umount_root') + # intentionally we do not call bootstrap hook here - it does not have sense + self._unlock_buildroot() + subv = util.find_btrfs_in_chroot(self.mockdir, p) + if subv: + util.do(["btrfs", "subv", "delete", "/" + subv]) + if not self.rootdir.startswith(self.basedir): + file_util.rmtree(self.rootdir, selinux=self.selinux) + file_util.rmtree(self.basedir, selinux=self.selinux) + self.chroot_was_initialized = False + self.plugins.call_hooks('postclean') + # intentionally we do not call bootstrap hook here - it does not have sense + + @property + def uses_bootstrap_image(self): + return self.is_bootstrap and self.use_chroot_image + + @property + def bootstrap_image_is_ready(self): + """ + True if bootstrap image is in use, bootstrap_image_ready==True, and no + config_opts field is set that would invalidate the bootstrap + "readiness" state. + """ + if not self.uses_bootstrap_image: + return False + + # TODO(praiskup): Can we have heuristic for checking if extracted + # buildroot is ready? + + # All the options checked here were copied from the corresponding + # `bootstrap_` prefixed config option. + if not self.config['image_ready']: + self.root_log.info("Bootstrap image not marked ready") + return False + + # Per configuration, we think that image is ready-made (e.g. ubi8 images + # have the 'dnf builddep' command available by default). But there + # still might be reasons to do some updates. + for check_option in [ + "module_enable", "module_install", "module_setup_commands", + "chroot_additional_packages" + ]: + if self.config.get(check_option): + self.root_log.info( + f"Package management in bootstrap (from image) is going to " + f"be updated because config_opts['{check_option}'] is set." + ) + return False + + self.root_log.info("Not updating bootstrap chroot, " + "bootstrap_image_ready=True") + return True + + @traceLog() + def install_as_root(self, *deps): + """ Becomes root user and calls self.install() """ + try: + self.uid_manager.becomeUser(0, 0) + self.install(*deps) + finally: + self.uid_manager.restorePrivs() + + @traceLog() + def install(self, *rpms): + """Call package manager to install the input rpms into the chroot""" + # pass build reqs (as strings) to installer + self.root_log.info("installing package(s): %s", " ".join(rpms)) + output = self.pkg_manager.install(*rpms, returnOutput=1) + self.root_log.info(output) + + @traceLog() + def remove(self, *rpms): + """Call package manager to remove the input rpms from the chroot""" + self.root_log.info("removing package(s): %s", " ".join(rpms)) + output = self.pkg_manager.remove(*rpms, returnOutput=1) + self.root_log.info(output) diff --git a/mock/py/mockbuild/config.py b/mock/py/mockbuild/config.py new file mode 100644 index 0000000..1ca5778 --- /dev/null +++ b/mock/py/mockbuild/config.py @@ -0,0 +1,1071 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING + +from __future__ import print_function + +from ast import literal_eval +from glob import glob +import json +import grp +import logging +import os +import os.path +import pwd +import re +import shlex +import socket +import sys +import uuid +import warnings + +from templated_dictionary import TemplatedDictionary +from . import exception +from . import text +from .constants import MOCKCONFDIR, PKGPYTHONDIR, VERSION +from .file_util import is_in_dir +from .trace_decorator import getLog, traceLog +from .uid import getresuid, getresgid +from .util import set_use_nspawn, setup_operations_timeout + +PLUGIN_LIST = ['tmpfs', 'root_cache', 'yum_cache', 'mount', 'bind_mount', + 'ccache', 'selinux', 'package_state', 'chroot_scan', + 'lvm_root', 'compress_logs', 'sign', 'pm_request', + 'hw_info', 'procenv', 'showrc', 'rpkg_preprocessor', + 'rpmautospec', 'buildroot_lock', 'export_buildroot_image'] + +def nspawn_supported(): + """Detect some situations where the systemd-nspawn chroot code won't work""" + with open("/proc/1/cmdline", "rb") as f: + # if PID 1 has a non-0 UID, then we're running in a user namespace + # without a PID namespace. systemd-nspawn won't work + if os.fstat(f.fileno()).st_uid != 0: + return False + + argv0 = f.read().split(b'\0')[0] + + # If PID 1 is not systemd, then we're in a PID namespace, or systemd + # isn't running on the system: systemd-nspawn won't work. + return os.path.basename(argv0) == b'systemd' + + +@traceLog() +def setup_default_config_opts(): + "sets up default configuration." + + alt_opts = {'dnf.conf': ['yum.conf', 'dnf5.conf', 'dnf4.conf']} + for alt in ["dnf_command", "dnf_common_opts", "dnf_install_command", + "dnf_disable_plugins", "dnf_avoid_opts", "dnf_builddep_opts"]: + alt_opts[alt.replace("dnf", "dnf4")] = [alt] + + for pm in ["dnf4", "dnf5", "yum", "microdnf"]: + alt_opts[f"{pm}_system_command"] = [f"system_{pm}_command"] + + alt_opts["dnf4_system_command"].append("system_dnf_command") + + config_opts = TemplatedDictionary(alias_spec=alt_opts) + + config_opts['config_paths'] = [] + config_opts['version'] = VERSION + config_opts['basedir'] = '/var/lib/mock' # root name is automatically added to this + config_opts['resultdir'] = '{{basedir}}/{{root}}/result' + config_opts['rootdir'] = '{{basedir}}/{{root}}/root' + config_opts['cache_topdir'] = '/var/cache/mock' + config_opts['clean'] = True + config_opts['check'] = True + config_opts['post_install'] = False + config_opts['chroothome'] = '/builddir' + config_opts['log_config_file'] = 'logging.ini' + config_opts['rpmbuild_timeout'] = 0 + config_opts['host_arch'] = os.uname()[-1] + config_opts['chrootuid'] = os.getuid() + try: + config_opts['chrootgid'] = grp.getgrnam("mock")[2] + except KeyError: + # 'mock' group doesn't exist, must set in config file + pass + config_opts['chrootgroup'] = 'mock' + config_opts['chrootuser'] = 'mockbuild' + config_opts['build_log_fmt_name'] = "unadorned" + config_opts['root_log_fmt_name'] = "detailed" + config_opts['state_log_fmt_name'] = "state" + config_opts['online'] = True + config_opts['isolation'] = None + config_opts['use_nspawn'] = None + config_opts['rpmbuild_networking'] = False + config_opts['nspawn_args'] = ['--capability=cap_ipc_lock'] + config_opts['use_container_host_hostname'] = True + + config_opts['use_bootstrap'] = True + config_opts['use_bootstrap_image'] = True + config_opts['bootstrap_image'] = 'fedora:latest' + config_opts['bootstrap_image_skip_pull'] = False + config_opts['bootstrap_image_ready'] = False + config_opts['bootstrap_image_fallback'] = True + config_opts['bootstrap_image_keep_getting'] = 120 + config_opts['bootstrap_image_assert_digest'] = None + + config_opts['use_buildroot_image'] = False + config_opts['buildroot_image'] = None + config_opts['buildroot_image_skip_pull'] = False + config_opts['buildroot_image_ready'] = False + config_opts['buildroot_image_fallback'] = False + config_opts['buildroot_image_keep_getting'] = 120 + config_opts['buildroot_image_assert_digest'] = None + + config_opts['internal_dev_setup'] = True + + # cleanup_on_* only take effect for separate --resultdir + # config_opts provides fine-grained control. cmdline only has big hammer + config_opts['cleanup_on_success'] = True + config_opts['cleanup_on_failure'] = True + + config_opts['exclude_from_homedir_cleanup'] = ['build/SOURCES', '.bash_history', + '.bashrc'] + + config_opts['createrepo_on_rpms'] = False + config_opts['createrepo_command'] = '/usr/bin/createrepo_c -d -q -x *.src.rpm' # default command + + config_opts['tar_binary'] = "/bin/tar" + config_opts['tar'] = "gnutar" + + config_opts['backup_on_clean'] = False + config_opts['backup_base_dir'] = "{{basedir}}/backup" + + config_opts['redhat_subscription_required'] = False + + config_opts['ssl_ca_bundle_path'] = None + + config_opts['ssl_copied_ca_trust_dirs'] = [ + ('/etc/pki/ca-trust', '/etc/pki/ca-trust'), + ('/usr/share/pki/ca-trust-source', '/usr/share/pki/ca-trust-source') + ] + + config_opts['ssl_extra_certs'] = None + + # (global) plugins and plugin configs. + # ordering constraings: tmpfs must be first. + # root_cache next. + # after that, any plugins that must create dirs (yum_cache) + # any plugins without preinit hooks should be last. + config_opts['plugins'] = PLUGIN_LIST + config_opts['plugin_dir'] = os.path.join(PKGPYTHONDIR, "plugins") + config_opts['plugin_conf'] = { + 'ccache_enable': False, + 'ccache_opts': { + 'max_cache_size': "4G", + 'compress': None, + 'dir': "{{cache_topdir}}/{{root}}/ccache/u{{chrootuid}}/", + 'hashdir': True, + 'debug': False, + 'show_stats': False, + }, + 'yum_cache_enable': True, + 'yum_cache_opts': { + 'max_age_days': 30, + 'max_metadata_age_days': 30, + 'online': True}, + 'root_cache_enable': True, + 'root_cache_opts': { + 'age_check': True, + 'max_age_days': 15, + 'dir': "{{cache_topdir}}/{{root}}/root_cache/", + 'compress_program': 'pigz', + 'decompress_program': None, + 'exclude_dirs': ["./proc", "./sys", "./dev", "./tmp/ccache", "./var/cache/yum", "./var/cache/dnf", + "./var/log"], + 'extension': '.gz'}, + 'bind_mount_enable': True, + 'bind_mount_opts': { + 'dirs': [ + # specify like this: + # ('/host/path', '/bind/mount/path/in/chroot/' ), + # ('/another/host/path', '/another/bind/mount/path/in/chroot/'), + ], + 'create_dirs': False}, + 'mount_enable': True, + 'mount_opts': {'dirs': [ + # specify like this: + # ("/dev/device", "/mount/path/in/chroot/", "vfstype", "mount_options"), + ]}, + 'tmpfs_enable': False, + 'tmpfs_opts': { + 'required_ram_mb': 900, + 'max_fs_size': None, + 'mode': '0755', + 'keep_mounted': False}, + 'selinux_enable': True, + 'selinux_opts': {}, + 'package_state_enable': True, + 'package_state_opts': { + 'available_pkgs': False, + 'installed_pkgs': True, + }, + 'buildroot_lock_enable': False, + 'buildroot_lock_opts': {}, + 'pm_request_enable': False, + 'pm_request_opts': {}, + 'lvm_root_enable': False, + 'lvm_root_opts': { + 'pool_name': 'mockbuild', + }, + 'chroot_scan_enable': False, + 'chroot_scan_opts': { + 'regexes': [ + "^[^k]?core(\\.\\d+)?$", "\\.log$", + ], + 'only_failed': True, + 'write_tar': False, + }, + 'sign_enable': False, + 'sign_opts': { + 'cmd': 'rpmsign', + 'opts': '--addsign %(rpms)s', + }, + 'hw_info_enable': True, + 'hw_info_opts': { + }, + 'procenv_enable': False, + 'procenv_opts': { + }, + 'showrc_enable': False, + 'showrc_opts': { + }, + 'compress_logs_enable': False, + 'compress_logs_opts': { + 'command': 'gzip', + }, + 'rpkg_preprocessor_enable': False, + 'rpkg_preprocessor_opts': { + 'requires': ['preproc-rpmspec'], + 'cmd': '/usr/bin/preproc-rpmspec %(source_spec)s --output %(target_spec)s', + }, + 'rpmautospec_enable': False, + 'rpmautospec_opts': { + 'requires': ['rpmautospec'], + 'cmd_base': [ + '/usr/bin/rpmautospec', + 'process-distgit', + ] + }, + 'export_buildroot_image_enable': False, + 'export_buildroot_image_opts': {}, + } + + config_opts['environment'] = { + 'TERM': 'vt100', + 'SHELL': '/bin/bash', + 'HOME': '/builddir', + 'HOSTNAME': 'mock', + 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', + 'PROMPT_COMMAND': r'printf "\033]0;\007"', + 'PS1': r' \s-\v\$ ', + 'LANG': 'C.UTF-8', + } + + runtime_plugins = [runtime_plugin + for (runtime_plugin, _) + in [os.path.splitext(os.path.basename(tmp_path)) + for tmp_path + in glob(config_opts['plugin_dir'] + "/*.py")] + if runtime_plugin not in config_opts['plugins']] + for runtime_plugin in sorted(runtime_plugins): + config_opts['plugins'].append(runtime_plugin) + config_opts['plugin_conf'][runtime_plugin + "_enable"] = False + config_opts['plugin_conf'][runtime_plugin + "_opts"] = {} + + # SCM defaults + config_opts['scm'] = False + config_opts['scm_opts'] = { + 'method': 'git', + 'cvs_get': 'cvs -d /srv/cvs co SCM_BRN SCM_PKG', + 'git_get': 'git clone SCM_BRN git://localhost/SCM_PKG.git SCM_PKG', + 'svn_get': 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG', + 'distgit_get': 'rpkg clone -a --branch SCM_BRN SCM_PKG SCM_PKG', + 'distgit_src_get': 'rpkg sources', + 'spec': 'SCM_PKG.spec', + 'int_src_dir': None, + 'ext_src_dir': os.devnull, + 'write_tar': False, + 'git_timestamps': False, + 'exclude_vcs': True, + } + + # dependent on guest OS + config_opts['use_host_resolv'] = False + config_opts['chroot_setup_cmd'] = ('groupinstall', 'buildsys-build') + config_opts['repo_arch'] = None + config_opts['repo_arch_map'] = {} + config_opts['target_arch'] = 'i386' + config_opts['releasever'] = None + config_opts['rpmbuild_arch'] = None # <-- None means set automatically from target_arch + config_opts['dnf_vars'] = {} + config_opts['yum_builddep_opts'] = [] + config_opts['yum_common_opts'] = [] + config_opts['update_before_build'] = True + config_opts['priorities.conf'] = '\n[main]\nenabled=0' + config_opts['rhnplugin.conf'] = '\n[main]\nenabled=0' + config_opts['subscription-manager.conf'] = '' + config_opts['more_buildreqs'] = {} + config_opts['nosync'] = False + config_opts['nosync_force'] = False + config_opts['files'] = {} + config_opts['macros'] = { + '%_topdir': '%s/build' % config_opts['chroothome'], + '%_rpmfilename': '%%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm', + # This is actually set in check_arch_combination() + # '%_platform_multiplier': 1, + } + config_opts['hostname'] = None + config_opts['module_enable'] = [] + config_opts['module_install'] = [] + config_opts['module_setup_commands'] = [] + config_opts['forcearch'] = None + + # Initialize all the bootstrap configuration (bootstrap_ prefixed) we don't + # want to automatically inherit from the target buildroot configuration. + config_opts['bootstrap_chroot_additional_packages'] = [] + config_opts['bootstrap_module_enable'] = [] + config_opts['bootstrap_module_install'] = [] + config_opts['bootstrap_module_setup_commands'] = [] + + # security config + config_opts['no_root_shells'] = False + config_opts['extra_chroot_dirs'] = [] + + config_opts['package_manager'] = 'dnf' + config_opts['package_manager_max_attempts'] = 1 + config_opts['package_manager_attempt_delay'] = 10 + + config_opts['dynamic_buildrequires'] = True + config_opts['dynamic_buildrequires_max_loops'] = 10 + + config_opts['external_buildrequires'] = False + + config_opts['dev_loop_count'] = 12 + + # configurable commands executables + config_opts['yum_command'] = '/usr/bin/yum' + config_opts['yum_system_command'] = '/usr/bin/yum' + config_opts['yum_install_command'] = 'install yum yum-utils' + config_opts['yum_builddep_command'] = '/usr/bin/yum-builddep' + config_opts["yum_avoid_opts"] = {} + + config_opts['dnf4_command'] = '/usr/bin/dnf-3' + config_opts['dnf4_system_command'] = '/usr/bin/dnf-3' + config_opts['dnf4_common_opts'] = ['--setopt=deltarpm=False', '--setopt=allow_vendor_change=yes', '--allowerasing'] + config_opts['dnf4_install_command'] = 'install python3-dnf python3-dnf-plugins-core' + config_opts['dnf4_disable_plugins'] = ['local', 'spacewalk', 'versionlock'] + config_opts["dnf4_avoid_opts"] = {} + + config_opts['dnf5_command'] = '/usr/bin/dnf5' + config_opts['dnf5_system_command'] = '/usr/bin/dnf5' + config_opts['dnf5_common_opts'] = ['--setopt=deltarpm=False', '--setopt=allow_vendor_change=yes', '--allowerasing'] + config_opts['dnf5_install_command'] = 'install dnf5 dnf5-plugins' + config_opts['dnf5_disable_plugins'] = [] + # No --allowerasing with remove, per + # https://github.com/rpm-software-management/dnf5/issues/729 + config_opts["dnf5_avoid_opts"] = { + "remove": ["--allowerasing"], + "repoquery": ["--allowerasing"], + "makecache": ["--allowerasing"], + "search": ["--allowerasing"], + "info": ["--allowerasing"], + } + + config_opts['microdnf_command'] = '/usr/bin/microdnf' + # "dnf-install" is special keyword which tells mock to use install but with DNF + config_opts['microdnf_install_command'] = \ + 'dnf-install microdnf dnf dnf-plugins-core' + config_opts['microdnf_builddep_command'] = '/usr/bin/dnf' + config_opts['microdnf_builddep_opts'] = [] + config_opts['microdnf_common_opts'] = [] + config_opts['microdnf_avoid_opts'] = {} + + config_opts['rpm_command'] = '/bin/rpm' + config_opts['rpmbuild_command'] = '/usr/bin/rpmbuild' + config_opts['user_agent'] = "Mock ({{ root }}; {{ target_arch }})" + config_opts['opstimeout'] = 0 + + config_opts['stderr_line_prefix'] = "" + + # Packages from this option are baked into the root-cache tarball. + config_opts['chroot_additional_packages'] = [] + + # This option is command-line only, packages are always re-installed (ie. + # not cached in the root-cache tarball). + config_opts['additional_packages'] = None + + config_opts["no-config"] = {} + + config_opts["seccomp"] = False + + config_opts["copy_host_users"] = [] + + # shadow-utils --prefix and --root options do not play well with + # FreeIPA-provided subids. Using the shadow-utils inside the + # chroot works around this but this is a niche situation so it is + # not the default. + # Upstream issue https://github.com/shadow-maint/shadow/issues/897 + config_opts["use_host_shadow_utils"] = True + + # mapping from target_arch (or forcearch) to arch in /usr/bin/qemu-*-static + config_opts["qemu_user_static_mapping"] = { + 'aarch64': 'aarch64', + 'armv7hl': 'arm', + 'i386': 'i386', + 'i686': 'i386', + 'ppc64': 'ppc64', + 'ppc64le': 'ppc64le', + 's390x': 's390x', + 'x86_64': 'x86_64', + } + + config_opts["recursion_limit"] = 5000 + + config_opts["calculatedeps"] = None + config_opts["hermetic_build"] = False + config_opts["mock_run_uuid"] = str(uuid.uuid4()) + + return config_opts + + +def multiply_platform_multiplier(config_opts): + """ Define '%_platform_multiplier' macro based on forcearch. + But respect possible overrides in config. + """ + if '%_platform_multiplier' not in config_opts["macros"]: + config_opts["macros"]["%_platform_multiplier"] = 10 if config_opts["forcearch"] else 1 + + +@traceLog() +def set_config_opts_per_cmdline(config_opts, options, args): + "takes processed cmdline args and sets config options." + + process_hermetic_build_config(options, config_opts) + + cli_opt_new = {} + for cli_opt in options.cli_config_opts: + k, v = cli_opt.split("=", 1) + # convert string to boolean and int if possible + if v in ['true', 'True']: + v = True + elif v in ['false', 'False']: + v = False + elif v in ['none', 'None']: + v = None + else: + try: + v = int(v) + except ValueError: + pass + if k not in cli_opt_new: + cli_opt_new[k] = v + elif isinstance(cli_opt_new[k], list): + cli_opt_new[k].append(v) + else: + if v == '': + # hack! + # specify k twice, second v is empty, this make it list with one value + cli_opt_new[k] = [cli_opt_new[k]] + else: + cli_opt_new[k] = [cli_opt_new[k], v] + config_opts.update(cli_opt_new) + + config_opts['verbose'] = options.verbose + if 'print_main_output' not in config_opts or config_opts['print_main_output'] is None: + config_opts['print_main_output'] = config_opts['verbose'] > 0 and sys.stderr.isatty() + + # do some other options and stuff + if options.arch: + config_opts['target_arch'] = options.arch + if options.rpmbuild_arch: + config_opts['rpmbuild_arch'] = options.rpmbuild_arch + elif config_opts['rpmbuild_arch'] is None: + config_opts['rpmbuild_arch'] = config_opts['target_arch'] + if options.forcearch: + config_opts['forcearch'] = options.forcearch + + if not config_opts['repo_arch']: + target = config_opts['target_arch'] + config_opts['repo_arch'] = config_opts['repo_arch_map'].get(target, target) + + if not options.clean: + config_opts['clean'] = options.clean + + if not options.check: + config_opts['check'] = options.check + + if options.post_install: + config_opts['post_install'] = options.post_install + + for option in options.rpmwith: + options.rpmmacros.append("_with_%s --with-%s" % + (option.replace("-", "_"), option)) + + for option in options.rpmwithout: + options.rpmmacros.append("_without_%s --without-%s" % + (option.replace("-", "_"), option)) + + for macro in options.rpmmacros: + try: + macro = macro.strip() + k, v = macro.split(" ", 1) + if not k.startswith('%'): + k = '%%%s' % k + config_opts['macros'].update({k: v}) + # pylint: disable=bare-except + except: + # pylint: disable=raise-missing-from + raise exception.BadCmdline( + "Bad option for '--define' (%s). Use --define 'macro expr'" + % macro) + + if options.macrofile: + config_opts['macrofile'] = os.path.expanduser(options.macrofile) + if not os.path.isfile(config_opts['macrofile']): + raise exception.BadCmdline( + "Input rpm macros file does not exist: %s" + % options.macrofile) + + if options.resultdir: + config_opts['resultdir'] = os.path.expanduser(options.resultdir) + if options.rootdir: + config_opts['rootdir'] = os.path.expanduser(options.rootdir) + if options.uniqueext: + config_opts['unique-ext'] = options.uniqueext + if options.rpmbuild_timeout is not None: + config_opts['rpmbuild_timeout'] = options.rpmbuild_timeout + if options.bootstrapchroot is not None: + config_opts['use_bootstrap'] = options.bootstrapchroot + if options.usebootstrapimage is not None: + config_opts['use_bootstrap_image'] = options.usebootstrapimage + if options.usebootstrapimage: + config_opts['use_bootstrap'] = True + + for i in options.disabled_plugins: + if i not in config_opts['plugins']: + raise exception.BadCmdline( + "Bad option for '--disable-plugin=%s'. Expecting one of: %s" + % (i, config_opts['plugins'])) + config_opts['plugin_conf']['%s_enable' % i] = False + for i in options.enabled_plugins: + if i not in config_opts['plugins']: + raise exception.BadCmdline( + "Bad option for '--enable-plugin=%s'. Expecting one of: %s" + % (i, config_opts['plugins'])) + config_opts['plugin_conf']['%s_enable' % i] = True + for option in options.plugin_opts: + try: + p, kv = option.split(":", 1) + k, v = kv.split("=", 1) + # pylint: disable=bare-except + except: + # pylint: disable=raise-missing-from + raise exception.BadCmdline( + "Bad option for '--plugin-option' (%s). Use --plugin-option 'plugin:key=value'" + % option) + if p not in config_opts['plugins']: + raise exception.BadCmdline( + "Bad option for '--plugin-option' (%s). No such plugin: %s" + % (option, p)) + try: + v = literal_eval(v) + # pylint: disable=bare-except + except: + pass + config_opts['plugin_conf'][p + "_opts"].update({k: v}) + + use_nspawn = None # auto-detect by default + + log = logging.getLogger() + + if config_opts['use_nspawn'] in [True, False]: + log.info("Use of obsoleted configuration option 'use_nspawn'.") + use_nspawn = config_opts['use_nspawn'] + + if config_opts['isolation'] in ['nspawn', 'simple']: + use_nspawn = config_opts['isolation'] == 'nspawn' + elif config_opts['isolation'] == 'auto': + use_nspawn = None # set auto detection, overrides use_nspawn + + if options.old_chroot: + use_nspawn = False + log.error('Option --old-chroot has been deprecated. Use --isolation=simple instead.') + if options.new_chroot: + use_nspawn = True + log.error('Option --new-chroot has been deprecated. Use --isolation=nspawn instead.') + + if options.isolation in ['simple', 'nspawn']: + use_nspawn = options.isolation == 'nspawn' + elif options.isolation == 'auto': + use_nspawn = None # re-set auto detection + elif options.isolation is not None: + raise exception.BadCmdline("Bad option for '--isolation'. Unknown " + "value: %s" % (options.isolation)) + if use_nspawn is None: + use_nspawn = nspawn_supported() + getLog().info("systemd-nspawn auto-detected: %s", use_nspawn) + + set_use_nspawn(use_nspawn, config_opts) + + if options.enable_network: + config_opts['rpmbuild_networking'] = True + config_opts['use_host_resolv'] = True + + if options.mode in ("rebuild",) and len(args) > 1 and not options.resultdir: + raise exception.BadCmdline( + "Must specify --resultdir when building multiple RPMS.") + + if options.mode == "chain" and options.resultdir: + raise exception.BadCmdline( + "The --chain mode doesn't support --resultdir, use --localrepo instead") + + if options.cleanup_after is False: + config_opts['cleanup_on_success'] = False + config_opts['cleanup_on_failure'] = False + + if options.cleanup_after is True: + config_opts['cleanup_on_success'] = True + config_opts['cleanup_on_failure'] = True + + check_config(config_opts) + # can't cleanup unless resultdir is separate from the root dir + basechrootdir = os.path.join(config_opts['basedir'], config_opts['root']) + config_resultdir = text.compat_expand_string(config_opts['resultdir'], config_opts) + if is_in_dir(config_resultdir, basechrootdir): + config_opts['cleanup_on_success'] = False + config_opts['cleanup_on_failure'] = False + + config_opts['cache_alterations'] = options.cache_alterations + + config_opts['online'] = options.online + + if options.pkg_manager: + config_opts['package_manager'] = options.pkg_manager + if options.mode == 'yum-cmd': + config_opts['package_manager'] = 'yum' + if options.mode == 'dnf-cmd': + # If config specifies a different package manager (e.g. yum), force + # using dnf. However, if config specifies its preference on either dnf + # or dnf5, respect that preference. + if config_opts['package_manager'] not in ['dnf', 'dnf5']: + config_opts['package_manager'] = 'dnf' + + if options.short_circuit: + config_opts['short_circuit'] = options.short_circuit + config_opts['clean'] = False + + if options.rpmbuild_opts: + config_opts['rpmbuild_opts'] = options.rpmbuild_opts + + config_opts['enable_disable_repos'] = options.enable_disable_repos + + if options.scm: + try: + # pylint: disable=unused-variable,unused-import,import-outside-toplevel + from . import scm + except ImportError as e: + raise exception.BadCmdline( + "Mock SCM module not installed: %s" % e) + + config_opts['scm'] = options.scm + for option in options.scm_opts: + try: + k, v = option.split("=", 1) + config_opts['scm_opts'].update({k: v}) + # pylint: disable=bare-except + except: + # pylint: disable=raise-missing-from + raise exception.BadCmdline( + "Bad option for '--scm-option' (%s). Use --scm-option 'key=value'" + % option) + + # This option is command-line only (contrary to chroot_additional_packages, + # which though affects root_cache). + config_opts["additional_packages"] = options.additional_packages + + config_opts["calculatedeps"] = options.calculatedeps + if config_opts["calculatedeps"]: + config_opts["plugin_conf"]["buildroot_lock_enable"] = True + + if options.buildroot_image: # --buildroot-image option + if os.path.exists(options.buildroot_image): + config_opts["buildroot_image"] = \ + "oci-archive:" + os.path.realpath(options.buildroot_image) + else: + config_opts["buildroot_image"] = options.buildroot_image + + if config_opts["buildroot_image"]: + config_opts["use_buildroot_image"] = True + +def check_config(config_opts): + if 'root' not in config_opts: + raise exception.ConfigError("Error in configuration " + "- option config_opts['root'] must be present in your config.") + + +regexp_include = re.compile(r'^\s*include\((.*)\)', re.MULTILINE) + + +@traceLog() +def include(config_file, search_path, paths): + if not os.path.isabs(config_file): + config_file = os.path.join(search_path, config_file) + + if os.path.exists(config_file): + if config_file in paths: + getLog().warning("Multiple inclusion of %s, skipping" % config_file) + return "" + + paths.add(config_file) + with open(config_file) as f: + content = f.read() + # Search for "include(FILE)" and for each "include(FILE)" replace with + # content of the FILE, in a perpective of search for includes and replace with his content. + include_arguments = regexp_include.findall(content) + if include_arguments is not None: + for include_argument in include_arguments: + # pylint: disable=eval-used + sub_config_file = eval(include_argument) + sub_content = include(sub_config_file, search_path, paths) + content = regexp_include.sub(sub_content, content, count=1) + return content + else: + raise exception.ConfigError("Could not find included config file: %s" % config_file) + + +@traceLog() +def update_config_from_file(config_opts, config_file): + """ + Parse a given Mock config file with Python interpreter. Return just the + 'config_opts' global merged with the 'config_opts'. + """ + config_file = os.path.realpath(config_file) + included_files = set() + content = include(config_file, config_opts["config_path"], included_files) + + # TODO: we should avoid doing exec() here long term, and switch to some + # better form of configuration (like YAML, json, or so?). But historically + # the configuration is just a python-syntax file, so as a poor safety + # measure we disallow this for root (saved set-*IDs checked, too!) + try: + exec(content) # pylint: disable=exec-used + except Exception as exc: + raise exception.ConfigError("Config error: {}: {}".format(config_file, str(exc))) + + # this actually allows multiple inclusion of one file, but not in a loop + new_paths = set(config_opts["config_paths"]) | included_files + config_opts["config_paths"] = list(new_paths) + + +def update_config_from_dict(config_opts, updates): + """ + Merge a dictionary into config_opts. No include supported. + """ + for key, value in updates.items(): + config_opts[key] = value + + +def process_hermetic_build_config(cmdline_opts, config_opts): + """ + Read the lockfile file generated by the previous + --calculate-build-dependencies run, and adjust the current set of options + in CONFIG_OPTS. + """ + + if not cmdline_opts.hermetic_build: + return + + config_opts["hermetic_build"] = True + + json_conf, repo_reference = cmdline_opts.hermetic_build + with open(json_conf, "r", encoding="utf-8") as fd: + data = json.load(fd) + + if not data["config"].get("bootstrap_image_ready"): + raise exception.BadCmdline( + f"The file {json_conf} did not record the bootstrap_image_ready=True " + "config which means we are not able to prepare the bootstrap chroot " + "in a hermetic mode.") + + update_config_from_dict(config_opts, data["config"]) + + final_offline_repo = repo_reference + file_pfx = "file://" + if final_offline_repo.startswith(file_pfx): + final_offline_repo = final_offline_repo[len(file_pfx):] + final_offline_repo = os.path.abspath(final_offline_repo) + if not os.path.exists(os.path.join(final_offline_repo, "repodata")): + raise exception.BadCmdline( + f"The {repo_reference} doesn't seem to be a valid " + "offline RPM repository (RPM metadata not found)") + + # Use the offline image for bootstrapping. + bootstrap_tarball = os.path.join(final_offline_repo, "bootstrap.tar") + config_opts["bootstrap_image"] = f"oci-archive:{bootstrap_tarball}" + + config_opts["offline_local_repository"] = final_offline_repo + + # We install all the packages at once (for now?). We could inherit the + # command from the previous "online" run, but it often employs a group + # installation command - and we have no groups in the offline repo. + config_opts["chroot_setup_cmd"] = "install *" + + # With hermetic builds, we always assert that we are reproducing the build + # with the same image. + config_opts["bootstrap_image_assert_digest"] = data["bootstrap"]["image_digest"] + + # It doesn't make sense to fallback to `dnf install dnf --installroot ...`, + # we simply don't have DNF stack pre-downloaded in offline_local_repository. + config_opts["bootstrap_image_fallback"] = False + + +@traceLog() +def nice_root_alias_error(name, alias_name, arch, no_configs, log): + """ + The epel-8 configs (and others in future) will be replaced with more + specific alternatives. This is the way to inform user about alternatives. + """ + any_alternative = False + + if alias_name not in no_configs: + return any_alternative + + arg_name = "{}-{}".format(alias_name, arch) + + aliases = no_configs[alias_name]["alternatives"] + order = 0 + + for alias_base, alias in aliases.items(): + short_name = "{}-{}".format(alias_base, arch) + filename = "{}.cfg".format(short_name) + cfg_path = os.path.join("/etc/mock", filename) + if not os.path.exists(cfg_path): + continue + if not any_alternative: + log.error("There are those alternatives:") + any_alternative = True + order += 1 + pfx = " " + log.error("") + log.error("[{}] {}".format(order, short_name)) + + alt_cmd = ['mock'] + [short_name if a == arg_name else shlex.quote(a) + for a in sys.argv[1:]] + + log.error("%sUse instead: %s ", pfx, ' '.join(alt_cmd)) + for line in alias["description"]: + log.error(pfx + line) + + log.error("%sEnable permanently by:", pfx) + homeconfig = os.path.join(os.path.expanduser('~'), '.config', 'mock') + if not os.path.exists(homeconfig): + log.error("%s$ mkdir -p %s", pfx, homeconfig) + + log.error("%s$ ln -s %s %s/%s-%s.cfg", pfx, cfg_path, + homeconfig, alias_name, arch) + + return any_alternative + + +@traceLog() +def do_update_config(log, config_opts, cfg, name, skipError=True): + if os.path.exists(cfg): + log.info("Reading configuration from %s", cfg) + update_config_from_file(config_opts, cfg) + setup_operations_timeout(config_opts) + check_macro_definition(config_opts) + return + + if skipError: + return + + log.error("Could not find required config file: %s", cfg) + + match = re.match(r"^([\w-]+)-(\w+)-(\w+)$", name) + no_configs = config_opts.get("no-config") + + if match and no_configs: + alias = "-".join([match[1], match[2]]) + if nice_root_alias_error(name, alias, match[3], no_configs, log): + raise exception.ConfigError( + "Mock config '{}' not found, see errors above.".format(name)) + + if name == "default": + log.error(" Did you forget to specify the chroot to use with '-r'?") + if "/" in cfg: + log.error(" If you're trying to specify a path, include the .cfg extension, e.g. -r ./target.cfg") + + raise exception.ConfigError("Non-existing Mock config '{}'".format(name)) + + +def print_description(config_path, config_filename): + basename_without_ext = parse_config_filename(config_filename)[2] + try: + config_opts = load_config(config_path, config_filename) + description = config_opts.get("description", "") + except exception.ConfigError: + description = 'error during parsing the config file' + print("{} {}".format(basename_without_ext.ljust(34), description)) + + +def parse_config_filename(config_filename): + """ Accepts filename and returns (dirname, basename, basename_without_ext) """ + basename = os.path.basename(config_filename) + dirname = os.path.dirname(config_filename) + basename_without_ext = os.path.splitext(basename)[0] + return (basename, dirname, basename_without_ext) + + +def get_global_configs(config_path): + """ Return filenames of global configs of chroots""" + result = [] + for config_filename in glob(f"{config_path}/*.cfg"): + if config_filename in ["/etc/mock/chroot-aliases.cfg", "/etc/mock/site-defaults.cfg"]: + continue + result.append(config_filename) + return result + + +def get_user_config_files(): + """ Return filenames of user configs of chroots """ + uid = os.getuid() + custom_path = os.path.join(os.path.expanduser('~' + pwd.getpwuid(uid)[0]), '.config/mock/*.cfg') + result = glob(custom_path) + return result + + +def traverse_chroot_configs(config_path=MOCKCONFDIR, + system_configs_traversed_cb=None, + include_eol=False): + """ + Traverse the configuration directories, starting from the system-wide + directories, then going through $HOME/.mock/, and yield the (config_path, + config_filename, eol=True|False) 3-aries. Config_path is just passed to + callee unchanged! + """ + for config_filename in sorted(get_global_configs(config_path)): + yield config_path, config_filename, False + + if include_eol: + for config_filename in sorted(get_global_configs( + os.path.join(config_path, "eol"))): + yield config_path, config_filename, True + + if system_configs_traversed_cb: + system_configs_traversed_cb() + + user_config_files = get_user_config_files() + if user_config_files: + # ~/.config/mock/CHROOTNAME.cfg + for config_filename in sorted(user_config_files): + yield config_path, config_filename, False + + +@traceLog() +def list_configs(config_path): + log = logging.getLogger() + log.disabled = True + # array to save config paths + print("{} {}".format("config name".ljust(34), "description")) + print("Global configs:") + + def _print_custom(): + print("Custom configs:") + + for cp, fn, _ in traverse_chroot_configs(config_path, _print_custom): + print_description(cp, fn) + log.disabled = False + + +@traceLog() +def simple_load_config(name, config_path=None): + """ wrapper around load_config() intended by use 3rd party SW """ + + for the_id in getresuid() + getresgid(): + if the_id != 0: + continue + warnings.warn( + "Parsing Mock configuration as root is highly discouraged, " + "see https://rpm-software-management.github.io/mock/#setup" + ) + break + + if config_path is None: + config_path = MOCKCONFDIR + return load_config(config_path, name) + + +@traceLog() +def load_config(config_path, name): + log = logging.getLogger() + config_opts = setup_default_config_opts() + + # array to save config paths + config_opts['config_path'] = config_path + config_opts['chroot_name'] = name + + uid = config_opts["chrootuid"] + + # Read in the config files: default, and then user specified + if name.endswith('.cfg'): + # If the .cfg is explicitly specified we take the root arg to + # specify a path, rather than looking it up in the configdir. + chroot_cfg_path = name + config_opts['chroot_name'] = os.path.splitext(os.path.basename(name))[0] + else: + # ~/.config/mock/CHROOTNAME.cfg + cfg = os.path.join(os.path.expanduser('~' + pwd.getpwuid(uid)[0]), + '.config/mock/{}.cfg'.format(name)) + if os.path.exists(cfg): + chroot_cfg_path = cfg + else: + chroot_cfg_path = '%s/%s.cfg' % (config_path, name) + config_opts['config_file'] = chroot_cfg_path + + # load the global config files + for cfg_file in [ + os.path.join(config_path, "site-defaults.cfg"), + os.path.join(config_path, "chroot-aliases.cfg"), + ]: + do_update_config(log, config_opts, cfg_file, name) + + # load the "chroot" specific config (-r option) + do_update_config(log, config_opts, chroot_cfg_path, name, skipError=False) + + # Read user specific config file + cfg = os.path.join(os.path.expanduser( + '~' + pwd.getpwuid(os.getuid())[0]), '.mock/user.cfg') + do_update_config(log, config_opts, cfg, name) + cfg = os.path.join(os.path.expanduser( + '~' + pwd.getpwuid(os.getuid())[0]), '.config/mock.cfg') + do_update_config(log, config_opts, cfg, name) + + if config_opts['use_container_host_hostname'] and '%_buildhost' not in config_opts['macros']: + config_opts['macros']['%_buildhost'] = socket.getfqdn() + + # Now when all options are correctly loaded from config files, turn the + # jinja templating ON. + config_opts['__jinja_expand'] = True + + # use_bootstrap_container is deprecated option + if 'use_bootstrap_container' in config_opts: + log.warning("config_opts['use_bootstrap_container'] is deprecated, " + "please use config_opts['use_bootstrap'] instead") + config_opts['use_bootstrap'] = config_opts['use_bootstrap_container'] + + return config_opts + + +@traceLog() +def check_macro_definition(config_opts): + for k in list(config_opts['macros']): + v = config_opts['macros'][k] + if not k or (not v and (v is not None)) or len(k.split()) != 1: + raise exception.BadCmdline( + "Bad macros 'config_opts['macros']['%s'] = ['%s']'" % (k, v)) + if not k.startswith('%'): + del config_opts['macros'][k] + k = '%{0}'.format(k) + config_opts['macros'].update({k: v}) diff --git a/mock/py/mockbuild/constants.py b/mock/py/mockbuild/constants.py new file mode 100644 index 0000000..6e3351e --- /dev/null +++ b/mock/py/mockbuild/constants.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING + +import os.path +import sys + +# all of the variables below are substituted by the build system +VERSION = "unreleased_version" +SYSCONFDIR = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "..", "etc") +PYTHONDIR = os.path.dirname(os.path.realpath(sys.argv[0])) +PKGPYTHONDIR = os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])), "mockbuild") +MOCKCONFDIR = os.path.join(SYSCONFDIR, "mock") +# end build system subs diff --git a/mock/py/mockbuild/exception.py b/mock/py/mockbuild/exception.py new file mode 100644 index 0000000..22b9048 --- /dev/null +++ b/mock/py/mockbuild/exception.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Originally written by Seth Vidal +# Sections taken from Mach by Thomas Vander Stichele +# Major reorganization and adaptation by Michael Brown +# Copyright (C) 2007 Michael E Brown +"""define most of the exceptions used.""" + +# classes + + +class Error(Exception): + resultcode = 1 + + def __init__(self, *args): + """ + A base class for our errors. The exit code can be specified as + self.resultcode. If multiple ARGS are specified, the second one is + used as the resultcode. + """ + super().__init__(*args) + self.msg = args[0] + if len(args) > 1: + self.resultcode = args[1] + + def __str__(self): + return self.msg + +# result/exit codes +# 0 = yay! +# 1 = something happened - it's bad +# 2 = run without setuid wrapper +# 3 = invalid configuration +# 4 = only some packages were build during --chain +# 5 = cmdline processing error +# 6 = invalid architecture +# 10 = problem building the package +# 11 = command timeouted +# 20 = error in the chroot of some kind +# 25 = LVM manipulation error +# 30 = Yum emitted an error of some sort +# 31 = Unknow external dependency +# 40 = some error in the pkg we're building +# 50 = error in mock command (varies for each command) +# 60 = buildroot locked +# 65 = LVM thinpool locked +# 70 = result dir could not be created +# 80 = unshare of namespace failed +# 90 = bootstrap preparation error +# 110 = unbalanced call to state functions +# 120 = weak dependent package not installed +# 129 = the main process get signal SIGHUP, the console was closed +# 141 = the main process get signal SIGPIPE, the pipe does not exist or was closed +# 143 = the main process get signal SIGTERM, something tries to kill mock process + +def get_class_by_code(exit_code): + if exit_code == 0: + return None + elif exit_code == 1: + return Error("Unknow error happened.") + elif exit_code == 2: + return Error("Run without setuid wrapper.", 2) + elif exit_code == 3: + return ConfigError("Invalid configuration.") + elif exit_code == 4: + return Error("Only some packages were build during --chain.", 4) + elif exit_code == 5: + return BadCmdline("Command-line processing error.") + elif exit_code == 6: + return InvalidArchitecture("Invalid architecture.") + elif exit_code == 10: + return BuildError("Error during rpmbuild phase. Check the build.log.") + elif exit_code == 11: + return commandTimeoutExpired("Command timeout expired.") + elif exit_code == 20: + return RootError("Error in the chroot. Check the root.log.") + elif exit_code == 25: + return LvmError("LVM manipulation failed.") + elif exit_code == 30: + return YumError("Package manager emitted an error of some sort.") + elif exit_code == 31: + return ExternalDepsError("Unknown external dependency") + elif exit_code == 40: + return PkgError("Error with the srpm given to us.") + elif exit_code == 50: + return Error("Error in mock command (varies for each command)", 50) + elif exit_code == 60: + return BuildRootLocked("Build-root in use by another process.") + elif exit_code == 65: + return LvmLocked("LVM thinpool is locked.") + elif exit_code == 70: + return ResultDirNotAccessible("Result dir could not be created.") + elif exit_code == 80: + return UnshareFailed("Call to C library unshare(2) syscall failed.") + elif exit_code == 90: + return BootstrapError("Can not prepare bootstrap chroot") + elif exit_code == 110: + return StateError("Unbalanced call to state functions. Check the state.log") + elif exit_code == 120: + return Error("Weak dependent package not installed.", 120) + elif exit_code == 129: + return Error("The main process get signal SIGHUP, the console was closed.", 129) + elif exit_code == 141: + return Error("The main process get signal SIGPIPE, the pipe does not exist or was closed.", 141) + elif exit_code == 143: + return Error("The main process get signal SIGTERM, something tries to kill mock process.", 143) + else: + return Error("Unknow error %{} happened.".format(exit_code), exit_code) + +class BuildError(Error): + "rpmbuild failed." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 10 + + +class commandTimeoutExpired(Error): + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 11 + +class RootError(Error): + "failed to set up chroot" + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 20 + + +class LvmError(Error): + "LVM manipulation failed." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 25 + + +class YumError(RootError): + "yum failed." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 30 + + +class ExternalDepsError(RootError): + "Unknown external dependency." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 31 + +class PkgError(Error): + "error with the srpm given to us." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 40 + + +class BuildRootLocked(Error): + "build root in use by another process." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 60 + + +class LvmLocked(Error): + "LVM thinpool is locked." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 65 + + +class BadCmdline(Error): + "user gave bad/inconsistent command line." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 5 + + +class InvalidArchitecture(Error): + "invalid host/target architecture specified." + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 6 + + +class ResultDirNotAccessible(Error): + """ +Could not create output directory for built rpms. The directory specified was: + %s + +Try using the --resultdir= option to select another location. Recommended location is --resultdir=~/mock/. +""" + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 70 + + +class UnshareFailed(Error): + "call to C library unshare(2) syscall failed" + + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 80 + + +class BootstrapError(Error): + "Can not prepare bootstrap chroot" + + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 90 + + +class StateError(Error): + "unbalanced call to state functions" + + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 110 + + +class ConfigError(Error): + "invalid configuration" + + def __init__(self, *args): + super().__init__(*args) + self.resultcode = 3 diff --git a/mock/py/mockbuild/external.py b/mock/py/mockbuild/external.py new file mode 100644 index 0000000..0edd311 --- /dev/null +++ b/mock/py/mockbuild/external.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: + +from .exception import ExternalDepsError +from .trace_decorator import traceLog + + +class ExternalDeps(object): + """ Handles external dependecies. E.g external:* """ + + def __init__(self, buildroot, bootstrap_buildroot, uid_manager): + self.buildroot = buildroot + self.bootstrap_buildroot = bootstrap_buildroot + self.uid_manager = uid_manager + + # when python 3.9 becomes standard, this can be replaced by string.removeprefix() + @classmethod + def _remove_prefix(cls, intext, prefix): + if intext.startswith(prefix): + return intext[len(prefix):] + return intext + + def extract_external_deps(self, requires): + """ accepts list of (build)requires, returns all external deps """ + return [i for i in requires if i.startswith('external:')] + + @traceLog() + def install_external_deps(self, deps): + """Install dependencies using native library manager""" + self.buildroot.root_log.info('Installing dependencies to satisfy external:*') + + prefix = 'external:pypi:' + pypi_deps = [self._remove_prefix(i, prefix) for i in deps if i.startswith(prefix)] + deps = [i for i in deps if not i.startswith(prefix)] + if pypi_deps: + self.install_external_deps_pypi(pypi_deps) + + prefix = 'external:crate:' + crate_deps = [self._remove_prefix(i, prefix) for i in deps if i.startswith(prefix)] + deps = [i for i in deps if not i.startswith(prefix)] + if crate_deps: + self.install_external_deps_crate(crate_deps) + + if deps: + raise ExternalDepsError("Unknown external dependencies: {}".format(', '.join(deps))) + + @traceLog() + def install_external_deps_pypi(self, deps): + """ deps is list of python modules. Without the prefix. """ + self.bootstrap_buildroot.install_as_root('/usr/bin/pip3', 'python3-setuptools') + command = [ + 'pip3', "install", + "--root", self.buildroot.make_chroot_path(), + "--prefix", "/usr", + ] + deps + try: + self.uid_manager.becomeUser(0, 0) + self.buildroot.doOutChroot(command, shell=False, printOutput=True) + except: # pylint: disable=bare-except + raise ExternalDepsError('Pip3 install failed') + finally: + self.uid_manager.restorePrivs() + self.install_fake_rpm('pypi', deps) + + @traceLog() + def install_external_deps_crate(self, deps): + """ deps is list of Rust modules. Without the prefix. """ + self.bootstrap_buildroot.install_as_root('/usr/bin/cargo') + command = ['cargo', "install", "--root", self.buildroot.make_chroot_path()] + deps + try: + self.uid_manager.becomeUser(0, 0) + self.buildroot.doOutChroot(command, shell=False, printOutput=True) + except: # pylint: disable=bare-except + raise ExternalDepsError('Cargo install failed') + finally: + self.uid_manager.restorePrivs() + self.install_fake_rpm('crate', deps) + + @traceLog() + def install_fake_rpm(self, external_type, deps): + """ Create and install fake packages using create-fake-rpm """ + list_of_packages = [] + self.buildroot.install_as_root('create-fake-rpm') + try: + self.uid_manager.becomeUser(0, 0) + for dep in deps: + command = ['/usr/bin/create-fake-rpm', '--print-result', + '--build', + 'external-{0}-{1}'.format(external_type, dep), + 'external:{0}:{1}'.format(external_type, dep)] + # The output here is: + # Wrote: /fake-external-pypi-bokeh-0-0.noarch.rpm\n + (output, _) = self.buildroot.doChroot( + command, returnOutput=True, returnStderr=False, + raiseExc=True) + package = self.buildroot.make_chroot_path(output.split()[1]) + list_of_packages.append(package) + finally: + self.uid_manager.restorePrivs() + self.buildroot.install_as_root(*list_of_packages) diff --git a/mock/py/mockbuild/file_downloader.py b/mock/py/mockbuild/file_downloader.py new file mode 100644 index 0000000..6506b6d --- /dev/null +++ b/mock/py/mockbuild/file_downloader.py @@ -0,0 +1,96 @@ +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +from email.message import Message +import shutil +import tempfile +from urllib.parse import urlsplit + +import backoff +import requests + +from .trace_decorator import getLog + + +def _filename_from_response(response): + if 'content-disposition' in response.headers: + # https://peps.python.org/pep-0594/#cgi + parser = Message() + parser["content-type"] = response.headers['content-disposition'] + if parser.get_param("filename"): + return parser.get_param("filename") + return urlsplit(response.url).path.rsplit('/', 1)[1] + + +class FileDownloader: + """ + Download files into a temporary storage with automatic cleanup with + cls.cleanup(), remembering both original URL and the local path name. + """ + + tmpdir = None + + @classmethod + def _initialize(cls): + if cls.tmpdir: + return + cls.backmap = {} + cls.tmpdir = tempfile.mkdtemp() + + @classmethod + def get(cls, pkg_url_or_local_file): + """ + If the pkg_url_or_local_file looks like a link, try to download it and + store it to a temporary directory - and return path to the local + downloaded file. + + If the pkg_url_or_local_file is not a link, do nothing - just return + the pkg_url_or_local_file argument. + """ + pkg = pkg_url_or_local_file + log = getLog() + + url_prefixes = ['http://', 'https://', 'ftp://'] + if not any(pkg.startswith(pfx) for pfx in url_prefixes): + log.debug("Local file: %s", pkg) + return pkg + + cls._initialize() + try: + log.info('Fetching remote file %s', pkg) + return cls._get_inner(pkg) + except requests.exceptions.RequestException as e: + log.error('Downloading error %s: %s', pkg, str(e)) + return None + + @classmethod + @backoff.on_exception(backoff.expo, requests.exceptions.RequestException, + max_tries=3, max_time=100) + def _get_inner(cls, url): + req = requests.get(url, timeout=30) + req.raise_for_status() + filename = _filename_from_response(req) + pkg = cls.tmpdir + '/' + filename + with open(pkg, 'wb') as filed: + for chunk in req.iter_content(4096): + filed.write(chunk) + cls.backmap[pkg] = url + return pkg + + @classmethod + def original_name(cls, localname): + """ Get the URL from the local name """ + if getattr(cls, 'backmap', None): + return cls.backmap.get(localname, localname) + return localname + + @classmethod + def cleanup(cls): + """ Remove the temporary storage with downloaded RPMs """ + if not cls.tmpdir: + return + getLog().debug("Cleaning the temporary download directory: %s", cls.tmpdir) + cls.backmap = {} + # cleaning up our download dir + shutil.rmtree(cls.tmpdir, ignore_errors=True) + cls.tmpdir = None diff --git a/mock/py/mockbuild/file_util.py b/mock/py/mockbuild/file_util.py new file mode 100644 index 0000000..83e117b --- /dev/null +++ b/mock/py/mockbuild/file_util.py @@ -0,0 +1,176 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +import errno +import os +import os.path +import shutil +import stat +import subprocess +import time + +from . import exception +from .trace_decorator import getLog, traceLog + + +@traceLog() +def mkdirIfAbsent(*args): + for dirName in args: + getLog().debug("ensuring that dir exists: %s", dirName) + try: + os.makedirs(dirName) + getLog().debug("created dir: %s", dirName) + except OSError as e: + if e.errno != errno.EEXIST: + getLog().exception("Could not create dir %s. Error: %s", dirName, e) + raise exception.Error("Could not create dir %s. Error: %s" % (dirName, e)) + + +@traceLog() +def touch(fileName): + getLog().debug("touching file: %s", fileName) + open(fileName, 'a').close() + + +@traceLog() +def rmtree(path, selinux=False, exclude=()): + """Version of shutil.rmtree that ignores no-such-file-or-directory errors, + tries harder if it finds immutable files and supports excluding paths""" + if os.path.islink(path): + raise OSError("Cannot call rmtree on a symbolic link: %s" % path) + try_again = True + retries = 10 + failed_to_handle = False + failed_filename = None + if path in exclude: + return + while try_again: + try_again = False + try: + names = os.listdir(path) + for name in names: + fullname = os.path.join(path, name) + if fullname not in exclude: + try: + mode = os.lstat(fullname).st_mode + except OSError: + mode = 0 + if stat.S_ISDIR(mode): + try: + rmtree(fullname, selinux=selinux, exclude=exclude) + except OSError as e: + if e.errno in (errno.EPERM, errno.EACCES, errno.EBUSY): + # we already tried handling this on lower level and failed, + # there's no point in trying again now + failed_to_handle = True + raise + else: + os.remove(fullname) + os.rmdir(path) + except OSError as e: + if failed_to_handle: + raise + if e.errno == errno.ENOENT: # no such file or directory + pass + elif e.errno == errno.ENOTEMPTY: # there's something left + if exclude: # but it is excluded + pass + else: # likely during Ctrl+C something additional data + try_again = True + retries -= 1 + if retries <= 0: + raise + time.sleep(2) + elif selinux and (e.errno == errno.EPERM or e.errno == errno.EACCES): + try_again = True + if failed_filename == e.filename: + raise + failed_filename = e.filename + os.system("chattr -R -i %s" % path) + elif e.errno == errno.EBUSY: + retries -= 1 + if retries <= 0: + raise + try_again = True + getLog().debug("retrying failed tree remove after sleeping a bit") + time.sleep(2) + else: + raise + + +def is_in_dir(path, directory): + """Tests whether `path` is inside `directory`.""" + # use realpath to expand symlinks + path = os.path.realpath(path) + directory = os.path.realpath(directory) + + return os.path.commonprefix([path, directory]) == directory + + +def get_fs_type(path): + cmd = ['/bin/stat', '-f', '-L', '-c', '%T', path] + p = subprocess.Popen(cmd, shell=False, stdout=subprocess.PIPE, + universal_newlines=True) + p.wait() + with p.stdout as f: + return f.readline().strip() + + +def find_non_nfs_dir(): + dirs = ('/dev/shm', '/run', '/tmp', '/usr/tmp', '/') + for d in dirs: + if not get_fs_type(d).startswith('nfs'): + return d + raise exception.Error('Cannot find non-NFS directory in: %s' % dirs) + + +def unlink_if_exists(path): + """ + Unlink, ignore FileNotFoundError, but keep raising other exceptions. + """ + try: + os.unlink(path) + except FileNotFoundError: + pass + + +def _best_effort_removal(path, use_rmtree=True): + try: + os.unlink(path) + except OSError: + pass + if not use_rmtree: + return + try: + shutil.rmtree(path) + except OSError: + pass + + +def update_tree(dest, src): + """ + Copy files from SRC directory into DEST, recursively. The DEST directory + is created, including subdirectories (if not existent). The files in DEST + are created or updated (shutil.copy2). If file is about to replace + directory or vice versa, it is done without asking. Files that are in DEST + and not in SRC are kept untouched. + """ + + getLog().debug("Updating files in %s with files from %s", dest, src) + + mkdirIfAbsent(dest) + + for dirpath, dirnames, filenames in os.walk(src): + raw_subpath = os.path.relpath(dirpath, src) + subpath = os.path.normpath(raw_subpath) + destpath = os.path.join(dest, subpath) + + for filename in filenames: + file_from = os.path.join(dirpath, filename) + file_to = os.path.join(destpath, filename) + _best_effort_removal(file_to) + shutil.copy2(file_from, file_to) + + for subdir in dirnames: + dest_subdir = os.path.join(destpath, subdir) + _best_effort_removal(dest_subdir, use_rmtree=False) + mkdirIfAbsent(dest_subdir) diff --git a/mock/py/mockbuild/installed_packages.py b/mock/py/mockbuild/installed_packages.py new file mode 100644 index 0000000..f305ad2 --- /dev/null +++ b/mock/py/mockbuild/installed_packages.py @@ -0,0 +1,146 @@ +""" +Helper methods for getting list of installed packages, and corresponding +packages' metadata +""" + +import os +import subprocess +import mockbuild.exception + + +def _subprocess_executor(command): + """ + We use doOutChroot normally in query_packages* methods, this is a helper + for testing purposes. + """ + return subprocess.check_output(command, env={"LC_ALL": "C"}).decode("utf-8") + + +def query_packages(fields, chrootpath=None, executor=_subprocess_executor): + """ + Query the list of installed packages, including FIELDS metadata, from + CHROOTPATH. + + The FIELDS argument is an array of RPM tags from 'rpm --querytags', without + the '%{}' syntax, for example ['name'] queries for %{name}'. There's an + additional non-standard "signature" field parsed from the standard + "%{sigpgp:pgpsig}" field (the last 8 hex characters). + + CHROOTPATH is the chroot directory with RPM DB. If CHROOTPATH is not + specified, the method uses the rpmdb from host. + + EXECUTOR is a callback accepting a single argument - command that will be + executed, and its standard output returned as unicode multiline string. + + The method returns a list of dictionaries (package metadata info) in a + format documented on + https://docs.pagure.org/koji/content_generator_metadata/#buildroots + For example: + + [{ + "license": "LicenseRef-Fedora-Public-Domain", + "name": "filesystem", + "version": "3.18", + "release": "23.fc41", + "arch": "x86_64", + "epoch": null, + "sigmd5": "dc6edb2b7e390e5f0994267d22b9dc1a", + "signature": null + }] + """ + package_list_cmd = ["rpm", "-qa"] + if chrootpath: + package_list_cmd += ["--root", chrootpath] + package_list_cmd.append("--qf") + + # HACK: Zero-termination is not possible with 'rpm -q --qf QUERYSTRIG', so + # this is a hack. But how likely we can expect the following string in the + # real packages' metadata? + separator = '|/@' + + def _query_key(key): + # The Koji Content Generator's "signature" field can be queried via %{sigpgp} + if key == "signature": + return "sigpgp:pgpsig" + return key + + query_fields = [_query_key(f) for f in fields] + package_list_cmd.append(separator.join(f"%{{{x}}}" for x in query_fields) + "\n") + + def _fixup(package): + """ polish the package's metadata output """ + key = "signature" + if key in package: + if package[key] == "(none)": + package[key] = None + else: + # RSA/SHA256, Mon Jul 29 10:12:32 2024, Key ID 2322d3d94bf0c9db + # Get just last 8 chars ---> ^^^^^^^^ + package[key] = package[key].split()[-1][-8:] + key = "epoch" + if package[key] == "(none)": + package[key] = None + return package + + return [_fixup(p) for p in [dict(zip(fields, line.split(separator))) for + line in + sorted(executor(package_list_cmd).splitlines())] + if p["name"] != "gpg-pubkey"] + + +def query_packages_location(packages, chrootpath=None, + executor=_subprocess_executor, dnf_cmd="/bin/dnf"): + """ + Detect the URLs of the PACKAGES - array of dictionaries (see the output + from query_packages()) in available RPM repositories (/etc/yum.repos.d). + This method modifies PACKAGES in-situ, it adds "url" field to every single + dictionary in the PACKAGES array. + + CHROOTPATH is the chroot directory with RPM DB, if not specified, rpmdb + from host is used. + + EXECUTOR is a callback accepting a single argument - command that will be + executed, and its standard output returned as unicode multiline string. + + Example output: + + [{ + "name": "filesystem", + "version": "3.18", + ... + "url": "https://example.com/fedora-repos-rawhide-42-0.1.noarch.rpm", + ... + }] + """ + + # Note: we do not support YUM in 2024+ + query_locations_cmd = [dnf_cmd] + if chrootpath: + query_locations_cmd += [f"--installroot={chrootpath}"] + # The -q is necessary because of and similar: + # https://github.com/rpm-software-management/dnf5/issues/1361 + query_locations_cmd += ["repoquery", "-q", "--location"] + query_locations_cmd += [ + f"{p['name']}-{p['version']}-{p['release']}.{p['arch']}" + for p in packages + ] + location_map = {} + for url in executor(query_locations_cmd).splitlines(): + basename = os.path.basename(url) + # name-arch pair should be unique on the box for every installed package + name, _, _ = basename.rsplit("-", 2) + arch = basename.split(".")[-2] + location_map[f"{name}.{arch}"] = url + + failed_packages = [] + for package in packages: + name_arch = f"{package['name']}.{package['arch']}" + try: + package["url"] = location_map[name_arch] + except KeyError: + failed_packages.append(package) + + if failed_packages: + failed_str = ", ".join([f"{p['name']}-{p['version']}-{p['release']}.{p['arch']}" + for p in failed_packages]) + raise mockbuild.exception.Error(f"Can't get location for {failed_str}") diff --git a/mock/py/mockbuild/mounts.py b/mock/py/mockbuild/mounts.py new file mode 100644 index 0000000..c528851 --- /dev/null +++ b/mock/py/mockbuild/mounts.py @@ -0,0 +1,362 @@ +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +import grp +import os +import os.path +from contextlib import contextmanager + +from . import file_util +from . import exception +from . import util +from .trace_decorator import traceLog + + +class MountPoint(object): + '''base class for mounts''' + @traceLog() + def __init__(self, mountsource, mountpath): + self.mountpath = mountpath + self.mountsource = mountsource + self.mounted = None + self._is_chroot = False + + def treat_as_chroot(self): + """ + If we use this directory as chroot, we might want to do special + actions while mounting and unmounting. + """ + self._is_chroot = True + return self + + @traceLog() + # pylint: disable=unused-argument + def umount(self, force=False, nowarn=False): + """ + Return None if not mounted, and True if successfully umounted + """ + if not self.mounted: + return None + + if self._is_chroot: + # Don't keep background processes running with unmounted + # /proc/self/root directory. + util.orphansKill(self.mountpath) + + if self._do_umount(): + self.mounted = False + return True + return False + + def mount(self): + """ + Mount the mountpoint. Return True if successful, False if not + successful and None if already mounted. + """ + raise NotImplementedError + + @contextmanager + def having_mounted(self): + """ + Mount this mountpoint and then automatically umount + """ + umount = self.mount() + try: + yield + finally: + if umount: + self.umount() + + def _do_umount(self): + raise NotImplementedError + + @traceLog() + def ismounted(self): + with open('/proc/mounts') as f: + if self.mountpath.rstrip('/') in [x.split()[1] for x in f]: + return True + return False + + def __repr__(self): + return "".format( + self.mountsource, self.mountpath) + + +class FileSystemMountPoint(MountPoint): + '''class for managing filesystem mounts in the chroot''' + @traceLog() + def __init__(self, path, filetype=None, device=None, options=None): + if not path: + raise RuntimeError("no path specified for mountpoint") + if not filetype: + raise RuntimeError("no filetype specified for mountpoint") + if filetype in ('pts', 'proc', 'sys', 'sysfs', 'tmpfs', 'devpts'): + device = filetype + if not device: + raise RuntimeError("no device file specified for mountpoint") + + MountPoint.__init__(self, mountsource=device, mountpath=path) + self.device = device + self.path = path + self.filetype = filetype + self.options = options + self.mounted = self.ismounted() + + @traceLog() + def mount(self): + if self.mounted: + return None + + file_util.mkdirIfAbsent(self.path) + cmd = ['/bin/mount', '-n', '-t', self.filetype] + if self.options: + cmd += ['-o', self.options] + cmd += [self.device, self.path] + util.do(cmd) + self.mounted = True + return True + + def _do_umount(self): + cmd = ['/bin/umount', '-n', '-l', self.path] + try: + util.do(cmd) + except exception.Error: + return False + return True + + def __repr__(self): + return ("".format( + self.device, self.path, self.filetype, self.options, self.mounted)) + + +class BindMountPoint(MountPoint): + '''class for managing bind-mounts in the chroot''' + @traceLog() + def __init__(self, srcpath, bindpath, recursive=False, options=None): + MountPoint.__init__(self, mountsource=srcpath, mountpath=bindpath) + self.srcpath = srcpath + self.bindpath = bindpath + self.recursive = recursive + self.options = options + self.mounted = self.ismounted() + + @traceLog() + def mount(self): + if self.mounted: + return None + if os.path.isdir(self.srcpath): + file_util.mkdirIfAbsent(self.bindpath) + elif not os.path.exists(self.bindpath): + normbindpath = os.path.normpath(self.bindpath) + file_util.mkdirIfAbsent(os.path.dirname(normbindpath)) + file_util.touch(self.bindpath) + bind_option = 'rbind' if self.recursive else 'bind' + util.do(['/bin/mount', '-n', '-o', bind_option, self.srcpath, + self.bindpath]) + self.mounted = True + # Remount the new bind-mount to set specified options (rhbz#1584443). + # Userspace must implement this as separate system calls anyway. + if self.options: + options = ','.join(['remount', self.options, bind_option]) + util.do(['/bin/mount', '-n', '-o', options, "--target", + self.bindpath]) + return True + + @traceLog() + def _do_umount(self): + cmd = ['/bin/umount', '-n'] + if self.recursive: + # The mount is busy because of the submounts - a lazy unmount + # implies a recursive unmount, so takes care of that. + # (-R also works, but is implemented in userspace, and thus racy) + cmd += ['-l'] + cmd.append(self.bindpath) + try: + util.do(cmd) + except exception.Error: + return False + return True + + def __repr__(self): + return "".format( + self.srcpath, self.bindpath, self.mounted) + +class Mounts(object): + '''class to manage all mountpoints''' + @traceLog() + def __init__(self, rootObj): + self.rootObj = rootObj + self.essential_mounts = [] # /proc, /sys ... normally managed by systemd + self.managed_mounts = [] # mounts owned by mock + self.user_mounts = [] # mounts injected by user + self.bootstrap_mounts = [] + + # Instead of mounting a fresh procfs and sysfs, we bind mount /proc + # and /sys. This avoids problems with kernel restrictions if running + # within a user namespace, and is pretty much identical otherwise. + # The bind mounts additionally need to be recursive, because the + # kernel forbids mounts that might reveal parts of the filesystem + # that a container runtime overmounted to hide from the container + # (rhbz#1745048). + for mount in ['proc', 'sys']: + mount_point = "/" + mount + device = 'mock_hide_{}fs_from_host'.format(mount) + host_path = rootObj.make_chroot_path(mount_point) + + self.essential_mounts += [ + # The recursive mount point needs to be later lazy umounted and + # it would affect hosts's counterpart sub-mounts as well. To + # avoid this, we need to make the mount point and parent mount + # point private in unshare()d namespace. But since the parent + # mount point of /sys and /proc so far was plain '/' mount (and + # we need to keep that one shared, to keep LVM/tmpfs features + # working) we crate a new parent mount for the final mountpoint + # on the same path. So the mount graph looks like: + # / (shared) -> /sys (private) -> /sys (recursive, private) + # + # Acknowledgement, IOW: We mount on host_path twice and it is + # expected. This is because when you umount 'rprivate' mount + # then parent mount point is notified .. so first we mount tmpfs + # stub which we actually never use -- but is private -- and only + # then we mount above the actual mount point. This prevents + # from umount events to propagate to host from chroot. + FileSystemMountPoint(filetype='tmpfs', + device=device, + path=host_path, + options="rprivate,mode=0755"), + BindMountPoint(srcpath=mount_point, + bindpath=host_path, + recursive=True, + options="nodev,noexec,nosuid,readonly,rprivate"), + ] + + if rootObj.config['internal_dev_setup']: + self.essential_mounts.append( + FileSystemMountPoint( + filetype='tmpfs', + device='mock_chroot_shmfs', + path=rootObj.make_chroot_path('/dev/shm') + ) + ) + opts = 'gid=%d,mode=0620,ptmxmode=0666' % grp.getgrnam('tty').gr_gid + if util.cmpKernelVer(os.uname()[2], '2.6.29') >= 0: + opts += ',newinstance' + self.essential_mounts.append( + FileSystemMountPoint( + filetype='devpts', + device='mock_chroot_devpts', + path=rootObj.make_chroot_path('/dev/pts'), + options=opts + ) + ) + self._essential_mounted = all(m.ismounted() for m in self.essential_mounts) + + @traceLog() + def add(self, mount): + self.managed_mounts.append(mount) + + @traceLog() + def add_device_bindmount(self, path): + mount = BindMountPoint(path, + self.rootObj.make_chroot_path(path), + options="noexec,nosuid,readonly") + self.essential_mounts.append(mount) + + @traceLog() + def add_user_mount(self, mount): + self.user_mounts.append(mount) + + @traceLog() + def mountall_essential(self): + self._essential_mounted = True + for m in self.essential_mounts: + m.mount() + + @traceLog() + def _mount_bootstrap(self): + with self.rootObj.uid_manager.elevated_privileges(): + for m in self.bootstrap_mounts: + m.mount() + + @traceLog() + def _umount_bootstrap(self): + # Kill leftover processes in the bind-mountpoint, typically these + # processes can be started by DNF/RPM via buggy scriptlets. + util.orphansKill(self.rootObj.make_chroot_path(), manual_forced=True) + with self.rootObj.uid_manager.elevated_privileges(): + for m in reversed(self.bootstrap_mounts): + m.umount() + + @contextmanager + def essential_mounted(self, noop=False): + """ + Convenience wrapper around commands that need essential mountpoints + mounted. + """ + do_umount = False + try: + if not noop and not self._essential_mounted: + do_umount = True + self.mountall_essential() + yield + finally: + if do_umount: + self.umountall_essential() + + + @contextmanager + def buildroot_in_bootstrap_mounted(self): + """ + Context manager! Mount the chroot into bootstrap recursively, execute + the command within the context, and (lazy) umount it. + """ + self._mount_bootstrap() + try: + yield + finally: + self._umount_bootstrap() + + @traceLog() + def mountall_managed(self): + if not util.USE_NSPAWN: + self.mountall_essential() + for m in self.managed_mounts: + m.mount() + + @traceLog() + def mountall_user(self): + for m in self.user_mounts: + m.mount() + + @traceLog() + # pylint: disable=unused-argument + def umountall(self, force=False, nowarn=False): + failed_old = 1 + failed_new = 0 + while (failed_new != failed_old): + # there can be deps, we will try to umount everything several times + # as long as in every loop at least one umount succeed. + failed_old = failed_new + failed_new = 0 + for m in reversed(self.managed_mounts + self.user_mounts): + if m.umount() is False: + failed_new += 1 + if self._essential_mounted: + self.umountall_essential() + + @traceLog() + def umountall_essential(self): + for m in reversed(self.essential_mounts): + m.umount() + self._essential_mounted = False + + @traceLog() + def get_mountpoints(self): + # including essentials (no matter if we use nspawn) + # this is used to exclude path in archiving etc. and we want to do that for essentials too + return [m.mountpath for m in self.essential_mounts + self.managed_mounts + self.user_mounts] + + def __repr__(self): + return "".format(repr(self.managed_mounts), + repr(self.user_mounts)) diff --git a/mock/py/mockbuild/package_manager.py b/mock/py/mockbuild/package_manager.py new file mode 100644 index 0000000..8a88480 --- /dev/null +++ b/mock/py/mockbuild/package_manager.py @@ -0,0 +1,769 @@ +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +import copy +import glob +import os.path +import shutil +import sys +import time +import re +from textwrap import dedent +from configparser import ConfigParser + +from . import file_util +from . import util +from .exception import BuildError, Error, YumError +from .trace_decorator import traceLog, getLog +from .mounts import BindMountPoint + +fallbacks = { + 'dnf4': ['dnf4', 'dnf5', 'yum'], + 'yum': ['yum', 'dnf4', 'dnf5'], + 'microdnf': ['microdnf', 'dnf4', 'dnf5', 'yum'], + 'dnf5': ['dnf5', 'dnf4', 'yum'], +} + + +def _package_manager_from_string(name): + if name == 'dnf5': + return Dnf5 + if name == 'yum': + return Yum + if name in 'dnf4': + return Dnf + if name == 'microdnf': + return MicroDnf + raise RuntimeError(f'Unrecognized package manager "{name}"') + + +def _package_manager_exists(pm_class, config_opts, chroot=None): + name = pm_class.name + command = pm_class.get_command(config_opts) + pathname = (chroot or "") + command + if not os.path.isfile(pathname): + return False + # resolve symlinks, and detect that e.g. /bin/yum doesn't point to /bin/dnf + real_pathname = os.path.realpath(pathname) + if name == 'dnf4': + # The DNF4 used to be /bin/dnf, not /bin/dnf4 + name = 'dnf' + return name in real_pathname + + +def _package_manager_class_fallback(config_opts, buildroot, fallback): + desired = config_opts['package_manager'] + + if desired == 'dnf': # backward compat + desired = 'dnf4' + + if not fallback: + return _package_manager_from_string(desired) + + getLog().debug("searching for '%s' package manager or alternatives", desired) + if desired not in fallbacks: + raise RuntimeError(f'Unexpected package manager "{desired}"') + + chroot_to_search_in = None # by default we search for PMs on host + if buildroot.uses_bootstrap_image: + # alternatively in the (extracted) bootstrap image + chroot_to_search_in = buildroot.make_chroot_path() + + bootstrap = buildroot.is_bootstrap + + for manager in fallbacks[desired]: + pm_class = _package_manager_from_string(manager) + if _package_manager_exists(pm_class, config_opts, chroot=chroot_to_search_in): + if desired == manager: + return pm_class + + getLog().info("Using '%s' instead of '%s'%s", manager, desired, + " for bootstrap chroot" if bootstrap else "") + + if 'dnf_warning' in config_opts and not config_opts['dnf_warning']: + return pm_class + + if not bootstrap: + # pylint: disable=consider-using-f-string + print("""WARNING! WARNING! WARNING! +You are building package for distribution which uses {0}. However your system +does not support {0}. You can continue with {1}, which will likely succeed, +but the installed chroot may look a little different. + 1. Please consider --bootstrap-chroot option, or + 2. install {0} on your host system. +You can suppress this warning when you put + config_opts['dnf_warning'] = False +in Mock config.""".format(desired.upper(), manager.upper())) + input("Press Enter to continue.") + + return pm_class + + raise RuntimeError(f"No package from {fallbacks[desired]} found, desired {desired}") + + +def package_manager(buildroot, bootstrap_buildroot, fallback): + cls = _package_manager_class_fallback(buildroot.config, buildroot, fallback) + return cls(buildroot.config, buildroot, buildroot.plugins, + bootstrap_buildroot) + + +class _PackageManager(object): + # pylint: disable=too-many-instance-attributes + name = None + command = None + builddep_command = None + resolvedep_command = None + # When support_installroot is False then command is run in target chroot + # you must ensure that `command` is available in the chroot + support_installroot = True + # We specify (historical reasons) opts before args for package managers. + # But DNF5 requires the common opts (--allowerasing etc.) specified after + # the sub-command (like 'install --allowerasing', instead of + # '--allowerasing install'). + place_common_opts_after = False + + @classmethod + def get_command(cls, config): + command = config.get(f"{cls.name}_command") + if config.get("use_bootstrap_image", True): + return command + if config.get("use_bootstrap", False): + sys_command = config.get(f"{cls.name}_system_command") + if sys_command: + return sys_command + return command + + + @traceLog() + def __init__(self, config, buildroot, plugins, bootstrap_buildroot): + self.config = config + self.plugins = plugins + # the buildroot we install into + self.buildroot = buildroot + self.init_install_output = "" + self.bootstrap_buildroot = bootstrap_buildroot + # self.buildroot generated from bootstrap image + self.is_bootstrap_image = buildroot.uses_bootstrap_image + self.pkg_manager_config = "" + + self.command = self.get_command(config) + + self.builddep_command = [self.command, 'builddep'] + builddep_override = config.get(f"{self.name}_builddep_command") + if builddep_override: + self.builddep_command = [builddep_override, "builddep"] + + self.common_opts = copy.copy(config.get(f'{self.name}_common_opts', [])) + + disabled_plugins = self.config.get(f"{self.name}_disable_plugins", []) + self.common_opts.extend([f"--disableplugin={x}" for x in + disabled_plugins]) + + if 'forcearch' in self.config and self.config['forcearch']: + self.common_opts.extend(['--forcearch', self.config['forcearch']]) + + def adjust_command_options(self, command, opts): + """ + E.g. --allowerasing is supported by DNF5 in general, but is not + supported by DNF5 builddep plugin (yet), therefore we can configure: + config_opts["dnf5_avoid_opts"]["builddep"] = ["--allowerasing"] + https://github.com/rpm-software-management/dnf5/issues/461 + """ + config = self.config.get(f"{self.name}_avoid_opts", {}) + if command in config: + return [o for o in opts if o not in config[command]] + return opts + + def log_package_management_packages(self): + """ + Log out the versions of packages related to package management. + """ + if self.buildroot.is_bootstrap: + # no-op for the bootstrap chroot; we don't care how this has been + # installed + return + + cmd = [ + "rpm", "-q", + "rpm", "rpm-sequoia", + "python3-dnf", "python3-dnf-plugins-core", + "yum", "yum-utils", + "dnf5", "dnf5-plugins", + ] + + + def _do(comment, *args, **kwargs): + info = "Buildroot is handled by package management" + output = util.do(*args, **kwargs, returnOutput=True, + raiseExc=False).strip() + output = "\n".join([" " + line for line in output.split("\n") + if "is not installed" not in line]) + getLog().info("%s %s:\n%s", info, comment, output) + + # We want to know the package versions in bootstrap + if self.bootstrap_buildroot: + if self.bootstrap_buildroot.use_chroot_image: + # rpm installed from the bootstrap image + _do("downloaded with a bootstrap image", cmd, + chrootPath=self.bootstrap_buildroot.make_chroot_path()) + else: + # rpm installed into bootstrap by host's package management + _do("installed into bootstrap", cmd + [ + "--root", self.bootstrap_buildroot.make_chroot_path() + ]) + else: + # Execute with installroot from host + _do("from host and used with --installroot", cmd) + + @traceLog() + def build_invocation(self, *args): + invocation = [] + args = list(args) + cmd = args[0] + opts = self.adjust_command_options(cmd, copy.copy(self.common_opts)) + if cmd == 'builddep': + args = args[1:] + invocation += self.builddep_command + opts += self.config.get(self.name + '_builddep_opts', []) + elif cmd == 'resolvedep': + if self.resolvedep_command: + args = args[1:] + invocation = self.resolvedep_command + else: + invocation = [self.command] + else: + invocation = [self.command] + if self.support_installroot and not self.is_bootstrap_image: + invocation += ['--installroot', self.buildroot.make_chroot_path('')] + if cmd in ['upgrade', 'update', 'module']: + invocation += ['-y'] + releasever = self.config['releasever'] + if releasever: + invocation += ['--releasever', str(releasever)] + if not self.config['online']: + invocation.append('-C') + if self.config['enable_disable_repos']: + invocation += self.config['enable_disable_repos'] + + invocation += (args + opts) if self.place_common_opts_after else (opts + args) + + return invocation + + @traceLog() + def get_pkg_manager_config(self): + if 'dnf.conf' in self.config: + return self.config['dnf.conf'] + else: + return self.config['yum.conf'] + + def execute(self, *args, **kwargs): + """ + Execute package manger command (via self._execute_mounted where the + args[] form the command, self.build_invocation). Make sure the + essential and the recursive buildroot-in-bootstrap mount points are + (un)mounted correctly). + """ + self.plugins.call_hooks("preyum") + + # systemd-nspawn v253.9 started to dislike our pre-created essential + # mountpoints in `-D rootdir`. Previous versions silently overmounted + # them (Copr issue#2906). Note that we still need essential mountpoints + # if DNF is run with NSPAWN with --installroot (so we do this only if + # is_bootstrap_image is True). + skip_essential_mounts = util.USE_NSPAWN and self.is_bootstrap_image + + try: + with self.buildroot.mounts.essential_mounted(noop=skip_essential_mounts): + with self.buildroot.mounts.buildroot_in_bootstrap_mounted(): + return self._execute_mounted(*args, **kwargs) + finally: + self.plugins.call_hooks("postyum") + + + @traceLog() + def _execute_mounted(self, *args, **kwargs): + + # intentionally we do not call bootstrap hook here - it does not have sense + env = self.config['environment'].copy() + env.update(util.get_proxy_environment(self.config)) + # installation-time specific homedir + env['HOME'] = self.buildroot.prepare_installation_time_homedir() + env['LC_MESSAGES'] = 'C.UTF-8' + if self.buildroot.nosync_path: + env['LD_PRELOAD'] = self.buildroot.nosync_path + invocation = self.build_invocation(*args) + self.buildroot.root_log.debug(invocation) + kwargs['printOutput'] = kwargs.get('printOutput', True) + if not self.config['print_main_output']: + kwargs.pop('printOutput', None) + else: + kwargs.setdefault("pty", True) + self.buildroot.nuke_rpm_db() + + error = None + max_attempts = int(self.config['package_manager_max_attempts']) + for attempt in range(max(max_attempts, 1)): + if error: + sleep_seconds = int(self.config['package_manager_attempt_delay']) + self.buildroot.root_log.warning( + "%s command failed, retrying, attempt #%s, sleeping %ss", + self.name.upper(), attempt + 1, sleep_seconds) + time.sleep(sleep_seconds) + + try: + # either it does not support --installroot (microdnf) or + # it is bootstrap image made by container with incomaptible dnf/rpm + personality = kwargs.pop("personality", None) + if self.buildroot.is_bootstrap and not self.buildroot.config["forcearch"]: + personality = self.buildroot.config["repo_arch"] + + if not self.support_installroot or self.is_bootstrap_image: + out = util.do(invocation, env=env, + chrootPath=self.buildroot.make_chroot_path(), + personality=personality, **kwargs) + elif self.bootstrap_buildroot is None: + + + out = util.do(invocation, env=env, + personality=personality, **kwargs) + else: + out = util.do(invocation, env=env, + chrootPath=self.bootstrap_buildroot.make_chroot_path(), + nspawn_args=self.bootstrap_buildroot.config['nspawn_args'], + personality=personality, **kwargs) + error = None + break + except Error as e: + error = YumError(str(e)) + + if error is not None: + raise error + + # intentionally we do not call bootstrap hook here - it does not have sense + return out + + @traceLog() + # pylint: disable=unused-argument + def install(self, *args, **kwargs): + return self.execute('install', *args) + + # pylint: disable=unused-argument + @traceLog() + def remove(self, *args, **kwargs): + return self.execute('remove', *args) + + # pylint: disable=unused-argument + @traceLog() + def update(self, *args, **kwargs): + return self.execute('update', *args) + + # pylint: disable=unused-argument + @traceLog() + def builddep(self, *args, **kwargs): + try: + result = self.execute('builddep', returnOutput=1, *args) + except (FileNotFoundError) as error: + er = str(error) + if "builddep" in er: + print(error) + print(""" +Error: Neither dnf-utils nor yum-utils are installed. Dnf-utils or yum-utils are needed to complete this action. + To install dnf-utils use: + $ dnf install dnf-utils + or yum-utils: + $ yum install yum-utils""") + sys.exit(120) + else: + raise + return result + + @traceLog() + def copy_distribution_gpg_keys(self): + # Copy the files from the host to avoid invoking package manager + # or rebuilding the cached bootstrap chroot. + keys_path = "/usr/share/distribution-gpg-keys" + dest_path = os.path.dirname(keys_path) + chroot_path = self.buildroot.make_chroot_path(dest_path) + file_util.mkdirIfAbsent(chroot_path) + self.buildroot.root_log.debug("Copying %s to the bootstrap chroot" % keys_path) + cmd = ["cp", "-a", keys_path, chroot_path] + util.do(cmd) + + @traceLog() + def copy_gpg_keys(self): + pki_dir = self.buildroot.make_chroot_path('etc', 'pki', 'mock') + file_util.mkdirIfAbsent(pki_dir) + for pki_file in glob.glob("/etc/pki/mock/RPM-GPG-KEY-*"): + shutil.copy(pki_file, pki_dir) + + @traceLog() + def copy_certs(self): + copied_ca_cert_paths = self.config['ssl_copied_ca_trust_dirs'] + if copied_ca_cert_paths: + for host_path, root_path in copied_ca_cert_paths: + self.buildroot.root_log.debug('copying CA trust dir into chroot: %s => %s', host_path, root_path) + dest_dir = self.buildroot.make_chroot_path(root_path) + file_util.update_tree(dest_dir, host_path) + + bundle_path = self.config['ssl_ca_bundle_path'] + if bundle_path: + self.buildroot.root_log.debug('copying CA bundle into chroot') + host_bundle = os.path.realpath('/etc/pki/tls/certs/ca-bundle.crt') + chroot_bundle_path = self.buildroot.make_chroot_path(bundle_path) + chroot_bundle_dir = os.path.dirname(chroot_bundle_path) + + file_util.mkdirIfAbsent(chroot_bundle_dir) + try: + shutil.copy(host_bundle, chroot_bundle_path) + except FileNotFoundError: + # when mock is not run on Fedora or EL + self.buildroot.root_log.debug("ca bundle not found on host") + + @traceLog() + def copy_extra_certs(self): + extra_certs = self.config['ssl_extra_certs'] + if extra_certs: + self.buildroot.root_log.debug('copying extra certificates into chroot') + for cert_src, cert_dest in zip(extra_certs[::2], extra_certs[1::2]): + host_cert_src = os.path.realpath(cert_src) + chroot_cert_dest = self.buildroot.make_chroot_path(cert_dest) + chroot_cert_dir = os.path.dirname(chroot_cert_dest) + file_util.mkdirIfAbsent(chroot_cert_dir) + try: + shutil.copy(host_cert_src, chroot_cert_dest) + except FileNotFoundError: + # when mock is not run on Fedora or EL + self.buildroot.root_log.debug("extra certificates not found on host") + + + def initialize(self): + self.copy_gpg_keys() + self.copy_certs() + if self.buildroot.is_bootstrap: + self.copy_distribution_gpg_keys() + self.copy_extra_certs() + self.initialize_config() + + try: + self._bind_mount_repos_to_bootstrap() + except Exception as e: + getLog().warning(e) + + def expand_url_vars(self, string): + """ + Expand DNF variables like $baseurl to proper values in string, and + return it. + """ + expand = { + "basearch": self.config["target_arch"] or '', + "releasever": self.config["releasever"] or '', + } + + if 'dnf_vars' in self.config: + for key in self.config['dnf_vars']: + expand[key] = self.config['dnf_vars'][key] + + for key, value in expand.items(): + # replace both $key and ${key} + # dnf allows braced variables + string = string.replace('$' + key, value) + string = string.replace('${' + key + '}', value) + + return string + + def _bind_mount_repos_to_bootstrap(self): + if not self.buildroot.is_bootstrap: + return + + parse = { + "baseurl": (True, re.compile("[ \t\n,]")), + "mirrorlist": (False, None), + "metalink": (False, None), + } + + # in dnf, the last occurence of the same option beats the previous + config = ConfigParser(strict=False, interpolation=None) + config.read_string(self.pkg_manager_config) + + # don't bindmount the same paths multiple times + tried = set() + + for section in config.sections(): + for option in parse: + directory, split_re = parse[option] + + if option not in config[section]: + continue + + raw = config[section][option] + items = split_re.split(raw) if split_re else [raw] + + for value in items: + value = value.strip() + if not value: + continue + + # triple slash, we only accept absolute pathnames + if value.startswith('file:///'): + srcpath = value[7:] + elif value.startswith('/'): + srcpath = value + else: + continue + + srcpath = self.expand_url_vars(srcpath) + + if srcpath in tried: + continue + + tried.add(srcpath) + + if directory and not os.path.isdir(srcpath): + continue + + if not os.path.exists(srcpath): + continue + + bindpath = self.buildroot.make_chroot_path(srcpath) + bind_mount_point = BindMountPoint(srcpath=srcpath, + bindpath=bindpath) + + # This is a very tricky hack. Note we configure the + # package_manager for the "bootstrap" chroot here, but these + # "local repo" mountpoints are actually needed by both + # "bootstrap" and "build" chroots. The "bootstrap" chroot + # needs this with the 'bootstrap_image' feature (we use + # package manager _in bootstrap_, not on host, to install + # into the bootstrap) and the "build" chroot package manager + # always needs this (but also mounted in bootstrap). That's + # why we are not using "essential mounts"; these are only + # automatically mounted by the corresponding package manager + # (we wouldn't mount bootstrap's mountpoints when installing + # into the "build" chroot). + self.buildroot.mounts.add(bind_mount_point) + + def initialize_config(self): + # there may be configs we get from container image + file_util.rmtree(self.buildroot.make_chroot_path('etc', 'yum.repos.d')) + + def _check_command(self): + """ Check if main command exists """ + + command = self.command + + if self.bootstrap_buildroot: + # the command in bootstrap may not exists yet + return + + if self.is_bootstrap_image: + # with bootstrap image, we don't work with the host's package + # manager at all. + command = self.buildroot.make_chroot_path(command) + + if not os.path.exists(command): + raise Exception("""Command {0} is not available. Either install package containing this command +or run mock with --yum or --dnf to overwrite config value. However this may +lead to different dependency solving!""".format(command)) + + +def check_yum_config(config, log): + if '\nreposdir' not in config: + log.warning(dedent("""\ + reposdir option is not set in yum config. That means Yum/DNF + will use system-wide repos. To suppress that behavior, put + reposdir=/dev/null to your yum.conf in mock config. + """)) + + +class Yum(_PackageManager): + name = 'yum' + support_installroot = True + + def __init__(self, config, buildroot, plugins, bootstrap_buildroot): + super(Yum, self).__init__(config, buildroot, plugins, bootstrap_buildroot) + self.builddep_command = [config['yum_builddep_command']] + if bootstrap_buildroot is not None: + # we are in bootstrap so use configured names + yum_deprecated_path = config['yum_command'] + yum_builddep_deprecated_path = config['yum_builddep_command'] + yum_deprecated_path = bootstrap_buildroot.make_chroot_path(yum_deprecated_path) + yum_builddep_deprecated_path = bootstrap_buildroot.make_chroot_path(yum_builddep_deprecated_path) + else: + # we are building bootstrap or do not use bootstrap at all + yum_deprecated_path = '/usr/bin/yum-deprecated' + yum_builddep_deprecated_path = '/usr/bin/yum-builddep-deprecated' + if os.path.exists(yum_deprecated_path): + yum_repoquery_command = '/usr/bin/repoquery' + if os.path.exists(yum_repoquery_command + '-deprecated'): + yum_repoquery_command = yum_repoquery_command + '-deprecated' + self.command = '/usr/bin/yum-deprecated' + self.resolvedep_command = [ + yum_repoquery_command, '--resolve', '--requires', + '--config', self.buildroot.make_chroot_path('etc', 'yum', 'yum.conf')] + if os.path.exists(yum_builddep_deprecated_path): + self.builddep_command = ['/usr/bin/yum-builddep-deprecated'] + + self._check_command() + + @traceLog() + def _write_plugin_conf(self, name): + """ Write 'name' file into pluginconf.d """ + conf_path = self.buildroot.make_chroot_path('etc', 'yum', 'pluginconf.d', name) + with open(conf_path, 'w+') as conf_file: + conf_file.write(self.config[name]) + + @traceLog() + def initialize_config(self): + super(Yum, self).initialize_config() + # use yum plugin conf from chroot as needed + pluginconf_dir = self.buildroot.make_chroot_path('etc', 'yum', 'pluginconf.d') + file_util.mkdirIfAbsent(pluginconf_dir) + config_content = self.get_pkg_manager_config().replace( + "plugins=1", dedent("""\ + plugins=1 + pluginconfpath={0}""".format(pluginconf_dir))) + + self.pkg_manager_config = config_content + check_yum_config(config_content, self.buildroot.root_log) + + # write in yum.conf into chroot + # always truncate and overwrite (w+) + self.buildroot.root_log.debug('configure yum') + yumconf_path = os.path.join('etc', 'yum', 'yum.conf') + # we need dnf too in case that yum is not installed and /usr/bin/yum points in fact to dnf + dnfconf_path = os.path.join('etc', 'dnf', 'dnf.conf') + for conf_path in (yumconf_path, dnfconf_path): + chroot_conf_path = self.buildroot.make_chroot_path(conf_path) + with open(chroot_conf_path, 'w+') as conf_file: + conf_file.write(config_content) + if os.path.exists(conf_path): + shutil.copystat(conf_path, chroot_conf_path) + + # write in yum plugins into chroot + self.buildroot.root_log.debug('configure yum priorities') + self._write_plugin_conf('priorities.conf') + self.buildroot.root_log.debug('configure yum rhnplugin') + self._write_plugin_conf('rhnplugin.conf') + if self.config['subscription-manager.conf']: + self.buildroot.root_log.debug('configure RHSM rhnplugin') + self._write_plugin_conf('subscription-manager.conf') + pem_dir = self.buildroot.make_chroot_path('etc', 'pki', 'entitlement') + file_util.mkdirIfAbsent(pem_dir) + for pem_file in glob.glob("/etc/pki/entitlement/*.pem"): + shutil.copy(pem_file, pem_dir) + consumer_dir = self.buildroot.make_chroot_path('etc', 'pki', 'consumer') + file_util.mkdirIfAbsent(consumer_dir) + for consumer_file in glob.glob("/etc/pki/consumer/*.pem"): + shutil.copy(consumer_file, consumer_dir) + shutil.copy('/etc/rhsm/rhsm.conf', + self.buildroot.make_chroot_path('etc', 'rhsm')) + self.execute('repolist') + + def install(self, *pkgs, **kwargs): + check = kwargs.pop('check', False) + if check: + out = self.execute('resolvedep', *pkgs, returnOutput=True, + printOutput=False, pty=False) + _check_missing(out) + out = super(Yum, self).install(*pkgs, **kwargs) + if check: + _check_missing(out) + + +def _check_missing(output): + for i, line in enumerate(output.split('\n')): + for msg in ('no package found for', 'no packages found for', + 'missing dependency', 'error:'): + if msg in line.lower(): + raise BuildError('\n'.join(output.split('\n')[i:])) + + +class Dnf(_PackageManager): + name = 'dnf4' + support_installroot = True + + def __init__(self, config, buildroot, plugins, bootstrap_buildroot): + super(Dnf, self).__init__(config, buildroot, plugins, bootstrap_buildroot) + self.resolvedep_command = [self.command, 'repoquery', '--resolve', '--requires'] + self._check_command() + + def initialize_vars(self): + self.buildroot.root_log.debug('configure DNF vars') + var_path = self.buildroot.make_chroot_path('etc/dnf/vars/') + for key in self.config['dnf_vars'].keys(): + with open(os.path.join(var_path, key), 'w+') as conf_file: + conf_file.write(self.config['dnf_vars'][key]) + + @traceLog() + def initialize_config(self): + super(Dnf, self).initialize_config() + config_content = self.get_pkg_manager_config() + self.pkg_manager_config = config_content + check_yum_config(config_content, self.buildroot.root_log) + file_util.mkdirIfAbsent(self.buildroot.make_chroot_path('etc', 'dnf')) + dnfconf_path = self.buildroot.make_chroot_path('etc', 'dnf', 'dnf.conf') + with open(dnfconf_path, 'w+') as dnfconf_file: + dnfconf_file.write(config_content) + self.initialize_vars() + + def builddep(self, *pkgs, **kwargs): + try: + super(Dnf, self).builddep(*pkgs, **kwargs) + except Error as e: + # pylint: disable=unused-variable + for i, line in enumerate(e.msg.split('\n')): + if 'no such command: builddep' in line.lower(): + raise BuildError("builddep command missing.\nPlease install package dnf-plugins-core.") + raise + + +class MicroDnf(Dnf): + name = 'microdnf' + support_installroot = False + + def __init__(self, config, buildroot, plugins, bootstrap_buildroot): + super(MicroDnf, self).__init__(config, buildroot, plugins, bootstrap_buildroot) + self.saved_releasever = config['releasever'] + self.config['releasever'] = None + + def _check_command(self): + """ Check if main command exists """ + super(MicroDnf, self)._check_command() + if not os.path.exists(self.config['dnf_command']): + raise Exception("""Command {0} is not available. Either install package containing this command +or run mock with --yum or --dnf to overwrite config value. However this may +lead to different dependency solving!""".format(self.config['dnf_command'])) + + @traceLog() + def execute(self, *args, **kwargs): + args_copy = list(copy.copy(args)) + cmd = args_copy[0] + if cmd not in ['update', 'remove', 'install']: + self.command = self.config['dnf_command'] + self.support_installroot = True + self.config['releasever'] = self.saved_releasever + # else it is builddep or resolvedep and we keep command == config['microdnf_command'] + if cmd == "dnf-install": + cmd = args_copy[0] = "install" + result = super(MicroDnf, self).execute(*args_copy, **kwargs) + # restore original value + self.command = self.config['microdnf_command'] + self.support_installroot = False + self.config['releasever'] = None + return result + + +class Dnf5(Dnf): + """ + DNF5 (c++) != DNF4 (python), it has been reimplemented from scratch. + Some options can be missing or have a different semantics. + """ + name = 'dnf5' + place_common_opts_after = True + + def execute(self, *args, **kwargs): + kwargs.setdefault("pty", False) + return super(Dnf, self).execute(*args, **kwargs) + + def update(self, *args, **_kwargs): + return self.execute('upgrade', *args) diff --git a/mock/py/mockbuild/plugin.py b/mock/py/mockbuild/plugin.py new file mode 100644 index 0000000..c6a64c4 --- /dev/null +++ b/mock/py/mockbuild/plugin.py @@ -0,0 +1,88 @@ +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +import importlib.machinery +import importlib.util +import sys + +from .exception import Error +from .trace_decorator import traceLog + +current_api_version = '1.1' + + +class Plugins(object): + @traceLog() + def __init__(self, config, state): + self.config = config + self._hooks = {} + self.state = state + + self.already_initialized = False + + self.plugins = config['plugins'] + self.plugin_conf = config['plugin_conf'] + self.plugin_dir = config['plugin_dir'] + + def __repr__(self): + return " + +# python library imports + +# our imports + +import os.path +from mockbuild.mounts import BindMountPoint +from mockbuild.trace_decorator import traceLog +from mockbuild import file_util + +requires_api_version = "1.1" +# Skip mounting user-specified mounts if we're in the bootstrap chroot +run_in_bootstrap = False + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + BindMount(plugins, conf, buildroot) + + +class BindMount(object): + """bind mount dirs from host into chroot""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = buildroot.config + self.state = buildroot.state + self.bind_opts = conf + plugins.add_hook("postinit", self._bindMountCreateDirs) + for srcdir, destdir in self.bind_opts['dirs']: + buildroot.mounts.add_user_mount( + BindMountPoint( + srcpath=srcdir, + bindpath=buildroot.make_chroot_path(destdir) + ) + ) + + @traceLog() + def _bindMountCreateDirs(self): + for srcdir, destdir in self.bind_opts['dirs']: + if os.path.isdir(srcdir): + file_util.mkdirIfAbsent(srcdir) + file_util.mkdirIfAbsent(self.buildroot.make_chroot_path(destdir)) + else: + dest_file = self.buildroot.make_chroot_path(destdir) + file_util.mkdirIfAbsent(os.path.dirname(dest_file)) + file_util.touch(dest_file) diff --git a/mock/py/mockbuild/plugins/buildroot_lock.py b/mock/py/mockbuild/plugins/buildroot_lock.py new file mode 100644 index 0000000..06bb98b --- /dev/null +++ b/mock/py/mockbuild/plugins/buildroot_lock.py @@ -0,0 +1,116 @@ +""" +Produce a lockfile for the prepared buildroot by Mock. Once available, we +should use the DNF built-in command from DNF5: +https://github.com/rpm-software-management/dnf5/issues/833 +""" + +import json +import os + +from mockbuild.podman import Podman, PodmanError +from mockbuild.installed_packages import query_packages, query_packages_location + +requires_api_version = "1.1" + + +def init(plugins, conf, buildroot): + """ The obligatory plugin entry point """ + BuildrootLockfile(plugins, conf, buildroot) + + +class BuildrootLockfile: + """ Produces buildroot_lock.json file in resultdir """ + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.state = buildroot.state + self.conf = conf + self.inst_done = False + plugins.add_hook("postdeps", self.produce_lockfile) + + def produce_lockfile(self): + """ + Upon a request ('produce_lockfile' option set True), generate + the mock-build-environment.json file in resultdir. The file describes + the Mock build environment, and the way to reproduce it. + """ + + filename = "buildroot_lock.json" + statename = "Generating the buildroot lockfile: " + filename + try: + with self.buildroot.uid_manager: + self.state.start(statename) + out_file = os.path.join(self.buildroot.resultdir, filename) + chrootpath = self.buildroot.make_chroot_path() + + # Ḿimic the Koji Content Generator metadata fields: + # https://docs.pagure.org/koji/content_generator_metadata/#buildroots + # + # The query_packages() method below sorts its output according + # to _values_ of the queried RPM headers, so keep name-arch pair + # first to have the output sorted reasonably. + query_fields = ["name", "arch", "license", "version", "release", + "epoch", "sigmd5", "signature"] + + def _executor(cmd): + out, _ = self.buildroot.doOutChroot(cmd, returnOutput=True, + returnStderr=False) + return out + + packages = query_packages(query_fields, chrootpath, _executor) + query_packages_location(packages, chrootpath, _executor, + dnf_cmd=self.buildroot.pkg_manager.command) + + data = { + # Try to semver. The future Mock versions (the tool) should + # be able to read older Minor versions of the same Major. + # Anytime we break this assumption, bump the Major. IOW, + # the latest Mock implementing with Major == 1 can read any + # version from the 1.Y.Z range. Mock implementing v2.Y.Z + # no longer reads v1.Y.Z variants. + "version": "1.1.0", + "buildroot": { + "rpms": packages, + }, + # Try to keep this as minimal as possible. If possible, + # implement the config options as DEFAULTS in the + # hermetic-build.cfg, or in the + # process_hermetic_build_config() method. + "config": {} + } + for cfg_option in [ + # These are hard-coded in the configuration file, but we + # work with a single-config-for-all-arches now. + "target_arch", + "legal_host_arches", + "dist", + "package_manager", + # At this point, we only support hermetic builds iff + # bootstrap_image_ready=True, so these two options are + # useful for implementing "assertion" in the + # process_hermetic_build_config() method. + "bootstrap_image", + "bootstrap_image_ready", + # Macros need to be inherited, e.g., to keep the original + # %vendor tag specification, or %_host_cpu hacks. + "macros", + ]: + if cfg_option in self.buildroot.config: + data["config"][cfg_option] = self.buildroot.config[cfg_option] + + if "bootstrap_image" in data["config"]: + # Optional object, only if bootstrap image used (we still + # produce lockfiles even if these are useless for hermetic + # builds). + with self.buildroot.uid_manager.elevated_privileges(): + try: + podman = Podman(self.buildroot, + data["config"]["bootstrap_image"]) + data["bootstrap"] = podman.inspect_hermetic_metadata() + data["bootstrap"]["image_digest"] = podman.get_oci_digest() + except PodmanError: + data["bootstrap"] = {} + + with open(out_file, "w", encoding="utf-8") as fdlist: + fdlist.write(json.dumps(data, indent=4, sort_keys=True) + "\n") + finally: + self.state.finish(statename) diff --git a/mock/py/mockbuild/plugins/ccache.py b/mock/py/mockbuild/plugins/ccache.py new file mode 100644 index 0000000..9b79482 --- /dev/null +++ b/mock/py/mockbuild/plugins/ccache.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Michael Brown +# Copyright (C) 2007 Michael E Brown + +# python library imports + +# our imports +from mockbuild.mounts import BindMountPoint +from mockbuild.trace_decorator import getLog, traceLog +from mockbuild import file_util + +requires_api_version = "1.1" +run_in_bootstrap = False + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + CCache(plugins, conf, buildroot) + + +class CCache(object): + """enables ccache in buildroot/rpmbuild""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = buildroot.config + self.state = buildroot.state + self.ccache_opts = conf + tmpdict = self.ccache_opts.copy() + tmpdict.update({'chrootuid': self.buildroot.chrootuid}) + self.ccachePath = self.ccache_opts['dir'] % tmpdict + buildroot.preexisting_deps.append("ccache") + plugins.add_hook("prebuild", self._ccacheBuildHook) + plugins.add_hook("preinit", self._ccachePreInitHook) + plugins.add_hook("postbuild", self._ccachePostBuildHook) + buildroot.mounts.add( + BindMountPoint(srcpath=self.ccachePath, bindpath=buildroot.make_chroot_path("/var/tmp/ccache"))) + + # ============= + # 'Private' API + # ============= + # set the max size before we actually use it during a build. ccache itself + # manages size and settings. we also set a few variables used by ccache to + # find the shared cache. + @traceLog() + def _ccacheBuildHook(self): + self.buildroot.doChroot(["ccache", "-M", str(self.ccache_opts['max_cache_size'])], shell=False) + if not self.ccache_opts.get("show_stats"): + return + # zero ccache stats + getLog().info("Zero ccache stats:") + self.buildroot.doChroot(["ccache", "--zero-stats"], printOutput=True, shell=False) + + # set up the ccache dir. + # we also set a few variables used by ccache to find the shared cache. + @traceLog() + def _ccachePreInitHook(self): + getLog().info("enabled ccache") + envupd = {"CCACHE_DIR": "/var/tmp/ccache", "CCACHE_UMASK": "002"} + if self.ccache_opts.get('compress') is not None: + envupd["CCACHE_COMPRESS"] = str(self.ccache_opts['compress']) + if self.ccache_opts.get('hashdir'): + envupd["CCACHE_HASHDIR"] = "1" + else: + envupd["CCACHE_NOHASHDIR"] = "1" + if self.ccache_opts.get('debug'): + envupd["CCACHE_DEBUG"] = "1" + else: + envupd["CCACHE_NODEBUG"] = "1" + self.buildroot.env.update(envupd) + + file_util.mkdirIfAbsent(self.buildroot.make_chroot_path('/var/tmp/ccache')) + file_util.mkdirIfAbsent(self.ccachePath) + self.buildroot.uid_manager.changeOwner(self.ccachePath, recursive=True) + + # get some cache stats + def _ccachePostBuildHook(self): + """ show the cache hit stats """ + if not self.ccache_opts.get("show_stats"): + return + getLog().info("ccache stats:") + self.buildroot.doChroot(["ccache", "--show-stats"], printOutput=True, shell=False) diff --git a/mock/py/mockbuild/plugins/chroot_scan.py b/mock/py/mockbuild/plugins/chroot_scan.py new file mode 100644 index 0000000..9823d39 --- /dev/null +++ b/mock/py/mockbuild/plugins/chroot_scan.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Clark Williams +# Copyright (C) 2013 Clark Williams + +# python library imports +import os +import os.path +import re +import shutil + +# our imports +from mockbuild.trace_decorator import getLog, traceLog +from mockbuild import util, file_util + +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + ChrootScan(plugins, conf, buildroot) + + +class ChrootScan(object): + """scan chroot for files of interest, copying to resultdir with relative paths""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = buildroot.config + self.state = buildroot.state + self.scan_opts = conf + plugins.add_hook("postbuild", self._scanChroot) + plugins.add_hook("initfailed", self._scanChroot) + getLog().info("chroot_scan: initialized") + + @property + def resultdir(self): + """ + The plugin's self.resultdir is a subdir of buildroot.resultdir, which + is, e.g., for --chain, changed for every single package. + """ + return os.path.join(self.buildroot.resultdir, "chroot_scan") + + def _only_failed(self): + """ Returns boolean value if option 'only_failed' is set. """ + return str(self.scan_opts.get('only_failed')) == 'True' + + def _tarball(self): + """ Returns boolean value if option 'write_tar' is set. """ + return str(self.scan_opts.get('write_tar')) == 'True' + + @traceLog() + def _scanChroot(self): + is_failed = self.state.result != "success" + if (self._only_failed() and is_failed) or not self._only_failed(): + self.__scanChroot() + + def __scanChroot(self): + regexstr = "|".join(self.scan_opts['regexes']) + regex = re.compile(regexstr) + chroot = self.buildroot.make_chroot_path() + self.buildroot.create_resultdir() + # self.resultdir != self.buildroot.resultdir + file_util.mkdirIfAbsent(self.resultdir) + count = 0 + logger = getLog() + logger.debug("chroot_scan: Starting scan of %s", chroot) + copied = [] + for root, _, files in os.walk(chroot): + for f in files: + m = regex.search(f) + if m: + srcpath = os.path.join(root, f) + # we intentionally ignore errors here: + # https://github.com/rpm-software-management/mock/issues/1455 + util.do(["cp", "--preserve=mode", "--parents", srcpath, + self.resultdir], raiseExc=False) + count += 1 + copied.append(srcpath) + logger.debug("chroot_scan: finished with %d files found", count) + if count: + logger.info("chroot_scan: %d files copied to %s", count, self.resultdir) + logger.info("\n".join(copied)) + self.buildroot.uid_manager.changeOwner(self.resultdir, recursive=True) + # some packages installs 555 perms on dirs, + # so user can't delete/move chroot_scan's results + util.do(['chmod', '-R', 'u+w', self.resultdir]) + if self._tarball(): + tarfile = self.resultdir + ".tar.gz" + logger.info("chroot_scan: creating tarball %s", tarfile) + __tar_cmd = self.config["tar_binary"] + util.do( + [__tar_cmd, "-czf", tarfile, self.resultdir], + shell=False, printOutput=True + ) + shutil.rmtree(self.resultdir) + self.buildroot.uid_manager.changeOwner(tarfile) diff --git a/mock/py/mockbuild/plugins/compress_logs.py b/mock/py/mockbuild/plugins/compress_logs.py new file mode 100644 index 0000000..c5448f8 --- /dev/null +++ b/mock/py/mockbuild/plugins/compress_logs.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING + +import os.path + +from mockbuild import util +from mockbuild.trace_decorator import getLog, traceLog + +requires_api_version = "1.1" + + +class CompressLogsPlugin(object): + """Compress logs in resultdir.""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = buildroot.config + self.state = buildroot.state + self.conf = conf + self.command = self.conf['command'] + plugins.add_hook("process_logs", self._compress_logs) + getLog().info("compress_logs: initialized") + + @traceLog() + def _compress_logs(self): + logger = getLog() + for f_name in ('root.log', 'build.log', 'state.log', 'available_pkgs.log', + 'installed_pkgs.log', 'hw_info.log', 'procenv.log', + 'showrc.log'): + f_path = os.path.join(self.buildroot.resultdir, f_name) + if os.path.exists(f_path): + command = "{0} {1}".format(self.command, f_path) + logger.debug("Running %s", command) + util.do(command, shell=True) + + +def init(plugins, compress_conf, buildroot): + CompressLogsPlugin(plugins, compress_conf, buildroot) diff --git a/mock/py/mockbuild/plugins/export_buildroot_image.py b/mock/py/mockbuild/plugins/export_buildroot_image.py new file mode 100644 index 0000000..ea854b1 --- /dev/null +++ b/mock/py/mockbuild/plugins/export_buildroot_image.py @@ -0,0 +1,65 @@ +""" +Generate OCI from prepared build chroot. +Use given OCI image as build chroot (TODO). +""" + +import os +import mockbuild.util +from mockbuild.trace_decorator import getLog + +requires_api_version = "1.1" + + +def init(plugins, conf, buildroot): + """ The obligatory plugin entry point """ + OCIAsBuildroot(plugins, conf, buildroot) + + +class OCIAsBuildroot: + """ + OCIAsBuildroot plugin (class). + """ + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.state = buildroot.state + self.conf = conf + plugins.add_hook("postdeps", self.produce_buildroot_image) + + def do(self, cmd): + """ Execute command on host (as root) """ + getLog().info("Executing %s", ' '.join(cmd)) + mockbuild.util.do(cmd, returnOutput=True, returnStderr=True) + + def _produce_image_as_root(self): + name = f"mock-container-{self.buildroot.config['root']}" + tarball = os.path.join(self.buildroot.resultdir, "buildroot-oci.tar") + chroot = self.buildroot.make_chroot_path() + + # Add the whole chroot directory into the container + self.do(["buildah", "from", "--name", name, "scratch"]) + self.do(["buildah", "add", "--contextdir", chroot, + "--exclude", "sys", "--exclude", "proc", + name, "/", "/"]) + + # Keep just /builddir directory, make it correctly owned + self.do(["buildah", "run", name, "rm", "-r", + self.buildroot.config["chroothome"] + "/build"]) + self.do(["buildah", "run", name, "chown", "-R", "mockbuild:mock", + self.buildroot.config["chroothome"]]) + + # When starting container, switch to mockbuild user directly + self.do(["buildah", "config", "--user", "mockbuild:mock", name]) + + # Export the image as OCI archive, and remove the WIP container + self.do(["buildah", "commit", "--format", "oci", name, + "oci-archive:" + tarball]) + self.do(["buildah", "rm", name]) + + def produce_buildroot_image(self): + """ Generate OCI image from buildroot using Buildah """ + try: + self.state.start("producing buildroot as OCI image") + with self.buildroot.uid_manager.elevated_privileges(): + self._produce_image_as_root() + finally: + self.state.finish("producing buildroot as OCI image") diff --git a/mock/py/mockbuild/plugins/hw_info.py b/mock/py/mockbuild/plugins/hw_info.py new file mode 100644 index 0000000..1e84496 --- /dev/null +++ b/mock/py/mockbuild/plugins/hw_info.py @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING + +# python library imports +import codecs + +# our imports +from mockbuild.trace_decorator import getLog, traceLog +import mockbuild.util + +requires_api_version = "1.1" +run_in_bootstrap = False + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + HwInfo(plugins, conf, buildroot) + + +class HwInfo(object): + # pylint: disable=too-few-public-methods + """caches root environment in a tarball""" + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.hw_info_opts = conf + self.config = buildroot.config + plugins.add_hook("preinit", self._PreInitHook) + + # ============= + # 'Private' API + # ============= + @traceLog() + def _unprivPreInitHook(self): + getLog().info("enabled HW Info plugin") + out_file = self.buildroot.resultdir + '/hw_info.log' + out = codecs.open(out_file, 'w', 'utf-8', 'replace') + + cmd = ["/usr/bin/lscpu"] + output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False) + out.write("CPU info:\n") + out.write(output) + + cmd = ["/bin/free", "--human"] + output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False) + out.write("\n\nMemory:\n") + out.write(output) + + cmd = ["/bin/df", "-H", "-T", self.buildroot.make_chroot_path(), self.config["cache_topdir"]] + output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False) + out.write("\n\nStorage (chroot, cache_topdir):\n") + out.write(output) + + out.close() + + @traceLog() + def _PreInitHook(self): + with self.buildroot.uid_manager: + self._unprivPreInitHook() diff --git a/mock/py/mockbuild/plugins/lvm_root.py b/mock/py/mockbuild/plugins/lvm_root.py new file mode 100644 index 0000000..4743387 --- /dev/null +++ b/mock/py/mockbuild/plugins/lvm_root.py @@ -0,0 +1,405 @@ +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +import errno +import fcntl +import os +import re +from textwrap import dedent +import time + +from mockbuild import mounts, util +from mockbuild.exception import LvmError, LvmLocked + +requires_api_version = "1.1" + + +def lvm_do(*args, **kwargs): + env = os.environ.copy() + env['LC_ALL'] = 'C.UTF-8' + output = util.do(*args, returnOutput=True, env=env, + returnStderr=False, **kwargs) + return output + + +def current_mounts(): + with open("/proc/mounts") as proc_mounts: + mount_lines = proc_mounts.read().strip().split('\n') + for line in mount_lines: + src, target = [os.path.realpath(x) for x in line.split()[:2]] + yield src, target + + +class Lock(object): + def __init__(self, path, name, sleep_time): + lock_name = '.{0}.lock'.format(name) + lock_path = os.path.join(path, lock_name) + self.lock_file = open(lock_path, 'a+') + self.sleep_time = sleep_time + + def lock(self, exclusive, block=False): + lock_type = fcntl.LOCK_EX if exclusive else fcntl.LOCK_SH + try: + fcntl.lockf(self.lock_file.fileno(), + lock_type | (0 if block else fcntl.LOCK_NB)) + except IOError as e: + if e.errno in (errno.EACCES, errno.EAGAIN): + raise LvmLocked("LVM is locked") + raise + + def cond_lock(self, cond_fn, acquired_fn, wait_fn=None, unsatisfied_fn=None): + waiting = False + while cond_fn(): + try: + self.lock(exclusive=True) + except LvmLocked: + if not waiting and wait_fn: + wait_fn() + waiting = True + time.sleep(self.sleep_time) + else: + acquired_fn() + return + if unsatisfied_fn: + unsatisfied_fn() + + +class LvmPlugin(object): + postinit_name = 'postinit' + prefix = 'mock' + + def __init__(self, plugins, lvm_conf, buildroot): + self.buildroot = buildroot + self.lvm_conf = lvm_conf + self.vg_name = lvm_conf.get('volume_group') + self.conf_id = '{0}.{1}'.format(self.prefix, buildroot.shared_root_name) + self.pool_name = lvm_conf.get('pool_name', self.conf_id) + self.ext = self.buildroot.config.get('unique-ext', 'head') + self.head_lv = '+{0}.{1}'.format(self.conf_id, self.ext) + self.fs_type = lvm_conf.get('filesystem', 'ext4') + self.sleep_time = lvm_conf.get('sleep_time', 1) + self.root_path = os.path.realpath(self.buildroot.make_chroot_path()) + if not self.vg_name: + raise LvmError("Volume group must be specified") + + snapinfo_name = '.snapinfo-{0}.{1}'.format(self.conf_id, self.ext) + self.basepath = buildroot.mockdir + self.snap_info = os.path.join(self.basepath, snapinfo_name) + self.lock = self.create_lock('lvm') + self.pool_lock = self.create_lock('lvm-pool') + self.mount = None + + prefix = 'hook_' + for member in dir(self): + if member.startswith(prefix): + method = getattr(self, member) + hook_name = member[len(prefix):] + plugins.add_hook(hook_name, method) + + def create_lock(self, name): + return Lock(self.basepath, '{0}-{1}'.format(name, self.conf_id), self.sleep_time) + + def prefix_name(self, name=''): + return self.conf_id + '.' + name + + def remove_prefix(self, name): + return name.replace(self.prefix_name(''), '') + + def query_lvs(self, *options): + out = lvm_do(['lvs', '--noheadings', '--options', ','.join(options), + '--separator', '###', self.vg_name]) + return [line.strip().split('###') for line in out.split('\n') if line.strip()] + + def query_lv(self, lv_name, *options): + query = [entry[1:] for entry in self.query_lvs('lv_name', *options) + if entry[0] == lv_name] + if query: + return query[0] + return None + + def get_lv_path(self, lv_name=None): + name = lv_name or self.head_lv + entry = self.query_lv(name, 'lv_path') + if entry: + return entry[0] + return None + + def lv_exists(self, lv_name=None): + name = lv_name or self.head_lv + return bool(self.query_lv(name, 'lv_name')) + + def _lv_entry_is_our(self, entry): + if not entry: + return False + name, attrs, pool_lv = entry + return (attrs[0] == 'V' and + pool_lv == self.pool_name and + name.replace('+', '').startswith(self.prefix_name())) + + def lv_is_our(self, name): + entry = self.query_lv(name, 'lv_name', 'lv_attr', 'pool_lv') + return self._lv_entry_is_our(entry) + + def list_our_lvs(self, *options): + lvs = self.query_lvs('lv_name', 'lv_attr', 'pool_lv', *options) + # pylint: disable=inconsistent-return-statements + return [[entry[0]] + entry[3:] for entry in lvs + if self._lv_entry_is_our(entry[:3])] + + def get_current_snapshot(self): + if os.path.exists(self.snap_info): + with open(self.snap_info) as ac_record: + name = ac_record.read().rstrip() + if self.lv_is_our(name): + return name + postinit = self.prefix_name(self.postinit_name) + if self.lv_is_our(postinit): + # We don't have a registered snapshot, but postinit exists, so use it + self.set_current_snapshot(postinit) + return postinit + return None + + def set_current_snapshot(self, name): + if not self.lv_is_our(name): + raise LvmError("Snapshot {0} doesn't exist".format(self.remove_prefix(name))) + with open(self.snap_info, 'w') as ac_record: + ac_record.write(name) + + def unset_current_snapshot(self): + if os.path.exists(self.snap_info): + os.remove(self.snap_info) + + def make_snapshot(self, name): + if self.lv_exists(name): + raise LvmError( + "Snapshot {name} already exists".format(name=self.remove_prefix(name))) + lvcreate = ['lvcreate', '-s', self.vg_name + '/' + self.head_lv, '-n', name] + lvm_do(lvcreate) + self.set_current_snapshot(name) + + def delete_head(self): + self.umount() + if self.lv_exists(): + lvm_do(['lvremove', '-f', self.vg_name + '/' + self.head_lv]) + + def hook_make_snapshot(self, name): + lv_name = self.prefix_name(name) + self.make_snapshot(lv_name) + self.buildroot.root_log.info("created {name} snapshot".format(name=name)) + + def hook_postclean(self): + self.delete_head() + + def hook_rollback_to(self, name): + name = self.prefix_name(name) + self.set_current_snapshot(name) + self.delete_head() + + def create_pool(self): + def acquired(): + size = self.lvm_conf['size'] + pool_id = self.vg_name + '/' + self.pool_name + create_pool = ['lvcreate', '-T', pool_id, '-L', str(size)] + if 'poolmetadatasize' in self.lvm_conf: + create_pool += ['--poolmetadatasize', self.lvm_conf['poolmetadatasize']] + lvm_do(create_pool) + self.buildroot.root_log.info( + "created LVM cache thinpool of size {size}".format(size=size)) + + def cond(): + return not self.lv_exists(self.pool_name) + + self.pool_lock.cond_lock(cond, acquired) + self.pool_lock.lock(exclusive=False, block=True) + + def create_base(self): + pool_id = self.vg_name + '/' + self.pool_name + size = self.lvm_conf['size'] + lvm_do(['lvcreate', '-T', pool_id, '-V', str(size), '-n', self.head_lv]) + mkfs = self.lvm_conf.get('mkfs_command', 'mkfs.' + self.fs_type) + mkfs_args = self.lvm_conf.get('mkfs_args', []) + util.do([mkfs, self.get_lv_path()] + mkfs_args) + + def allocated_pool_data(self): + """ Returns percent of allocated space in thin pool """ + pool_id = self.vg_name + '/' + self.pool_name + output = lvm_do(['lvdisplay', pool_id]) + # pylint: disable=no-member + compiled_re = re.compile(r'.*Allocated pool data\s*([0-9.]*)%$.*', re.DOTALL | re.MULTILINE) + r_match = compiled_re.match(output) + if r_match: + return float(r_match.group(1)) + return None + + def allocated_pool_metadata(self): + """ Returns percent of allocated metadata in thin pool """ + pool_id = self.vg_name + '/' + self.pool_name + output = lvm_do(['lvdisplay', pool_id]) + compiled_re = re.compile(r'.*Allocated metadata\s*([0-9.]*)%$.*', re.DOTALL | re.MULTILINE) + r_match = compiled_re.match(output) + if r_match: + return float(r_match.group(1)) + return None + + def force_umount_root(self): + self.buildroot.root_log.warning("Forcibly unmounting root volume") + util.do(['umount', '-l', self.root_path], env=self.buildroot.env) + + def prepare_mount(self): + mount_options = self.lvm_conf.get('mount_options') + lv_path = self.get_lv_path() + if lv_path: + for src, target in current_mounts(): + if target == self.root_path: + if src != os.path.realpath(lv_path): + self.force_umount_root() + self.mount = mounts.FileSystemMountPoint(self.root_path, self.fs_type, + lv_path, options=mount_options) + + def umount(self): + if not self.mount: + self.prepare_mount() + if self.mount: + self.buildroot.mounts.umountall() + self.mount.umount() + + def create_head(self, snapshot_name): + lvm_do(['lvcreate', '-s', self.vg_name + '/' + snapshot_name, + '-n', self.head_lv, '--setactivationskip', 'n']) + self.buildroot.root_log.info( + "rolled back to {name} snapshot".format( + name=self.remove_prefix(snapshot_name))) + + def check_pool_size(self): + size_data = self.allocated_pool_data() + size_metadata = self.allocated_pool_metadata() + self.buildroot.root_log.info( + "LVM plugin enabled. Allocated pool data: {0}%. Allocated metadata: {1}%.".format(size_data, size_metadata)) + if ('check_size' not in self.lvm_conf or ('check_size' in self.lvm_conf and self.lvm_conf['check_size'])) and \ + ((size_metadata and size_metadata > 90) or (size_data and size_data > 90)): + raise LvmError("Thin pool {0}/{1} is over 90%. Please enlarge it.".format(self.vg_name, self.pool_name)) + if size_metadata and size_metadata > 75: + self.buildroot.root_log.warning( + "LVM Thin pool metadata are nearly filled up ({0}%). You may experience weird errors. " + "Consider growing up your thin pool.".format(size_metadata)) + if size_data and size_data > 75: + self.buildroot.root_log.warning( + "LVM Thin pool is nearly filled up ({0}%). You may experience weird errors. " + "Consider growing up your thin pool.".format(size_data)) + + def hook_mount_root(self): + def acquired(): + # Locked as exclusive so only one mock initializes postinit snapshot + if not self.lv_exists(self.pool_name): + self.create_pool() + self.create_base() + elif not self.get_current_snapshot(): + self.pool_lock.lock(exclusive=False, block=True) + if self.lv_exists(): + self.umount() + # We've got the exclusive lock but there's no postinit + # This means init failed and we need to start over + lvm_do(['lvremove', '-f', self.vg_name + '/' + self.head_lv]) + self.create_base() + else: + self.lock.lock(exclusive=False, block=True) + + def unsatisfied(): + self.lock.lock(exclusive=False, block=True) + + def cond(): + return not self.get_current_snapshot() + + def waiting(): + self.buildroot.root_log.info("Waiting for LVM init lock") + + self.lock.cond_lock(cond, acquired, unsatisfied_fn=unsatisfied, wait_fn=waiting) + self.pool_lock.lock(exclusive=False, block=True) + + if not self.lv_exists(): + self.create_head(self.get_current_snapshot()) + + self.prepare_mount() + self.check_pool_size() + self.mount.mount() + + def hook_umount_root(self): + self.umount() + + def hook_postumount(self): + if self.mount and self.lvm_conf.get('umount_root'): + self.mount.umount() + + def hook_postinit(self): + if not self.buildroot.chroot_was_initialized: + snapshot_name = self.prefix_name(self.postinit_name) + self.make_snapshot(snapshot_name) + self.buildroot.root_log.info( + "created {name} snapshot".format(name=self.postinit_name)) + # Relock as shared for following operations, noop if shared already + self.lock.lock(exclusive=False) + + def hook_scrub(self, what): + if what not in ('lvm', 'all') or not self.lv_exists(self.pool_name): + return + self.pool_lock.lock(exclusive=True) + lvs = self.list_our_lvs('lv_path') + for name, lv_path in lvs: + for src, _ in current_mounts(): + if src == os.path.realpath(lv_path): + util.do(['umount', '-l', src]) + self.buildroot.root_log.info("removing {0} volume".format(name)) + lvm_do(['lvremove', '-f', self.vg_name + '/' + name]) + remaining = [name for name, attr, pool_lv in self.query_lvs('lv_name', 'lv_attr', 'pool_lv') + if pool_lv == self.pool_name and attr[0] == 'V'] + if not remaining: + lvm_do(['lvremove', '-f', self.vg_name + '/' + self.pool_name]) + self.buildroot.root_log.info("deleted LVM cache thinpool") + self.unset_current_snapshot() + + def hook_list_snapshots(self): + current = self.get_current_snapshot() + lvs = self.list_our_lvs() + print('Snapshots for {0}:'.format(self.buildroot.shared_root_name)) + for [name] in lvs: + if name == current: + print('* ' + self.remove_prefix(name)) + elif not name.startswith('+'): + print(' ' + self.remove_prefix(name)) + + def hook_remove_snapshot(self, name): + if name == self.postinit_name: + raise LvmError(dedent("""\ + Won't remove postinit snapshot. To remove all logical + volumes associated with this buildroot, use --scrub lvm""")) + lv_name = self.prefix_name(name) + if not self.lv_is_our(lv_name): + raise LvmError("Snapshot {0} doesn't exist".format(name)) + self.umount() + if lv_name == self.get_current_snapshot(): + self.set_current_snapshot(self.prefix_name(self.postinit_name)) + lvm_do(['lvremove', '-f', self.vg_name + '/' + lv_name]) + self.buildroot.root_log.info("deleted {name} snapshot".format(name=name)) + + def update_snapshot_name(self, name): + # If the previous snapshot was already updated, remove + # the _updated_ postfix from its name. + name = re.sub(r"_updated_[\-\_0-9]{16}$", "", name) + + timestamp = time.localtime(time.time()) + name = name + "_updated_{y}-{mo:02d}-{d:02d}_{h:02d}-{m:02d}".format( + y=timestamp.tm_year, mo=timestamp.tm_mon, d=timestamp.tm_mday, + h=timestamp.tm_hour, m=timestamp.tm_min + ) + + return name + + def hook_postupdate(self): + name = self.update_snapshot_name(self.get_current_snapshot()) + self.make_snapshot(name) + self.buildroot.root_log.info("created updated snapshot {name}".format( + name=self.remove_prefix(name))) + + +def init(plugins, lvm_conf, buildroot): + LvmPlugin(plugins, lvm_conf, buildroot) diff --git a/mock/py/mockbuild/plugins/mount.py b/mock/py/mockbuild/plugins/mount.py new file mode 100644 index 0000000..91c29b3 --- /dev/null +++ b/mock/py/mockbuild/plugins/mount.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Daniel Mach +# Copyright (C) 2011 Daniel Mach + + +""" +# The mount plugin is enabled by default. +# To disable it, use following option: +config_opts['plugin_conf']['mount_enable'] = False + + +# To configure the mount plugin, for each mount point use following option: +config_opts['plugin_conf']['mount_opts']['dirs'].append( + ("/dev/device", "/mount/path/in/chroot/", "vfstype", "mount_options")) + +# A real life example: +config_opts['plugin_conf']['mount_opts']['dirs'].append( + ("server.example.com:/exports/data", "/mnt/data", "nfs", "rw,hard,intr,nosuid,nodev,noatime,tcp")) +""" + +from mockbuild.mounts import FileSystemMountPoint +from mockbuild.trace_decorator import traceLog +from mockbuild import file_util + +requires_api_version = "1.1" +# Skip mounting user-specified mounts if we're in the bootstrap chroot +run_in_bootstrap = False + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + Mount(plugins, conf, buildroot) + + +class Mount(object): + """mount dirs into chroot""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = buildroot.config + self.state = buildroot.state + self.opts = conf + plugins.add_hook("postinit", self._mountCreateDirs) + for device, dest_dir, vfstype, mount_opts in self.opts['dirs']: + buildroot.mounts.add_user_mount( + FileSystemMountPoint(buildroot.make_chroot_path(dest_dir), + filetype=vfstype, + device=device, + options=mount_opts)) + + @traceLog() + def _mountCreateDirs(self): + # pylint: disable=unused-variable + for device, dest_dir, vfstype, mount_opts in self.opts['dirs']: + file_util.mkdirIfAbsent(self.buildroot.make_chroot_path(dest_dir)) diff --git a/mock/py/mockbuild/plugins/overlayfs.py b/mock/py/mockbuild/plugins/overlayfs.py new file mode 100644 index 0000000..065cd47 --- /dev/null +++ b/mock/py/mockbuild/plugins/overlayfs.py @@ -0,0 +1,903 @@ +# -*- coding: utf-8 -*- +# This file is part of overlayfs plugin for mock +# Copyright (C) 2018 Zdeněk Žamberský ( https://github.com/zzambers ) +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + + +# About this plugin: +# +# This plugin implements snapshot functionality using overlayfs. From user +# perspective it works similar to LVM plugin, but unlike LVM plugin there is no +# need for LVM volume. It only needs (base) directory, where internal data are +# stored (layers etc., see lower). +# +# Configuration: +# config_opts['plugin_conf']['root_cache_enable'] = False +# config_opts['plugin_conf']['overlayfs_enable'] = True +# config_opts['plugin_conf']['overlayfs_opts']['base_dir'] = /some/directory +# config_opts['plugin_conf']['overlayfs_opts']['touch_rpmdb'] = False +# config_opts['plugin_conf']['overlayfs_opts']['trace_hooks'] = False +# +# ( Plugin uses postinit snapshot, similary to LVM, root chache is pointless. ) +# +# base_dir - directory where all plugin's data are stored. It includes data +# asociated with snapshots (layers, refs etc., see lower for details) +# It is further namespaced by configname so the same directory can be +# used in multiple mock configs without problems. +# touch_rpmdb - automatically "touch" rpmdb files after each mount to copy +# them to upper layer to overcome rpm/yum issue, +# when calling them directly in chroot. See: +# https://bugzilla.redhat.com/show_bug.cgi?id=1213602 +# pylint: disable=line-too-long +# https://docs.docker.com/storage/storagedriver/overlayfs-driver/#limitations-on-overlayfs-compatibility +# pylint: enable=line-too-long +# ( Option is not required when installing using mock --install, +# issue is work-arounded automatically there) +# defult: False +# trace_hooks - print info messages about plugin's hooks being called, +# default: False +# +# plugin's resources asociated with config can be released by: +# mock -r scrub all + + + +# Technical details: +# +# Overlayfs is special pseudo-filesystem (in kernel), which allows to place +# multiple directories as overlays on each other and combine them to single +# filesystem. It is done by supplying "lower" dir(s) and "upper" dir to +# the mount command. "Lower" dir(s) are read-only and all changes (writes) +# therefore go to the "upper" dir. Special files are used to mark deleted files. +# +# For more details about overlayfs see: +# https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt +# +# So overlayfs itself does not have notion of snapshots, but snapshots can be +# implemented using overlayfs. This is what this plugin does. +# +# Several levels of abstraction are used to achieve this. These are: +# +# LAYERS ( yes, LAYERS represent single layer of abstraction ... :) ) +# - layers represent individual layers (overlays) for overlayfs +# - layer consists of filesystem (actually directory supplied to overlayfs) +# and metadata. +# - layers are uniquely identified by their layerId, which is currently string +# with randomly generated UUID +# - Each layer has it's parent layer (with exception of "base" layer). +# Parent layer is layer, on top of which layer was created and +# which should be placed immediatly under it, when mounting it using +# overlayfs. ( used to determine list of layers which should be mounted, +# using overlayfs ) +# - One layer can be parent to multiple layers, but can only have single parent. +# - Layers have reference counter to track how many times are referenced +# (from other layers and REFs (see lower). When reference counter reaches +# zero, layer is deleted. +# - Layer can be marked immutable, which means no more changes should be +# done to it (to it's fs). That is, it is no longer allowed use it as "upper" +# layer. This is one way change. New layer needs to be created on top of +# immutable layer, if write access is needed. +# +# REFS +# - these are human readable names (aliases) for layers +# - when REF is created, reference counter of target layer is increased. +# Reference counter of target layer is decreased when REF is deleted. +# ( changing target actually means deleting REF and creating new one with +# the same name) +# - layer may be target of zero or more REFs +# - refs are used to implement SNAPSHOTS (see lower) +# - refs starting with '.' are reserved for internal use (special) +# - special refs are: +# .base - poits to "base" layer, which is direct or indirect parent of all +# other layers (it is only layer without parent). This layer is both +# empty (it's fs contains no files) and immutable. +# .current - points to "current" layer, which is layer representing "current" +# snapshot. It is also top-most layer from lower list when mount +# (using overlayfs) is performed. +# .upper - points to layer used as "upper", when mount is performed +# (overlayfs). This layer is where all changes (writes) happen. +# When .upper layer points to immutable layer ( e.g. after creating +# snapshot), new layer, which has current .upper layer as parent +# needs to be created and .upper REF updated to this new layer, +# prior to mount. +# +# SNAPSHOTS +# - used to implement snapshots in mock +# - snapshots are implemented using REFs +# - snapsot names map directly to refs +# - operations on snapshots also involve operations on special REFs (see higher) +# - when snapshot is made it's layer is made immutable +# +# HOOKS +# - methods actually called by mock +# - they mostly call SNAPSHOTS methods and other internal methods +# - additional locking is performed, to make sure, they are not concurently used +# in a way, which could lead to corruption of internal file structures. +# - I also tried to make these only methods, which contain mock specific code... +# +# CONCURRENCY / LOCKING STRATEGY +# - improper concurrent use of mock commands ( generally hooks calls ) could +# cause corruption of internal data structures. Therefore plugin does +# additional locking to prevent this corruption from happening. +# - locking should enforce following rules: +# 1. Snapshot operations are prevented when when other snapshot operation +# is currently in progress +# 2. Snapshot oprations are prevented, when buildroot is mounted, be it +# explicitly (mock --mount) or implicitly ( by mock --init, --shell, +# --chroot, --install etc.) +# implicit postinit snapshot is somewhat special case here +# ( unmount and mount of buildroot is actually done in postinit hook, +# relying on mock not to allow any other commands operationg +# on buildroot, when init operation is in progress ). +# 3. Explicit mount fails, if buildroot is currently implicitly mounted by +# mock command (mock --init, --shell, --chroot, --install etc.) and +# buildroot will be unmounted after that command finishes +# 4. When buildroot is explicitly mounted other mock operations are not +# permited until root is explicitly unmounted by mock --umount +# 5. Mount operations are prevented, when one is currently in progress +# - two locks are used to enforce rules listed higher: +# snapshot lock - prevents concurrent running of snapshot operations +# - also prevens snapshot operations when root is mounted and +# root to be mounted implicitly and explicitly at the same +# time ( because lock is acquired prior to mount and released +# after umount ) +# mount lock - prevents running multiple mount operations (and some other +# related operaions) concurrently. This is because mount +# operations are actually composed of several operations, +# so other mount operations must be prevented, +# when one is already in progress. + + +import os +import os.path +import shutil +import subprocess +import uuid +import re + +requires_api_version = "1.1" + +def init(plugins, conf, buildroot): + OverlayFsPlugin(plugins, conf, buildroot) + +class OverlayFsPlugin(object): + + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.configName = buildroot.shared_root_name + self.pluginBaseDir = conf.get('base_dir') + if not self.pluginBaseDir: + raise Exception("base_dir is not configured") + self.rootDir = buildroot.rootdir + if not self.rootDir: + raise Exception("Failed to get root dir") + self.traceHooks = conf.get('trace_hooks') + if not self.traceHooks: + self.traceHooks = False + self.touchRpmdbEnabled = conf.get('touch_rpmdb') + if not self.touchRpmdbEnabled: + self.touchRpmdbEnabled = False + # variables used to correctly handle explicit mounts + self.mountHookCalled = False + self.preinitHookCalled = False + self.failedMount = False + plugins.add_hook("make_snapshot", self.hook_make_snapshot) + plugins.add_hook("remove_snapshot", self.hook_remove_snapshot) + plugins.add_hook("rollback_to", self.hook_rollback_to) + plugins.add_hook("list_snapshots", self.hook_list_snapshots) + plugins.add_hook("mount_root", self.hook_mount_root) + plugins.add_hook("umount_root", self.hook_umount_root) + plugins.add_hook("postumount", self.hook_postumount) + plugins.add_hook("postinit", self.hook_postinit) + plugins.add_hook("postclean", self.hook_postclean) + plugins.add_hook("scrub", self.hook_scrub) + plugins.add_hook("preyum", self.hook_preyum) + plugins.add_hook("preinit", self.hook_preinit) + + ################ + # FILES # + ################ + + # directory where rootfs for current mock config should be mounted + def getRootDir(self): + return self.rootDir + + + # directory which contains all data asocied with this plugin + def getPluginBaseDir(self): + return self.pluginBaseDir + + # directory which contains all data asocied with this intance of plugin + # ( takes mock config name into account ) + def getPluginInstanceDir(self): + return os.path.join(self.getPluginBaseDir(), self.configName) + + + # directory where layers are stored + def getLayersDir(self): + return os.path.join(self.getPluginInstanceDir(), "layers") + + # directory with all data asocied with specific layer + def getLayerDir(self, layerId): + return os.path.join(self.getLayersDir(), layerId) + + # file which stores name of parent layer of layer with layerId + def getLayerParentFile(self, layerId): + return os.path.join(self.getLayerDir(layerId), "parent") + + # file which acts as referece couner of layer with layerId + def getLayerRefCounterFile(self, layerId): + return os.path.join(self.getLayerDir(layerId), "refcounter") + + # directory which contains actual filesystem (layer) mounted using overlayfs + def getLayerFsDir(self, layerId): + return os.path.join(self.getLayerDir(layerId), "fs") + + # file to mark layer should be treated as immutable (used for snapshots) + def getLayerImmutableFlagFile(self, layerId): + return os.path.join(self.getLayerDir(layerId), "immutable") + + + # directory which holds references (human redable names) for layers + def getRefsDir(self): + return os.path.join(self.getPluginInstanceDir(), "refs") + + # file which contains layerId of layer referenced by name + def getRefFile(self, name): + return os.path.join(self.getRefsDir(), name) + + + # directory with lock files + def getLocksDir(self): + return os.path.join(self.getPluginInstanceDir(), "locks") + + # lock file for snapshot locking + def getSnapshotLockFile(self): + return os.path.join(self.getLocksDir(), "snapshot.lock") + + # lock file for mount locking + def getMountLockFile(self): + return os.path.join(self.getLocksDir(), "mount.lock") + + + # directory used as workdir for overlayfs + def getWorkDir(self): + return os.path.join(self.getPluginInstanceDir(), "workdir") + + def rootMountFlagFile(self): + return os.path.join(self.getPluginInstanceDir(), ".root-mounted") + + + # file operations + + @staticmethod + def readFile(filename): + with open(filename) as fileObj: + value = fileObj.read() + return value + + @staticmethod + def writeFile(filename, value): + with open(filename, "w") as fileObj: + fileObj.write(value) + + + ################ + # LAYERS # + ################ + + # ref counter + + def getLayerRefcount(self, layerId): + layerCounterFile = self.getLayerRefCounterFile(layerId) + return int(self.readFile(layerCounterFile)) + + def setLayerRefCount(self, layerId, count): + layerCounterFile = self.getLayerRefCounterFile(layerId) + self.writeFile(layerCounterFile, str(count)) + + def refLayer(self, layerId): + counter = self.getLayerRefcount(layerId) + counter += 1 + self.setLayerRefCount(layerId, counter) + return counter + + def unrefLayer(self, layerId): + counter = self.getLayerRefcount(layerId) + if counter <= 0: + # should not happen + errMsg = "refcounter is already <= 0: {} !".format(layerId) + raise Exception(errMsg) + counter -= 1 + self.setLayerRefCount(layerId, counter) + return counter + + # layer operations + + def layerExists(self, layerId): + layerDir = self.getLayerDir(layerId) + return os.path.exists(layerDir) + + @staticmethod + def isSameLayer(layerId1, layerId2): + return layerId1 == layerId2 + + + def getParentLayer(self, layerId): + layerDir = self.getLayerDir(layerId) + parentFile = os.path.join(layerDir, "parent") + if os.path.exists(parentFile): + return self.readFile(parentFile) + return None + + def setParentLayer(self, layerId, parentLayerId): + parentFile = self.getLayerParentFile(layerId) + self.writeFile(parentFile, parentLayerId) + + def setLayerImmutable(self, layerId): + if not self.isLayerImmutable(layerId): + immutableFile = self.getLayerImmutableFlagFile(layerId) + self.writeFile(immutableFile, "") + + def isLayerImmutable(self, layerId): + immutableFile = self.getLayerImmutableFlagFile(layerId) + return os.path.exists(immutableFile) + + + def createLayer(self, parentLayerId): + newLayerId = str(uuid.uuid4()) + if self.layerExists(newLayerId): + # paranoia... :) + errMsg = "Layer already exists: {} !".format(newLayerId) + raise Exception(errMsg) + + # create directory for the new layer + newLayerDir = self.getLayerDir(newLayerId) + os.mkdir(newLayerDir) + + # crete reference counter for the new layer and set it to zero + layerCounterFile = self.getLayerRefCounterFile(newLayerId) + self.writeFile(layerCounterFile, str(0)) + + # create directory containg actual filesystem + newLayerFsDir = self.getLayerFsDir(newLayerId) + os.mkdir(newLayerFsDir) + + # all layers hase parent except for bottom most base layer + if not parentLayerId is None: + # create file with name of "parent" layer in the new layer + self.setParentLayer(newLayerId, parentLayerId) + # increase ref counter of parent layer + self.refLayer(parentLayerId) + return newLayerId + + + def unrefOrDeleteLayer(self, layerId): + if not self.layerExists(layerId): + errMsg = "Layer does not exist: {} !".format(layerId) + raise Exception(errMsg) + counter = self.unrefLayer(layerId) + if not counter > 0: + parentLayerId = self.getParentLayer(layerId) + layerDir = self.getLayerDir(layerId) + shutil.rmtree(layerDir) + self.unrefOrDeleteLayer(parentLayerId) + + + ############## + # REFS # + ############## + + # special refs + + @staticmethod + def getBaseLayerRef(): + return ".base" + + @staticmethod + def getCurrentLayerRef(): + return ".current" + + @staticmethod + def getUpperLayerRef(): + return ".upper" + + @staticmethod + def getPostinitLayerRef(): + return "postinit" + + # operations on refs + + def getLayerFromRef(self, name): + if not self.refExists(name): + errMsg = "Ref does not exist: {} !".format(name) + raise Exception(errMsg) + refFile = self.getRefFile(name) + return self.readFile(refFile) + + def createRef(self, name, layerId): + if self.refExists(name): + errMsg = "Ref already exists: {} !".format(name) + raise Exception(errMsg) + refFile = self.getRefFile(name) + self.writeFile(refFile, layerId) + self.refLayer(layerId) + + def deleteRef(self, name): + refFile = self.getRefFile(name) + layerId = self.getLayerFromRef(name) + os.remove(refFile) + self.unrefOrDeleteLayer(layerId) + + def refExists(self, name): + refFile = self.getRefFile(name) + return os.path.exists(refFile) + + def createLayerAndRef(self, name, parentLayerId): + if self.refExists(name): + errMsg = "Ref already exists: {} !".format(name) + raise Exception(errMsg) + newLayerId = self.createLayer(parentLayerId) + self.createRef(name, newLayerId) + return newLayerId + + # creates ref if necessary, changes ref if it already exists + def setLayerRef(self, name, layerId): + if self.refExists(name): + currentLayerId = self.getLayerFromRef(name) + if not self.isSameLayer(currentLayerId, layerId): + self.deleteRef(name) + self.createRef(name, layerId) + else: + self.createRef(name, layerId) + + def listRefs(self, includeSpecial): + refsDir = self.getRefsDir() + allRefsList = os.listdir(refsDir) + if not includeSpecial: + refsList = [] + for ref in allRefsList: + if not ref.startswith( '.' ): + refsList.append(ref) + else: + refsList = allRefsList + return refsList + + + ################### + # SNAPSHOTS # + ################### + + # snapshot operations + + def createSnapshot(self, snapshotName): + upperLayerId = self.getLayerFromRef(self.getUpperLayerRef()) + self.createRef(snapshotName, upperLayerId) + self.setLayerImmutable(upperLayerId) + currentLayerRef = self.getCurrentLayerRef() + self.setLayerRef(currentLayerRef, upperLayerId) + + def restoreSnapshot(self, snapshotName): + upperLayerRef = self.getUpperLayerRef() + snapshotLayerId = self.getLayerFromRef(snapshotName) + self.setLayerRef(upperLayerRef, snapshotLayerId) + currentLayerRef = self.getCurrentLayerRef() + self.setLayerRef(currentLayerRef, snapshotLayerId) + + def deleteSnapshot(self, snapshotName): + self.deleteRef(snapshotName) + + def listSnapshots(self): + return self.listRefs(False) + + @staticmethod + def checkSnapshotName(snapshotName): + snapshotNamePattern = "[A-Za-z0-9_-][A-Za-z0-9_.-]*" + if not re.match(snapshotNamePattern, snapshotName): + formatStr = "Invalid snapshot name: {}, needs to has form of: {} !" + errMsg = formatStr.format(snapshotName, snapshotNamePattern) + raise Exception(errMsg) + + + ####################### + # OTHER INTERNAL # + ####################### + + # create basic directory structure + def basicInit(self): + pluginBaseDir = self.getPluginBaseDir() + if not os.path.exists(pluginBaseDir): + os.mkdir(pluginBaseDir) + dataBaseDir = self.getPluginInstanceDir() + if not os.path.exists(dataBaseDir): + os.mkdir(dataBaseDir) + layersDir = self.getLayersDir() + if not os.path.exists(layersDir): + os.mkdir(layersDir) + refsDir = self.getRefsDir() + if not os.path.exists(refsDir): + os.mkdir(refsDir) + locksDir = self.getLocksDir() + if not os.path.exists(locksDir): + os.mkdir(locksDir) + + # init basic refs/layers setup (create special layers/refs) + def initLayers(self): + baseLayerRef = self.getBaseLayerRef() + if not self.refExists(baseLayerRef): + self.createLayerAndRef(baseLayerRef, None) + self.setLayerImmutable(self.getLayerFromRef(baseLayerRef)) + upperLayerRef = self.getUpperLayerRef() + if not self.refExists(upperLayerRef): + self.createRef(upperLayerRef, self.getLayerFromRef(baseLayerRef)) + currentLayerRef = self.getCurrentLayerRef() + if not self.refExists(currentLayerRef): + self.createRef(currentLayerRef, self.getLayerFromRef(baseLayerRef)) + + # this makes sure nothing is written to snapshot layers + # ( once snapshot is done it's layer becomes read only ) + def prepareLayersForMount(self): + upperLayerRef = self.getUpperLayerRef() + upperLayer = self.getLayerFromRef(upperLayerRef) + # if upperLayerRef points to layer, which is marked immutable + # we cannot use that layer as upper layer, we need to create new one + # which has current upperLayer as parent and set it as upperLayer + if self.isLayerImmutable(upperLayer): + newLayerId = self.createLayer(upperLayer) + self.deleteRef(upperLayerRef) + self.createRef(upperLayerRef, newLayerId) + + # create list of layer and all its parents + # (used when mounting it as overlayfs) + def createLayerList(self, layerId): + layerList = [] + self.createLayerList2(layerList,layerId) + return layerList + + def createLayerList2(self, layerList,layerId): + parentLayerId = self.getParentLayer(layerId) + layerList.append(layerId) + if parentLayerId is not None: + self.createLayerList2(layerList,parentLayerId) + + # mount root: upperLayer (+ its parents) using overlayfs + def mountRoot(self): + self.prepareLayersForMount() + + upperLayerRef = self.getUpperLayerRef() + upperLayerId = self.getLayerFromRef(upperLayerRef) + + lowerTopLayerId = self.getParentLayer(upperLayerId) + lowerList = self.createLayerList(lowerTopLayerId) + + workDir = self.getWorkDir() + if os.path.exists(workDir): + shutil.rmtree(workDir) + os.mkdir(workDir) + + # make sure kernel has required module loaded + modprobeCmds = ["modprobe", "overlay"] + subprocess.check_call(modprobeCmds) + + mountCmds = [] + mountCmds.append("mount") + mountCmds.append("-t") + mountCmds.append("overlay") + mountCmds.append("overlay") + + optionsArg="-olowerdir=" + + firstLower=True + for lowerId in lowerList: + if not firstLower: + optionsArg += ":" + firstLower = False + optionsArg += self.getLayerFsDir(lowerId) + + optionsArg += ",upperdir=" + self.getLayerFsDir(upperLayerId) + optionsArg += ",workdir=" + workDir + + mountCmds.append(optionsArg) + mountCmds.append(self.getRootDir()) + subprocess.check_call(mountCmds) + + self.recordRootMounted(True) + + + # unmount root + def unmountRoot(self): + if self.isRootMounted(): + umountCmds = [] + umountCmds.append("umount") + umountCmds.append(self.getRootDir()) + subprocess.check_call(umountCmds) + + self.recordRootMounted(False) + workDir = self.getWorkDir() + if os.path.exists(workDir): + shutil.rmtree(workDir) + + + def recordRootMounted(self, mounted): + rootMountFlagFile = self.rootMountFlagFile() + if mounted: + self.writeFile(rootMountFlagFile, "") + else: + os.remove(rootMountFlagFile) + + + def isRootMounted(self): + rootMountFlagFile = self.rootMountFlagFile() + isRootMounted = os.path.exists(rootMountFlagFile) + return isRootMounted + + # lock on snapshot operations ( used to prevent concurent modification of + # refs/layers by mock ) + def snapshotLock(self): + snapshotLockFile = self.getSnapshotLockFile() + try: + os.mkdir(snapshotLockFile) + except OSError: + raise Exception("Failed to obtain snapshot lock !") + + def snapshotUnlock(self): + snapshotLockFile = self.getSnapshotLockFile() + if os.path.exists(snapshotLockFile): + os.rmdir(snapshotLockFile) + + # lock on mount operations + def mountLock(self): + mountLockFile = self.getMountLockFile() + try: + os.mkdir(mountLockFile) + except OSError: + raise Exception("Failed to obtain mount lock !") + + def mountUnlock(self): + mountLockFile = self.getMountLockFile() + os.rmdir(mountLockFile) + + def traceHook(self, name): + if self.traceHooks: + debugMsg = "Overalyfs pluin: {}".format(name) + self.buildroot.root_log.info(debugMsg) + + # touch rpmdb files to make overlayfs copy them to upper layer to overcome + # yum/rpm problems, due to overlayfs limitations. For more details see + # documentation of touch_rpmdb option documentation on beginning + # of this file. + def touchRpmdb(self): + rpmDbDir = os.path.join(self.rootDir, "var", "lib", "rpm") + if os.path.exists(rpmDbDir): + rpmDbFileNames = os.listdir(rpmDbDir) + for rpmDbFileName in rpmDbFileNames: + rpmDbFile = os.path.join(rpmDbDir, rpmDbFileName) + with open(rpmDbFile, "ab") as _rpmDbFileObj: + pass + + # Methods needed to implement explicit mount support + # ( to decide if buildroot should be unmounted at the end ) + + def isMountFail(self): + # mount hook was called but failed + return self.mountHookCalled and self.failedMount + + def isExplicitMount(self): + if not self.mountHookCalled: + # hook was not called at all -> not an explicit mount + return False + if self.preinitHookCalled: + # if preinit hook was called, mount was implicit + return False + # othervise mount should be explicit one + return True + + ############### + # HOOKS # + ############### + + # These are methods ( hooks ) actually called by mock + + # snapshots + + def hook_make_snapshot(self, name): + self.traceHook("hook_make_snapshot") + self.checkSnapshotName(name) + self.basicInit() + self.snapshotLock() + try: + self.initLayers() + self.createSnapshot(name) + finally: + self.snapshotUnlock() + + def hook_remove_snapshot(self, name): + self.traceHook("hook_remove_snapshot") + self.checkSnapshotName(name) + self.basicInit() + self.snapshotLock() + try: + self.initLayers() + self.deleteSnapshot(name) + finally: + self.snapshotUnlock() + + def hook_rollback_to(self, name): + self.traceHook("hook_rollback_to") + self.checkSnapshotName(name) + self.basicInit() + self.snapshotLock() + try: + self.initLayers() + self.restoreSnapshot(name) + finally: + self.snapshotUnlock() + + def hook_list_snapshots(self): + self.traceHook("hook_list_snapshots") + self.basicInit() + self.snapshotLock() + try: + self.initLayers() + snapshots = self.listSnapshots() + currentRef = self.getCurrentLayerRef() + currentLayer = self.getLayerFromRef(currentRef) + for snapshot in snapshots: + snapshotLayer = self.getLayerFromRef(snapshot) + if self.isSameLayer(currentLayer, snapshotLayer): + print('* ' + snapshot) + else: + print(' ' + snapshot) + finally: + self.snapshotUnlock() + + # mounting + + def hook_mount_root(self): + self.traceHook("hook_mount_root") + # mount is considered fail until buildroot successfully mounted + self.failedMount = True + self.mountHookCalled = True + self.basicInit() + self.mountLock() + try: + # prevent snapshot operations (by mock) while root is mounted + self.snapshotLock() + self.initLayers() + self.mountRoot() + self.failedMount = False + if self.touchRpmdbEnabled: + self.touchRpmdb() + finally: + self.mountUnlock() + + def hook_umount_root(self): + self.traceHook("hook_umount_root") + pluginInstanceDir = self.getPluginInstanceDir() + # pluginInstance dir exists -> it does not follow scub + if os.path.exists(pluginInstanceDir): + self.basicInit() + self.mountLock() + try: + self.buildroot.mounts.umountall() + self.unmountRoot() + # again allow snapshot operations (by mock) after unmount + self.snapshotUnlock() + finally: + self.mountUnlock() + + def hook_postumount(self): + self.traceHook("hook_postumount") + pluginInstanceDir = self.getPluginInstanceDir() + # pluginInstance dir exists -> it does not follow scub + if os.path.exists(pluginInstanceDir): + self.basicInit() + self.mountLock() + try: + # Do not unmount buildroot if mount was attempted and failed, + # it is either as result of some error and buildroot should not + # be mounted or maybe was already explicitly mounted previously, + # in which case we do not want to unmount it + if self.isMountFail(): + return + # Do not umount buildroot on the end if mount was + # done explicitly + if self.isExplicitMount(): + return + self.buildroot.mounts.umountall() + self.unmountRoot() + # again allow snapshot operations (by mock) after unmount + self.snapshotUnlock() + finally: + self.mountUnlock() + + # mock init / clean / scrub + + # this one is tricky it is called with mounted fiesystems + # (root + managed mounts), but to do snapshot root cannot be mounted + def hook_postinit(self): + self.traceHook("hook_postinit") + self.basicInit() + self.mountLock() + try: + if self.isRootMounted(): + # we do not acquire snapshot lock here, because fact that root + # is mounted means it was already acquired by mount_root hook + + postinitSnapshotName = self.getPostinitLayerRef() + # if postinit snapshot was not created yet... + if not self.refExists(postinitSnapshotName): + # unmount everything, so we can do snapshot + self.buildroot.mounts.umountall() + self.unmountRoot() + # do snapshot + self.initLayers() + self.createSnapshot(postinitSnapshotName) + # mount everything again + self.mountRoot() + self.buildroot.mounts.mountall_managed() + if self.touchRpmdbEnabled: + self.touchRpmdb() + finally: + self.mountUnlock() + + def hook_postclean(self): + self.traceHook("hook_postclean") + pluginInstanceDir = self.getPluginInstanceDir() + # pluginInstance dir exists -> it does not follow scub + if os.path.exists(pluginInstanceDir): + self.basicInit() + self.snapshotLock() + try: + self.initLayers() + currentSnapshotName = self.getCurrentLayerRef() + self.restoreSnapshot(currentSnapshotName) + finally: + self.snapshotUnlock() + + def hook_scrub(self, what): + self.traceHook("hook_scrub") + self.basicInit() + self.snapshotLock() + try: + self.initLayers() + if what in ("all", "overlayfs"): + baseSnapshotName = self.getBaseLayerRef() + self.restoreSnapshot(baseSnapshotName) + postinitSnapshotName = self.getPostinitLayerRef() + if self.refExists(postinitSnapshotName): + self.deleteSnapshot(postinitSnapshotName) + for snapshot in self.listSnapshots(): + self.deleteSnapshot(snapshot) + pluginInstanceDir = self.getPluginInstanceDir() + shutil.rmtree(pluginInstanceDir) + finally: + self.snapshotUnlock() + + def hook_preyum(self): + self.traceHook("hook_preyum") + self.basicInit() + self.mountLock() + try: + if self.isRootMounted(): + self.touchRpmdb() + finally: + self.mountUnlock() + + def hook_preinit(self): + self.traceHook("hook_preinit") + # used as mechanism to detect implicit mount + self.preinitHookCalled = True diff --git a/mock/py/mockbuild/plugins/package_state.py b/mock/py/mockbuild/plugins/package_state.py new file mode 100644 index 0000000..23184a2 --- /dev/null +++ b/mock/py/mockbuild/plugins/package_state.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Seth Vidal +# Copyright (C) 2012 Red Hat, Inc + +# this plugin dumps out two lists of pkgs: +# A list of all available pkgs + repos + other data +# A list of all installed pkgs + repos + other data +# into the results dir +# two files - available_pkgs.log +# installed_pkgs.log + +# our imports +from mockbuild.trace_decorator import traceLog +import mockbuild.util + +# repoquery used +repoquery_avail_opts = \ + "--qf '%{name}-%{epoch}:%{version}-%{release}.%{arch} %{buildtime} %{size} %{pkgid} %{repoid}' '*'" + +# set up logging, module options +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + PackageState(plugins, conf, buildroot) + + +class PackageState(object): + """dumps out a list of packages available and in the chroot""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.state = buildroot.state + self.conf = conf + self.available_pkgs_enabled = self.conf['available_pkgs'] + self.installed_pkgs_enabled = self.conf['installed_pkgs'] + self.avail_done = False + self.inst_done = False + self.online = self.buildroot.config['online'] + plugins.add_hook("postyum", self._availablePostYumHook) + plugins.add_hook("postdeps", self._installedPreBuildHook) + + @traceLog() + def _availablePostYumHook(self): + if self.online and not self.avail_done and self.available_pkgs_enabled: + with self.buildroot.uid_manager: + self.state.start("Outputting list of available packages") + out_file = self.buildroot.resultdir + '/available_pkgs.log' + chrootpath = self.buildroot.make_chroot_path() + if self.buildroot.config['package_manager'] in ['dnf', 'microdnf']: + cmd = "/usr/bin/dnf --installroot={0} repoquery -c {0}/etc/dnf/dnf.conf {1} | sort > {2}".format( + chrootpath, repoquery_avail_opts, out_file) + else: + cmd = "/usr/bin/repoquery --installroot={0} -c {0}/etc/yum.conf {1} | sort > {2}".format( + chrootpath, repoquery_avail_opts, out_file) + mockbuild.util.do(cmd, shell=True, env=self.buildroot.env) + self.avail_done = True + self.state.finish("Outputting list of available packages") + + @traceLog() + def _installedPreBuildHook(self): + if self.inst_done or not self.installed_pkgs_enabled: + return + + out_file = self.buildroot.resultdir + '/installed_pkgs.log' + self.state.start("Outputting list of installed packages") + + try: + cmd = "rpm -qa --root '%s' --qf '%%{nevra} %%{buildtime} %%{size} %%{pkgid} installed\\n'" % ( + self.buildroot.make_chroot_path()) + with self.buildroot.uid_manager: + output, _ = self.buildroot.doOutChroot(cmd, returnOutput=1, shell=True) + with open(out_file, 'w') as out_fd: + out_fd.write(output) + finally: + self.inst_done = True + self.state.finish("Outputting list of installed packages") diff --git a/mock/py/mockbuild/plugins/pm_request.py b/mock/py/mockbuild/plugins/pm_request.py new file mode 100644 index 0000000..c3fca5b --- /dev/null +++ b/mock/py/mockbuild/plugins/pm_request.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python +# License: GPL2 or later see COPYING +# Written by Michael Simacek +# Copyright (C) 2015 Red Hat, Inc. + +import logging +import multiprocessing +import os +import shlex +import socket +import sys + +from io import StringIO + +from mockbuild import file_util +from mockbuild.exception import Error +from mockbuild.trace_decorator import traceLog + +requires_api_version = "1.1" + +RUNDIR = '/var/run/mock' +SOCKET_NAME = 'pm-request' +MAX_CONNECTIONS = 10 + + +@traceLog() +def init(plugins, conf, buildroot): + PMRequestPlugin(plugins, conf, buildroot) + + +class OutputFilter(object): + @staticmethod + def filter(record): + return record.levelno == logging.DEBUG + + +class PMRequestPlugin(object): + """ + Executes package manager commands requested by processes runninng in the + chroot. + """ + + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = conf + plugins.add_hook("earlyprebuild", self.start_listener) + plugins.add_hook("preshell", self.start_listener) + plugins.add_hook("postbuild", self.log_executed) + + @traceLog() + def start_listener(self): + process = multiprocessing.Process( + name="pm-request-listener", + target=lambda: PMRequestListener(self.config, self.buildroot).listen()) + process.daemon = True + self.buildroot.env['PM_REQUEST_SOCKET'] = os.path.join(RUNDIR, SOCKET_NAME) + self.buildroot.root_log.info("Enabled pm_request plugin") + process.start() + + @traceLog() + def log_executed(self): + """ Obtains the list of executed commands from the daemon process """ + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.connect(self.buildroot.make_chroot_path(RUNDIR, SOCKET_NAME)) + sock.sendall(b'!LOG_EXECUTED\n') + executed_commands = sock.makefile().read() + if executed_commands: + self.buildroot.root_log.warning( + "The pm_request plugin executed following commands:\n" + + executed_commands + + "\nThe build may not be reproducible.\n") + except socket.error: + pass + finally: + sock.close() + + +class PMRequestListener(object): + """ Daemon process that responds to requests """ + + def __init__(self, config, buildroot): + self.config = config + self.buildroot = buildroot + self.rundir = buildroot.make_chroot_path(RUNDIR) + self.socket_path = os.path.join(self.rundir, SOCKET_NAME) + self.executed_commands = [] + # util.do cannot return output when the command fails, we need to + # capture it's logging + self.log_buffer = StringIO() + self.log = logging.getLogger("mockbuild.plugin.pm_request") + self.log.level = logging.DEBUG + self.log.addFilter(OutputFilter()) + self.log.propagate = False + self.log.addHandler(logging.StreamHandler(self.log_buffer)) + + def prepare_socket(self): + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + sock.connect(self.socket_path) + except (socket.error, OSError): + try: + os.unlink(self.socket_path) + except OSError: + pass + else: + # there's another process listening + sys.exit(0) + + file_util.mkdirIfAbsent(self.rundir) + # Don't allow regular users to access the socket as they may not be in + # the mock group + os.chown(self.rundir, self.buildroot.chrootuid, self.buildroot.chrootgid) + os.chmod(self.rundir, 0o770) + sock.bind(self.socket_path) + os.chown(self.socket_path, self.buildroot.chrootuid, self.buildroot.chrootgid) + return sock + + def listen(self): + sock = self.prepare_socket() + sock.listen(MAX_CONNECTIONS) + while True: + try: + connection, _ = sock.accept() + try: + line = connection.makefile().readline() + command = shlex.split(line) + # pylint:disable=E1101 + if command == ["!LOG_EXECUTED"]: + connection.sendall('\n'.join(self.executed_commands).encode()) + elif command: + success, out = self.execute_command(command) + connection.sendall(b"ok\n" if success else b"nok\n") + connection.sendall(out.encode()) + if success: + self.executed_commands.append(line.strip()) + finally: + connection.close() + except socket.error: + continue + + def execute_command(self, command): + try: + self.buildroot.pkg_manager.execute( + *command, printOutput=False, logger=self.log, + returnOutput=False, pty=False, raiseExc=True) + success = True + except Error: + success = False + out = self.log_buffer.getvalue() + self.log_buffer.seek(0) + self.log_buffer.truncate() + return success, out diff --git a/mock/py/mockbuild/plugins/procenv.py b/mock/py/mockbuild/plugins/procenv.py new file mode 100644 index 0000000..4f85a72 --- /dev/null +++ b/mock/py/mockbuild/plugins/procenv.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING + +# python library imports +import codecs + +# our imports +from mockbuild.trace_decorator import getLog, traceLog +import mockbuild.util + +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + ProcEnv(plugins, conf, buildroot) + + +class ProcEnv(object): + # pylint: disable=too-few-public-methods + """Get the runtime process environment""" + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.procenv_opts = conf + self.config = buildroot.config + + # ensure procenv is installed into the buildroot + buildroot.preexisting_deps.append('procenv') + + # actually run our plugin at this step + plugins.add_hook("prebuild", self._PreBuildHook) + + # ============= + # 'Private' API + # ============= + @traceLog() + def _PreBuildHook(self): + getLog().info("enabled ProcEnv plugin") + + out_file = self.buildroot.resultdir + '/procenv.log' + with codecs.open(out_file, 'w', 'utf-8', 'replace') as out: + + cmd = ["/usr/bin/procenv"] + output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False) + out.write(output) + + self.buildroot.uid_manager.changeOwner(out_file, gid=self.config['chrootgid']) diff --git a/mock/py/mockbuild/plugins/root_cache.py b/mock/py/mockbuild/plugins/root_cache.py new file mode 100644 index 0000000..94031f1 --- /dev/null +++ b/mock/py/mockbuild/plugins/root_cache.py @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Michael Brown +# Copyright (C) 2007 Michael E Brown + +# python library imports +import fcntl +import os +import time + +# our imports +from mockbuild.trace_decorator import getLog, traceLog +import mockbuild.util +import mockbuild.text + +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + RootCache(plugins, conf, buildroot) + + +class RootCache(object): + """caches root environment in a tarball""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.root_cache_opts = conf + self.config = buildroot.config + self.state = buildroot.state + self.rootSharedCachePath = self.root_cache_opts['dir'] % self.root_cache_opts + self.rootCacheFile = os.path.join(self.rootSharedCachePath, "cache.tar") + self.rootCacheLock = None + self.compressProgram = self.root_cache_opts['compress_program'] + if self.compressProgram == 'pigz' and not os.path.exists('/bin/pigz'): + getLog().warning("specified 'pigz' as the root cache compress program but not available; using gzip") + self.compressProgram = 'gzip' + + self.decompressProgram = self.root_cache_opts.get('decompress_program') + if not self.decompressProgram: + if self.config['tar'] == 'bsdtar': + # Contrary to GNU tar, BSD tar doesn't automatically add the "-d" + # option to the compressing utility while decompressing. + self.decompressProgram = "{0} {1}".format(self.compressProgram, "-d") + else: + self.decompressProgram = self.compressProgram + + if self.compressProgram: + self.compressArgs = ['--use-compress-program', self.compressProgram] + self.rootCacheFile = self.rootCacheFile + self.root_cache_opts['extension'] + else: + self.compressArgs = [] + if self.decompressProgram: + self.decompressArgs = ['--use-compress-program', self.decompressProgram] + else: + self.decompressArgs = [] + plugins.add_hook("preinit", self._rootCachePreInitHook) + plugins.add_hook("preshell", self._rootCachePreShellHook) + plugins.add_hook("prechroot", self._rootCachePreShellHook) + plugins.add_hook("preyum", self._rootCachePreYumHook) + plugins.add_hook("postinit", self._rootCachePostInitHook) + plugins.add_hook("postshell", self._rootCachePostShellHook) + plugins.add_hook("postchroot", self._rootCachePostShellHook) + plugins.add_hook("postyum", self._rootCachePostShellHook) + plugins.add_hook("postupdate", self._rootCachePostUpdateHook) + self.exclude_dirs = self.root_cache_opts['exclude_dirs'] + self.exclude_tar_opts = [] + for ex_dir in self.exclude_dirs: + self._tarExcludeOption(ex_dir) + + def _tarExcludeOption(self, ex_dir): + if self.config['tar'] == 'bsdtar': + anchor = '^' + else: + anchor = '' + + self.exclude_tar_opts.append('--exclude=' + anchor + ex_dir) + + # ============= + # 'Private' API + # ============= + @traceLog() + def _rootCacheLock(self, shared=1): + lockType = fcntl.LOCK_EX + if shared: + lockType = fcntl.LOCK_SH + try: + fcntl.lockf(self.rootCacheLock.fileno(), lockType | fcntl.LOCK_NB) + except IOError: + self.state.start("Waiting for rootcache lock") + fcntl.lockf(self.rootCacheLock.fileno(), lockType) + self.state.finish("Waiting for rootcache lock") + + @traceLog() + def _rootCacheUnlock(self): + fcntl.lockf(self.rootCacheLock.fileno(), fcntl.LOCK_UN) + + @traceLog() + def _rootCachePreInitHook(self): + getLog().info("enabled root cache") + self._unpack_root_cache() + + def _haveVolatileRoot(self): + # pylint: disable=unneeded-not + return self.config['plugin_conf']['tmpfs_enable'] \ + and not (str(self.config['plugin_conf']['tmpfs_opts']['keep_mounted']) == 'True') + + @traceLog() + def _unpack_root_cache(self): + # check cache status + try: + if self.root_cache_opts['age_check']: + # see if it aged out + statinfo = os.stat(self.rootCacheFile) + file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 * 24) + if file_age_days > self.root_cache_opts['max_age_days']: + getLog().info("root cache aged out! cache will be rebuilt") + os.unlink(self.rootCacheFile) + else: + # make sure no config file is newer than the cache file + for cfg in self.config['config_paths']: + if os.stat(cfg).st_mtime > statinfo.st_mtime: + getLog().info("%s newer than root cache; cache will be rebuilt", cfg) + os.unlink(self.rootCacheFile) + break + else: + getLog().info("skipping root_cache aging check") + except OSError: + pass + + mockbuild.file_util.mkdirIfAbsent(self.rootSharedCachePath) + # lock so others dont accidentally use root cache while we operate on it. + if self.rootCacheLock is None: + self.rootCacheLock = open(os.path.join(self.rootSharedCachePath, "rootcache.lock"), "a+") + + # optimization: don't unpack root cache if chroot was not cleaned (unless we are using tmpfs) + if os.path.exists(self.rootCacheFile): + if (not self.buildroot.chroot_was_initialized or self._haveVolatileRoot()): + self.state.start("unpacking root cache") + self._rootCacheLock() + # deal with NFS homedir and root_squash + prev_cwd = None + cwd = mockbuild.util.pretty_getcwd() + if mockbuild.file_util.get_fs_type(cwd).startswith('nfs'): + prev_cwd = os.getcwd() + os.chdir(mockbuild.file_util.find_non_nfs_dir()) + mockbuild.file_util.mkdirIfAbsent(self.buildroot.make_chroot_path()) + + __tar_cmd = self.config["tar_binary"] + mockbuild.util.do( + [__tar_cmd] + self.decompressArgs + ["-xf", self.rootCacheFile, + "-C", self.buildroot.make_chroot_path()], + shell=False, printOutput=True + ) + for item in self.exclude_dirs: + mockbuild.file_util.mkdirIfAbsent(self.buildroot.make_chroot_path(item)) + + self._rootCacheUnlock() + self.buildroot.chrootWasCached = True + self.state.finish("unpacking root cache") + if prev_cwd: + os.chdir(prev_cwd) + + @traceLog() + def _rootCachePreShellHook(self): + if self._haveVolatileRoot(): + self._unpack_root_cache() + + @traceLog() + def _rootCachePreYumHook(self): + if self._haveVolatileRoot(): + if not os.listdir(self.buildroot.make_chroot_path()) or self.config['cache_alterations']: + self._unpack_root_cache() + + @traceLog() + def _root_cache_handle_mounts(self): + br_path = self.buildroot.make_chroot_path() + for m in self.buildroot.mounts.get_mountpoints(): + if m.startswith('/'): + if m.startswith(br_path): + self._tarExcludeOption('./' + m[len(br_path):]) + else: + self._tarExcludeOption('.' + m) + else: + self._tarExcludeOption('./' + m) + + @traceLog() + def _rootCachePostInitHook(self): + self._rebuild_root_cache() + + @traceLog() + def _rebuild_root_cache(self, after_update=False): + try: + self._rootCacheLock(shared=0) + # nuke any rpmdb tmp files + self.buildroot.nuke_rpm_db() + + # truncate the sparse files in /var/log + for logfile in ('/var/log/lastlog', '/var/log/faillog'): + try: + with open(self.buildroot.make_chroot_path(logfile), "w") as f: + f.truncate(0) + except (IOError, OSError): + pass + + # never rebuild cache unless it was a clean build, or we are explicitly caching alterations + if not self.buildroot.chroot_was_initialized or self.config['cache_alterations'] or after_update: + mockbuild.util.do(["sync"], shell=False) + self._root_cache_handle_mounts() + self.state.start("creating root cache") + __tar_cmd = [self.config["tar_binary"], "--one-file-system"] + if self.config['tar'] == 'gnutar': + __tar_cmd += ["--exclude-caches", "--exclude-caches-under"] + __tar_cmd += self.compressArgs + \ + ["-cf", self.rootCacheFile, + "-C", self.buildroot.make_chroot_path()] + \ + self.exclude_tar_opts+ ["."] + try: + mockbuild.util.do(__tar_cmd, shell=False) + except: + if os.path.exists(self.rootCacheFile): + os.remove(self.rootCacheFile) + raise + # now create the cache log file + with open(os.path.join(self.rootSharedCachePath, "cache.log"), "wb") as cache_log: + cache_log.write(self.buildroot.pkg_manager.init_install_output.encode(mockbuild.text.encoding)) + self.state.finish("creating root cache") + finally: + self._rootCacheUnlock() + + @traceLog() + def _rootCachePostShellHook(self): + if self._haveVolatileRoot() and self.config['cache_alterations']: + self._rebuild_root_cache() + + @traceLog() + def _rootCachePostUpdateHook(self): + self._rebuild_root_cache(after_update=True) diff --git a/mock/py/mockbuild/plugins/rpkg_preprocessor.py b/mock/py/mockbuild/plugins/rpkg_preprocessor.py new file mode 100644 index 0000000..ae7eff5 --- /dev/null +++ b/mock/py/mockbuild/plugins/rpkg_preprocessor.py @@ -0,0 +1,107 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by clime +# Copyright (C) 2020 clime + +# python library imports +import os +import os.path +import configparser +import shlex + +# our imports +from mockbuild.trace_decorator import getLog, traceLog +from mockbuild.exception import PkgError + +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + RpkgPreprocessor(plugins, conf, buildroot) + + +class RpkgPreprocessor(object): + """preprocess spec file by using rpkg utilities""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = buildroot.config + self.state = buildroot.state + self.opts = conf + self.log = getLog() + plugins.add_hook("pre_srpm_build", self._preprocess_proxy) + self.log.info("rpkg_preprocessor: initialized") + + @traceLog() + def _install_requires(self, requires): + try: + self.buildroot.uid_manager.becomeUser(0, 0) + self.buildroot.pkg_manager.install(*requires, check=True) + finally: + self.buildroot.uid_manager.restorePrivs() + + @traceLog() + def _preprocess_proxy(self, host_chroot_spec, host_chroot_sources): + if not host_chroot_sources or not os.path.isdir(host_chroot_sources): + self.log.debug("Sources not specified or not a directory. " + "Skipping rpkg preprocessing step.") + return + + self._preprocess(host_chroot_spec, host_chroot_sources) + + @traceLog() + def _preprocess(self, host_chroot_spec, host_chroot_sources): + rpkg_conf_path = os.path.join(host_chroot_sources, 'rpkg.conf') + force_enable = self.opts.get('force_enable', False) + + if not force_enable: + if not os.path.isfile(rpkg_conf_path): + self.log.info("rpkg.conf not found. " + "Skipping rpkg preprocessing step.") + return + + parser = configparser.ConfigParser( + interpolation=configparser.ExtendedInterpolation()) + + try: + parser.read(rpkg_conf_path) + except configparser.ParsingError as e: + raise PkgError("Parsing of %s failed with error: %s" % (rpkg_conf_path, repr(e))) + + try: + preprocess_spec = parser.getboolean('rpkg', 'preprocess_spec') + except (configparser.Error, ValueError): + self.log.warning( + "Could not get boolean value of rpkg.preprocess_spec option from rpkg.conf.") + preprocess_spec = False + + if not preprocess_spec: + self.log.info("preprocess_spec not enabled in rpkg.conf. " + "Skipping rpkg preprocessing step.") + return + + # try to locate spec file in SOURCES, which will be our input + host_chroot_sources_spec = os.path.join(host_chroot_sources, + os.path.basename(host_chroot_spec)) + + if not os.path.isfile(host_chroot_sources_spec): + raise PkgError("%s is not a file. Spec file needs to be among sources." % + host_chroot_sources_spec) + + self.log.info("Installing rpkg preprocessing requires...") + self._install_requires(self.opts.get('requires', [])) + + # get rid of host rootdir prefixes + rootdir_prefix = self.buildroot.make_chroot_path() + chroot_spec = host_chroot_spec.replace(rootdir_prefix, '') + chroot_sources = host_chroot_sources.replace(rootdir_prefix, '') + chroot_sources_spec = host_chroot_sources_spec.replace(rootdir_prefix, '') + + command_str = self.opts.get('cmd') % {'source_spec': chroot_sources_spec, + 'target_spec': chroot_spec} + command = shlex.split(command_str) + self.buildroot.doChrootPlugin(command, cwd=chroot_sources) diff --git a/mock/py/mockbuild/plugins/rpmautospec.py b/mock/py/mockbuild/plugins/rpmautospec.py new file mode 100644 index 0000000..fd501e9 --- /dev/null +++ b/mock/py/mockbuild/plugins/rpmautospec.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Copyright (C) 2023 Stephen Gallagher +# Copyright (C) 2023 Nils Philippsen +"""A mock plugin to pre-process spec files using rpmautospec.""" + +from pathlib import Path +from typing import Optional, Union + +from rpmautospec_core import specfile_uses_rpmautospec + +from mockbuild.exception import ConfigError, PkgError +from mockbuild.trace_decorator import getLog, traceLog + +requires_api_version = "1.1" + + +@traceLog() +def init(plugins, conf, buildroot): + """Register the rpmautospec plugin with mock.""" + RpmautospecPlugin(plugins, conf, buildroot) + + +class RpmautospecPlugin: + """Fill in release and changelog from git history using rpmautospec""" + + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = buildroot.config + self.opts = conf + self.log = getLog() + + if "cmd_base" not in self.opts: + raise ConfigError("The 'rpmautospec_opts.cmd_base' is unset") + + plugins.add_hook("pre_srpm_build", self.attempt_process_distgit) + self.log.info("rpmautospec: initialized") + + @traceLog() + def attempt_process_distgit( + self, + host_chroot_spec: Union[Path, str], + host_chroot_sources: Optional[Union[Path, str]], + ) -> None: + """Attempt to process a spec file with rpmautospec.""" + # Set up variables and check prerequisites. + if not host_chroot_sources: + self.log.debug("Sources not specified, skipping rpmautospec preprocessing.") + return + + host_chroot_spec = Path(host_chroot_spec) + host_chroot_sources = Path(host_chroot_sources) + if not host_chroot_sources.is_dir(): + self.log.debug( + "Sources not a directory, skipping rpmautospec preprocessing." + ) + return + + distgit_git_dir = host_chroot_sources / ".git" + if not distgit_git_dir.is_dir(): + self.log.debug( + "Sources is not a git repository, skipping rpmautospec preprocessing." + ) + return + + host_chroot_sources_spec = host_chroot_sources / host_chroot_spec.name + if not host_chroot_sources_spec.is_file(): + self.log.debug( + "Sources doesn’t contain spec file, skipping rpmautospec preprocessing." + ) + return + + with host_chroot_spec.open("rb") as spec, host_chroot_sources_spec.open( + "rb" + ) as sources_spec: + if spec.read() != sources_spec.read(): + self.log.warning( + "Spec file inside and outside sources are different, skipping rpmautospec" + " preprocessing." + ) + return + + if not specfile_uses_rpmautospec(host_chroot_sources_spec): + self.log.debug( + "Spec file doesn’t use rpmautospec, skipping rpmautospec preprocessing." + ) + return + + # Install the `rpmautospec` command line tool into the build root. + if self.opts.get("requires", None): + try: + self.buildroot.install_as_root(*self.opts["requires"]) + except Exception as exc: + raise PkgError( + "Can’t install rpmautospec dependencies into chroot: " + + ", ".join(self.opts["requires"]) + ) from exc + + # Get paths inside the chroot by chopping off the leading paths + chroot_dir = Path(self.buildroot.make_chroot_path()) + chroot_spec = Path("/") / host_chroot_spec.relative_to(chroot_dir) + chroot_sources = Path("/") / host_chroot_sources.relative_to(chroot_dir) + chroot_sources_spec = Path("/") / host_chroot_sources_spec.relative_to(chroot_dir) + + # Call subprocess to perform the specfile rewrite + command = list(self.opts["cmd_base"]) + command += [chroot_sources_spec] # + command += [chroot_spec] # + + # Run the rpmautospec tool in the chroot sandbox. This minimizes + # external dependencies in the host, e.g. the Koji build system. As a + # bonus, spec files will be processed in the environment they will be + # built for, reducing the impact of the host system on the outcome, + # leading to more deterministic results and better repeatable builds. + self.buildroot.doChrootPlugin(command, cwd=chroot_sources) diff --git a/mock/py/mockbuild/plugins/selinux.py b/mock/py/mockbuild/plugins/selinux.py new file mode 100644 index 0000000..f8029cd --- /dev/null +++ b/mock/py/mockbuild/plugins/selinux.py @@ -0,0 +1,108 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Jan Vcelak +# Copyright (C) 2010 Jan Vcelak + +# python library imports +import atexit +import os +import stat +import tempfile + +# our imports +from mockbuild.mounts import BindMountPoint, FileSystemMountPoint +from mockbuild.trace_decorator import getLog, traceLog +import mockbuild.util + +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + if mockbuild.util.selinuxEnabled(): + getLog().info("selinux enabled") + SELinux(plugins, conf, buildroot) + else: + getLog().info("selinux disabled") + + +class SELinux(object): + """On SELinux enabled box, this plugin will pretend, that SELinux is disabled in build environment. + + - fake /proc/filesystems is mounted into build environment, excluding selinuxfs + - fake /sys/fs/selinux directory mount point + - option '--setopt=tsflags=nocontext' is appended to each 'yum' command + """ + # pylint: disable=too-few-public-methods + + @traceLog() + def __init__(self, plugins, conf, buildroot): + self._originalUtilDo = mockbuild.util.do + + self.buildroot = buildroot + self.config = buildroot.config + self.state = buildroot.state + self.conf = conf + + self.filesystems = self._selinuxCreateFauxFilesystems() + self.chrootFilesystems = buildroot.make_chroot_path("/proc/filesystems") + + atexit.register(self._selinuxAtExit) + + self.buildroot.mounts.essential_mounts.append( + BindMountPoint(srcpath=self.filesystems, bindpath=self.chrootFilesystems) + ) + + self.buildroot.mounts.essential_mounts.append( + # essential mounts since we _always_ need to hide it + FileSystemMountPoint(filetype='tmpfs', + device='mock_hide_selinux_fs', + path=buildroot.make_chroot_path('/sys/fs/selinux')) + ) + + plugins.add_hook("preyum", self._selinuxPreYumHook) + plugins.add_hook("postyum", self._selinuxPostYumHook) + + @staticmethod + @traceLog() + def _selinuxCreateFauxFilesystems(): + (fd, path) = tempfile.mkstemp(prefix="mock-selinux-plugin.") + with os.fdopen(fd, 'w') as out: + with open("/proc/filesystems") as host: + for line in host: + if "selinuxfs" not in line: + out.write(line) + + os.chmod(path, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + + return path + + def _selinuxAtExit(self): + if os.path.exists(self.filesystems): + try: + os.unlink(self.filesystems) + except OSError as e: + getLog().warning("unable to delete selinux filesystems (%s): %s", self.filesystems, e) + + @traceLog() + def _selinuxPreYumHook(self): + mockbuild.util.do = self._selinuxDoYum + + @traceLog() + def _selinuxPostYumHook(self): + mockbuild.util.do = self._originalUtilDo + + @traceLog() + def _selinuxDoYum(self, command, *args, **kargs): + option = "--setopt=tsflags=nocontexts" + + if isinstance(command, list): + if command[0].startswith(self.buildroot.pkg_manager.command): + command.append(option) + elif isinstance(command, str): + if command.startswith(self.buildroot.pkg_manager.command): + command += " %s" % option + + return self._originalUtilDo(command, *args, **kargs) diff --git a/mock/py/mockbuild/plugins/showrc.py b/mock/py/mockbuild/plugins/showrc.py new file mode 100644 index 0000000..61bb659 --- /dev/null +++ b/mock/py/mockbuild/plugins/showrc.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING + +# python library imports +import codecs + +# our imports +from mockbuild.trace_decorator import getLog, traceLog +import mockbuild.util + +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + ShowRC(plugins, conf, buildroot) + + +class ShowRC(object): + # pylint: disable=too-few-public-methods + """Get the runtime rpm --showrc""" + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.showrc_opts = conf + self.config = buildroot.config + + # actually run our plugin at this step + plugins.add_hook("prebuild", self._PreBuildHook) + + # ============= + # 'Private' API + # ============= + @traceLog() + def _PreBuildHook(self): + getLog().info("enabled ShowRC plugin") + + out_file = self.buildroot.resultdir + '/showrc.log' + with codecs.open(out_file, 'w', 'utf-8', 'replace') as out: + + cmd = ["/usr/bin/rpm", "--showrc"] + output = mockbuild.util.do(cmd, shell=False, returnOutput=True, raiseExc=False) + out.write(output) + + self.buildroot.uid_manager.changeOwner(out_file, gid=self.config['chrootgid']) diff --git a/mock/py/mockbuild/plugins/sign.py b/mock/py/mockbuild/plugins/sign.py new file mode 100644 index 0000000..f2c4d8e --- /dev/null +++ b/mock/py/mockbuild/plugins/sign.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Julien BALLET +# Copyright (C) 2014 Facebook + +# python library imports +import os +import subprocess + +from mockbuild.trace_decorator import getLog, traceLog + +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + Sign(plugins, conf, buildroot) + + +class Sign(object): + """Automatically sign package after build""" + + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.plugins = plugins + self.conf = conf + self.buildroot = buildroot + self.plugins.add_hook('postbuild', self.sign_results) + getLog().info(conf) + getLog().info("enabled package signing") + + def sign_results(self): + if self.buildroot.final_rpm_list: + rpms = [os.path.join(self.buildroot.resultdir, rpm) for rpm in self.buildroot.final_rpm_list] + if rpms: + getLog().info("Signing %s", ', '.join(rpms)) + opts = self.conf['opts'] % {'rpms': ' '.join(rpms), 'resultdir': self.buildroot.resultdir} + cmd = "{0} {1}".format(self.conf['cmd'], opts) + getLog().info("Executing %s", cmd) + with self.buildroot.uid_manager: + subprocess.check_call(cmd, shell=True, env=os.environ) diff --git a/mock/py/mockbuild/plugins/tmpfs.py b/mock/py/mockbuild/plugins/tmpfs.py new file mode 100644 index 0000000..01a8110 --- /dev/null +++ b/mock/py/mockbuild/plugins/tmpfs.py @@ -0,0 +1,100 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Michael Brown +# Copyright (C) 2007 Michael E Brown + +# python library imports +import os + +# our imports +from mockbuild.trace_decorator import getLog, traceLog +import mockbuild.util + +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + system_ram_bytes = os.sysconf(os.sysconf_names['SC_PAGE_SIZE']) * os.sysconf(os.sysconf_names['SC_PHYS_PAGES']) + system_ram_mb = system_ram_bytes / (1024 * 1024) + if system_ram_mb > conf['required_ram_mb']: + Tmpfs(plugins, conf, buildroot) + else: + getLog().warning( + "Tmpfs plugin disabled. " + "System does not have the required amount of RAM to enable the tmpfs plugin. " + "System has %sMB RAM, but the config specifies the minimum required is %sMB RAM. ", + system_ram_mb, conf['required_ram_mb']) + + +class Tmpfs(object): + """Mounts a tmpfs on the chroot dir""" + # pylint: disable=too-few-public-methods + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.main_config = buildroot.config + self.state = buildroot.state + self.conf = conf + self.maxSize = self.conf['max_fs_size'] + self.mode = self.conf['mode'] + self.optArgs = ['-o', 'mode=%s' % self.mode] + self.optArgs += ['-o', 'nr_inodes=0'] + if self.maxSize: + self.optArgs += ['-o', 'size=' + self.maxSize] + plugins.add_hook("mount_root", self._tmpfsMount) + plugins.add_hook("postumount", self._tmpfsPostUmount) + plugins.add_hook("umount_root", self._tmpfsUmount) + if not os.path.ismount(self.buildroot.make_chroot_path()): + self.mounted = False + else: + self.mounted = True + getLog().info("tmpfs initialized") + + @traceLog() + def _tmpfsMount(self): + getLog().info("mounting tmpfs at %s.", self.buildroot.make_chroot_path()) + + if not self.mounted: + mountCmd = ["mount", "-n", "-t", "tmpfs"] + self.optArgs + \ + ["mock_chroot_tmpfs", self.buildroot.make_chroot_path()] + mockbuild.util.do(mountCmd, shell=False) + else: + getLog().info("reusing tmpfs at %s.", self.buildroot.make_chroot_path()) + self.mounted = True + + @traceLog() + def _tmpfsPostUmount(self): + if "keep_mounted" in self.conf and self.conf["keep_mounted"]: + self.mounted = False + else: + self._tmpfsUmount() + + @traceLog() + def _tmpfsUmount(self): + if not self.mounted: + return + force = False + getLog().info("unmounting tmpfs.") + umountCmd = ["umount", "-n", self.buildroot.make_chroot_path()] + # since we're in a separate namespace, the mount will be cleaned up + # on exit, so just warn if it fails here + try: + mockbuild.util.do(umountCmd, shell=False) + # pylint: disable=bare-except + except: + getLog().warning("tmpfs-plugin: exception while umounting tmpfs! (cwd: %s)", mockbuild.util.pretty_getcwd()) + force = True + + if force: + # try umounting with force option + umountCmd = ["umount", "-R", "-n", "-f", self.buildroot.make_chroot_path()] + try: + mockbuild.util.do(umountCmd, shell=False) + # pylint: disable=bare-except + except: + getLog().warning( + "tmpfs-plugin: exception while force umounting tmpfs! (cwd: %s)", mockbuild.util.pretty_getcwd()) + self.mounted = False diff --git a/mock/py/mockbuild/plugins/yum_cache.py b/mock/py/mockbuild/plugins/yum_cache.py new file mode 100644 index 0000000..c732b43 --- /dev/null +++ b/mock/py/mockbuild/plugins/yum_cache.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Michael Brown +# Copyright (C) 2007 Michael E Brown + +# python library imports +import fcntl +import glob +import os +import time + +# our imports +from mockbuild.mounts import BindMountPoint +from mockbuild.trace_decorator import getLog, traceLog +import mockbuild.util + +# set up logging, module options +requires_api_version = "1.1" + + +# plugin entry point +@traceLog() +def init(plugins, conf, buildroot): + YumCache(plugins, conf, buildroot) + + +class CacheDir: + def __init__(self, buildroot, pkg_manager): + self.buildroot = buildroot + self.cache_path = os.path.join('/var/cache', pkg_manager) + self.host_cache_path = os.path.join(self.buildroot.cachedir, + pkg_manager + '_cache') + self.mount_path = self.buildroot.make_chroot_path(self.cache_path) + self.buildroot.mounts.add(BindMountPoint( + srcpath=self.host_cache_path, + bindpath=self.mount_path, + )) + + mockbuild.file_util.mkdirIfAbsent(self.host_cache_path) + + +class YumCache(object): + """ + Pre-mount /var/cache/yum and /var/cache/dnf machine to chroot, because + yum/dnf stores the caches below --installroot directory, which is cleaned + up with --clean or --scrub=chroot. + """ + # pylint: disable=too-few-public-methods + + METADATA_EXTS = (".sqlite", ".xml", ".bz2", ".gz", ".xz", ".solv", ".solvx") + + @traceLog() + def __init__(self, plugins, conf, buildroot): + self.buildroot = buildroot + self.config = buildroot.config + self.state = buildroot.state + self.yum_cache_opts = conf + self.cache_dirs = [ + CacheDir(buildroot, 'yum'), + CacheDir(buildroot, 'dnf'), + ] + self.yumSharedCachePath = self.cache_dirs[0].host_cache_path + self.online = self.config['online'] + plugins.add_hook("preyum", self._yumCachePreYumHook) + plugins.add_hook("postyum", self._yumCachePostYumHook) + plugins.add_hook("preinit", self._yumCachePreInitHook) + + self.yumCacheLock = open(os.path.join(buildroot.cachedir, "yumcache.lock"), "a+") + + + # ============= + # 'Private' API + # ============= + # lock the shared yum cache (when enabled) before any access + # by yum, and prior to cleaning it. This prevents simultaneous access from + # screwing things up. This can possibly happen, eg. when running multiple + # mock instances with --uniqueext= + @traceLog() + def _yumCachePreYumHook(self): + try: + fcntl.lockf(self.yumCacheLock.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except IOError: + self.state.start("Waiting for yumcache lock") + fcntl.lockf(self.yumCacheLock.fileno(), fcntl.LOCK_EX) + self.state.finish("Waiting for yumcache lock") + + @traceLog() + def _yumCachePostYumHook(self): + fcntl.lockf(self.yumCacheLock.fileno(), fcntl.LOCK_UN) + + @traceLog() + def _prune_repo_data(self, directory): + for (dirpath, _, filenames) in os.walk(directory): + for filename in filenames: + fullPath = os.path.join(dirpath, filename) + statinfo = os.stat(fullPath) + file_age_days = (time.time() - statinfo.st_ctime) / (60 * 60 * 24) + # prune repodata so yum redownloads. + # prevents certain errors where yum gets stuck due to bad metadata + for ext in self.METADATA_EXTS: + if filename.endswith(ext) and file_age_days > self.yum_cache_opts['max_metadata_age_days']: + os.unlink(fullPath) + fullPath = None + break + + if fullPath is None: + continue + if file_age_days > self.yum_cache_opts['max_age_days']: + os.unlink(fullPath) + continue + + + @traceLog() + def _yumCachePreInitHook(self): + getLog().info("enabled package manager cache") + + for cdir in self.cache_dirs: + mockbuild.file_util.mkdirIfAbsent(cdir.host_cache_path) + + # lock so others dont accidentally use yum cache while we operate on it. + self._yumCachePreYumHook() + + if self.online: + state = "cleaning package manager metadata" + self.state.start(state) + for cdir in self.cache_dirs: + self._prune_repo_data(cdir.host_cache_path) + self.state.finish(state) + + # yum made an rpmdb cache dir in $cachedir/installed for a while; + # things can go wrong in a specific mock case if this happened. + # So - just nuke the dir and all that's in it. + if os.path.exists(self.yumSharedCachePath + '/installed'): + for fn in glob.glob(self.yumSharedCachePath + '/installed/*'): + os.unlink(fn) + os.rmdir(self.yumSharedCachePath + '/installed') + + self._yumCachePostYumHook() diff --git a/mock/py/mockbuild/podman.py b/mock/py/mockbuild/podman.py new file mode 100644 index 0000000..3ef8007 --- /dev/null +++ b/mock/py/mockbuild/podman.py @@ -0,0 +1,264 @@ +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +import hashlib +import json +import os +import logging +import subprocess +from contextlib import contextmanager + +import backoff +from mockbuild.trace_decorator import getLog, traceLog + + +class PodmanError(Exception): + """ + Exception raised by mockbuild.podman.Podman + """ + + +def podman_get_oci_digest(image, logger=None, podman_binary=None): + """ + Get sha256 digest of RootFS layers. This must be identical for + all images containing same order of layers, thus it can be used + as the check that we've loaded same image. + """ + logger = logger or logging.getLogger() + podman = podman_binary or "/usr/bin/podman" + logger.info("Calculating %s image OCI digest", image) + check = [podman, "image", "inspect", image] + result = subprocess.run(check, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False, + encoding="utf8") + if result.returncode: + logger.error("Can't get %s podman image digest: %s", image, result.stderr) + return None + result = result.stdout.strip() + + try: + data = json.loads(result)[0] + except json.JSONDecodeError: + logger.error("The manifest data of %s are not json-formatted.", image) + return None + + if 'RootFS' not in data: + logger.error("RootFS section of %s is missing.", image) + return None + if data['RootFS']['Type'] != 'layers': + logger.error("Unexpected format for RootFS in %s.", image) + return None + + # data which should be sufficient to confirm the image + data = { + 'RootFS': data['RootFS'], + 'Config': data['Config'], + } + # convert to json string with ordered dicts and create hash + data = json.dumps(data, sort_keys=True) + return hashlib.sha256(data.encode()).hexdigest() + + +def podman_check_native_image_architecture(image, logger=None, podman_binary=None): + """ + Return True if image's architecture is "native" for this host. + Relates: + https://github.com/containers/podman/issues/19717 + https://github.com/fedora-copr/copr/issues/2875 + """ + + logger = logger or logging.getLogger() + podman = podman_binary or "/usr/bin/podman" + logger.info("Checking that %s image matches host's architecture", image) + sys_check_cmd = [podman, "version", "--format", "{{.OsArch}}"] + image_check_cmd = [podman, "image", "inspect", + "--format", "{{.Os}}/{{.Architecture}}", image] + + def _podman_query(cmd): + return subprocess.check_output(cmd, encoding="utf8").strip() + + try: + system_arch = _podman_query(sys_check_cmd) + image_arch = _podman_query(image_check_cmd) + if system_arch != image_arch: + logger.error("Image architecture %s doesn't match system arch %s", + image_arch, system_arch) + return False + except subprocess.SubprocessError as exc: + logger.error("Subprocess failed: %s", exc) + return False + + return True + + +def pull_fail_handler(details): + """ + Raise an error when image pull fails, because lambdas can't raise. + """ + raise PodmanError("Image pull failed") + + +class Podman: + """ interacts with podman to create build chroot """ + + @traceLog() + def __init__(self, buildroot, image): + self.podman_binary = "/usr/bin/podman" + if not os.path.exists(self.podman_binary): + raise PodmanError(f"'{self.podman_binary}' not installed") + + self.buildroot = buildroot + self.image = image + self.image_id = None + getLog().info("Using container image: %s", image) + + @traceLog() + def pull_image(self): + """ pull the latest image, return True if successful """ + logger = getLog() + logger.info("Pulling image: %s", self.image) + cmd = [self.podman_binary, "pull", self.image] + + res = subprocess.run(cmd, env=self.buildroot.env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=False) + if res.returncode != 0: + logger.error("%s\n%s", res.stdout, res.stderr) + return False + + # Record the image id for later use. This is needed for the + # oci-image:tarball images that not necessarily have tags/names. + self.image_id = res.stdout.decode("utf-8").strip() + return True + + @property + def _tagged_id(self): + uuid = self.buildroot.config["mock_run_uuid"] + bootstrap = "-bootstrap" if self.buildroot.is_bootstrap else "" + return f"mock{bootstrap}-{uuid}" + + def tag_image(self): + """ + Tag the pulled image as mock-{uuid}, or mock-bootstrap-{uuid}. + """ + if not self.image_id: + raise PodmanError("No image to tag. Image pull failed or was not attempted") + cmd = ["podman", "tag", self.image_id, self._tagged_id] + getLog().info("Tagging container image as %s", self._tagged_id) + subprocess.run(cmd, env=self.buildroot.env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=True) + + def retry_image_pull(self, max_time): + """ Try pulling the image multiple times """ + @backoff.on_predicate(backoff.expo, lambda x: not x, + max_time=max_time, jitter=backoff.full_jitter, + on_giveup=pull_fail_handler) + def _keep_trying(): + return self.pull_image() + _keep_trying() + + @contextmanager + def mounted_image(self): + """ + Using the "podman image mount" command, mount the image as a temporary + read-only directory so we can copy-paste the contents into the final + chroot directory. + """ + logger = getLog() + cmd_mount = [self.podman_binary, "image", "mount", self.image_id] + cmd_umount = [self.podman_binary, "image", "umount", self.image_id] + result = subprocess.run(cmd_mount, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=False, + encoding="utf8") + if result.returncode: + message = "Podman mount failed: " + result.stderr + raise PodmanError(message) + + mountpoint = result.stdout.strip() + logger.info("mounting %s with podman image mount", self.image_id) + try: + logger.info("image %s as %s", self.image_id, mountpoint) + yield mountpoint + finally: + logger.info("umounting image %s (%s) with podman image umount", + self.image_id, mountpoint) + subprocess.run(cmd_umount, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=True) + + def get_oci_digest(self): + """ + Get sha256 digest of RootFS layers. This must be identical for + all images containing same order of layers, thus it can be used + as the check that we've loaded same image. + """ + the_image = self.image + if the_image.startswith("oci-archive:"): + # We can't query digest from tarball directly, but note + # the image needs to be tagged first! + the_image = self._tagged_id + digest = podman_get_oci_digest(the_image, logger=getLog()) + if digest is None: + raise PodmanError(f"Getting OCI digest for image {self.image} failed") + return digest + + def check_native_image_architecture(self): + """ + Check that self.image has been generated for the current + host's architecture. + """ + return podman_check_native_image_architecture(self.image_id, getLog()) + + @traceLog() + def cp(self, destination, tar_cmd): + """ copy content of container to destination directory """ + getLog().info("Copy content of container %s to %s", self.image_id, destination) + + with self.mounted_image() as mount_path: + # pipe-out the temporary mountpoint with the help of tar utility + cmd_podman = [tar_cmd, "-C", mount_path, "-c", "."] + with subprocess.Popen(cmd_podman, stdout=subprocess.PIPE) as podman: + # read the tarball from stdin, and extract to the destination + # directory (chroot directory) + cmd_tar = [tar_cmd, "-xC", destination, "-f", "-"] + with subprocess.Popen(cmd_tar, stdin=podman.stdout) as tar: + tar.communicate() + podman.communicate() + + def untag(self): + """ + Remove the additional image ID we created - which means the image itself + is garbage-collected if there's no other tag. + """ + cmd = ["podman", "rmi", self._tagged_id] + getLog().info("Removing image %s", self._tagged_id) + subprocess.run(cmd, env=self.buildroot.env, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, check=True) + + def read_image_id(self): + """ + Given self.image (name), get the image Id. + """ + cmd = ["podman", "image", "inspect", self.image, "--format", + "{{ .Id }}"] + getLog().info("Reading image .ID from %s", self.image) + res = subprocess.run(cmd, env=self.buildroot.env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=True) + self.image_id = res.stdout.decode("utf-8").strip() + + + def inspect_hermetic_metadata(self): + """ + Get the image metadata needed for the subsequent hermetic build. + """ + get_query = '{"pull_digest": "{{ .Digest }}", "id": "{{.Id}}", "architecture": "{{ .Architecture }}"}' + getLog().info("Reading image %s from %s", get_query, self.image) + cmd = ["podman", "image", "inspect", "--format", get_query, self.image] + res = subprocess.run(cmd, env=self.buildroot.env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + check=True) + return json.loads(res.stdout.decode("utf-8").strip()) + + + def __repr__(self): + return "Podman({}({}))".format(self.image, self.image_id) diff --git a/mock/py/mockbuild/rebuild.py b/mock/py/mockbuild/rebuild.py new file mode 100644 index 0000000..ba46658 --- /dev/null +++ b/mock/py/mockbuild/rebuild.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING + +import logging +import os.path +import sys +import time +from . import util +from .exception import BadCmdline +from .trace_decorator import traceLog + +log = logging.getLogger() + + +@traceLog() +def rebuild_generic(items, commands, buildroot, config_opts, cmd, post=None, clean=True): + start = time.time() + try: + for item in items: + log.info("Start(%s) Config(%s)", item, buildroot.shared_root_name) + if clean: + commands.clean() + commands.init(prebuild=not config_opts.get('short_circuit')) + ret = cmd(item) + elapsed = time.time() - start + log.info("Done(%s) Config(%s) %d minutes %d seconds", + item, config_opts['chroot_name'], elapsed // 60, elapsed % 60) + log.info("Results and/or logs in: %s", buildroot.resultdir) + commands.plugins.call_hooks("process_logs") + + if config_opts["cleanup_on_success"]: + log.info("Cleaning up build root ('cleanup_on_success=True')") + commands.clean() + if post: + post() + return ret + + except (Exception, KeyboardInterrupt): + elapsed = time.time() - start + log.error("Exception(%s) Config(%s) %d minutes %d seconds", + item, buildroot.shared_root_name, elapsed // 60, elapsed % 60) + log.info("Results and/or logs in: %s", buildroot.resultdir) + commands.plugins.call_hooks("process_logs") + if config_opts["cleanup_on_failure"]: + log.info("Cleaning up build root ('cleanup_on_failure=True')") + commands.clean() + raise + + +@traceLog() +def do_rebuild(config_opts, commands, buildroot, options, srpms): + "rebuilds a list of srpms using provided chroot" + if len(srpms) < 1: + log.critical("No package specified to rebuild command.") + sys.exit(50) + + if len(srpms) > 1 and options.spec: + log.critical("--spec argument only supported with single srpm.") + sys.exit(50) + + util.checkSrpmHeaders(srpms) + clean = config_opts['clean'] and not config_opts['scm'] + + def build(srpm): + commands.build(srpm, timeout=config_opts['rpmbuild_timeout'], + check=config_opts['check'], spec=options.spec) + + def post_build(): + if config_opts['post_install']: + if buildroot.chroot_was_initialized: + commands.install_build_results(commands.build_results) + else: + commands.init() + commands.install_build_results(commands.build_results) + if config_opts["cleanup_on_success"]: + log.info("Cleaning up build root ('cleanup_on_success=True')") + commands.clean() + + if config_opts["createrepo_on_rpms"]: + log.info("Running createrepo on binary rpms in resultdir") + with buildroot.uid_manager: + util.createrepo(config_opts, buildroot.resultdir) + + rebuild_generic(srpms, commands, buildroot, config_opts, cmd=build, + post=post_build, clean=clean) + + +# pylint: disable=unused-argument +@traceLog() +def do_buildsrpm(config_opts, commands, buildroot, options, args): + # verify the input command line arguments actually exist + if not os.path.isfile(options.spec): + raise BadCmdline("Input specfile does not exist: %s" % options.spec) + if options.sources and not os.path.isdir(options.sources) and not os.path.isfile(options.sources): + raise BadCmdline("Input sources directory or file does not exist: %s" % options.sources) + clean = config_opts['clean'] + + def cmd(spec): + return commands.buildsrpm(spec=spec, sources=options.sources, + timeout=config_opts['rpmbuild_timeout'], + follow_links=options.symlink_dereference) + + return rebuild_generic([options.spec], commands, buildroot, config_opts, + cmd=cmd, post=None, clean=clean) diff --git a/mock/py/mockbuild/scm.py b/mock/py/mockbuild/scm.py new file mode 100644 index 0000000..77bdc4c --- /dev/null +++ b/mock/py/mockbuild/scm.py @@ -0,0 +1,226 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Marko Myllynen +# Copyright (C) 2010 Marko Myllynen + +import os +import shlex +import shutil +import subprocess +import sys +import tempfile + +from . import util +from . import file_util +from .trace_decorator import traceLog + + +class scmWorker(object): + """Build RPMs from SCM""" + @traceLog() + def __init__(self, log, config_opts, macros): + opts = config_opts['scm_opts'] + self.log = log + self.log.debug("Initializing SCM integration...") + + self.name = self.version = None + self.wrk_dir = self.src_dir = None + self.sources = [] + self.macros = macros + self.branch = None + self.postget = [] + self.config = config_opts + + self.method = opts['method'] + if self.method == "cvs": + self.get = opts['cvs_get'] + elif self.method == "svn": + self.get = opts['svn_get'] + elif self.method == "git": + self.get = opts['git_get'] + elif self.method == "distgit": + self.get = opts['distgit_get'] + self.postget = [opts['distgit_src_get']] + else: + self.log.error("Unsupported SCM method: %s", self.method) + sys.exit(5) + + if 'branch' in opts: + self.branch = opts['branch'] + if self.branch: + if self.method == "cvs": + self.get = self.get.replace("SCM_BRN", "-r " + self.branch) + elif self.method == "git": + self.postget = ["git checkout " + self.branch] + if "--recursive" in self.get or "--recurse-submodules" in self.get: + self.postget.append("git submodule update --init --recursive") + elif self.method == "distgit": + self.get = self.get.replace("SCM_BRN", self.branch) + elif self.method == "svn": + self.get = self.get.replace("SCM_BRN", self.branch) + else: + self.log.error("Unsupported SCM method: %s", self.method) + sys.exit(5) + elif self.method == "svn": + self.get = self.get.replace("SCM_BRN", "trunk") + self.get = self.get.replace("SCM_BRN", "") + + if 'package' in opts: + self.pkg = opts['package'] + else: + self.log.error("Trying to use SCM, package not defined") + sys.exit(5) + self.get = self.get.replace("SCM_PKG", self.pkg) + + self.spec = opts['spec'] + self.spec = self.spec.replace("SCM_PKG", self.pkg) + + self.int_src_dir = opts['int_src_dir'] + self.ext_src_dir = opts['ext_src_dir'] + self.write_tar = opts['write_tar'] + self.exclude_vcs = opts['exclude_vcs'] + + self.git_timestamps = opts['git_timestamps'] + + self.log.debug("SCM checkout command: %s", self.get) + for command in self.postget: + self.log.debug("SCM checkout post command: %s", command) + + @traceLog() + def get_sources(self): + self.wrk_dir = tempfile.mkdtemp(".mock-scm." + os.path.basename(self.pkg)) + self.src_dir = self.wrk_dir + "/" + os.path.basename(self.pkg) + self.log.debug("SCM checkout directory: %s", self.wrk_dir) + try: + util.do(shlex.split(self.get), shell=False, cwd=self.wrk_dir, env=os.environ) + except PermissionError: + self.log.error("{} does not exist or cannot be executed due permissions." + .format(shlex.split(self.get)[0])) + sys.exit(5) + + for command in self.postget: + try: + util.do(shlex.split(command), shell=False, cwd=self.src_dir, env=os.environ) + except PermissionError: + self.log.error("{} does not exist or cannot be executed due permissions." + .format(shlex.split(command)[0])) + sys.exit(5) + + self.log.debug("Fetched sources from SCM") + + @traceLog() + def adjust_git_timestamps(self): + cwd_dir = util.pretty_getcwd() + self.log.debug("Adjusting timestamps in %s", self.src_dir) + os.chdir(self.src_dir) + proc = subprocess.Popen( + ['git', 'ls-files', '-z'], + shell=False, stdout=subprocess.PIPE, universal_newlines=True, + ) + for f in proc.communicate()[0].split('\0')[:-1]: + rev = subprocess.Popen( + ['git', 'rev-list', 'HEAD', f], + shell=False, stdout=subprocess.PIPE, universal_newlines=True, + ).stdout.readlines()[0].rstrip('\n') + ts = subprocess.Popen( + ['git', 'show', '--pretty=format:%ai', '--no-patch', rev, f], + shell=False, stdout=subprocess.PIPE, universal_newlines=True, + ).stdout.readlines()[0].rstrip('\n') + subprocess.Popen(['touch', '-d', ts, f], shell=False) + os.chdir(cwd_dir) + + @traceLog() + def prepare_sources(self): + # import rpm after setarch + # pylint: disable=import-outside-toplevel + import rpm + self.log.debug("Preparing SCM sources") + + # Check some helper files + if os.path.exists(self.src_dir + "/.write_tar"): + self.log.debug(".write_tar detected, will write tarball on the fly") + self.write_tar = True + + # Figure out the spec file + sf = self.src_dir + "/" + self.spec + if not os.path.exists(sf): + sf = self.src_dir + "/" + self.spec.lower() + if not os.path.exists(sf): + self.log.error("Can't find spec file %s/%s", self.src_dir, self.spec) + self.clean() + sys.exit(5) + self.spec = sf + + # Use specified sources directory + sources_dir = self.src_dir + if self.int_src_dir: + sources_dir = os.path.join(sources_dir, self.int_src_dir) + + # Add passed RPM macros before parsing spec file + for macro, expression in list(self.macros.items()): + # pylint: disable=no-member + rpm.addMacro(macro.lstrip('%'), str(expression)) + + # Dig out some basic information from the spec file + self.sources = [] + ts = rpm.ts() + # Spec might %include its sources + # pylint: disable=no-member + rpm.addMacro("_sourcedir", sources_dir) + rpm_spec = ts.parseSpec(self.spec) + self.name = rpm.expandMacro("%{name}") + self.version = rpm.expandMacro("%{version}") + tarball = None + for (filename, num, flags) in rpm_spec.sources: + self.sources.append(filename.split("/")[-1]) + if num == 0 and flags == 1: + tarball = filename.split("/")[-1] + self.log.debug("Sources: %s", self.sources) + + # Adjust timestamps for Git checkouts + if self.method == "git" and self.git_timestamps: + self.adjust_git_timestamps() + + # Generate a tarball from the checked out sources if needed + if str(self.write_tar).lower() == "true" and self.method != "distgit": + tardir = self.name + "-" + self.version + if tarball is None: + tarball = tardir + ".tar.gz" + taropts = "" + + __tar_cmd = self.config["tar_binary"] + + # Always exclude vcs data from tarball unless told not to + if str(self.exclude_vcs).lower() == "true" and self.config['tar'] == "gnutar": + proc = subprocess.Popen([__tar_cmd, '--help'], shell=False, stdout=subprocess.PIPE) + proc_result = proc.communicate()[0] + proc_result = proc_result.decode() + if "--exclude-vcs" in proc_result: + taropts = "--exclude-vcs" + + self.log.debug("Writing %s/%s...", sources_dir, tarball) + cwd_dir = os.getcwd() + os.chdir(self.wrk_dir) + os.rename(self.name, tardir) + cmd = "{0} caf {1} {2} {3}".format(__tar_cmd, tarball, taropts, tardir) + util.do(shlex.split(cmd), shell=False, cwd=self.wrk_dir, env=os.environ) + os.rename(tarball, tardir + "/" + (self.int_src_dir or "") + "/" + tarball) + os.rename(tardir, self.name) + os.chdir(cwd_dir) + + # Get possible external sources from an external sources directory + for f in self.sources: + if not os.path.exists(sources_dir + "/" + f) and \ + os.path.exists(self.ext_src_dir + "/" + f): + self.log.debug("Copying %s/%s to %s/%s", self.ext_src_dir, f, sources_dir, f) + shutil.copy2(self.ext_src_dir + "/" + f, sources_dir + "/" + f) + + self.log.debug("Prepared sources for building src.rpm") + + return (sources_dir, self.spec) + + @traceLog() + def clean(self): + self.log.debug("Clean SCM checkout directory") + file_util.rmtree(self.wrk_dir) diff --git a/mock/py/mockbuild/scrub_all.py b/mock/py/mockbuild/scrub_all.py new file mode 100644 index 0000000..ee9fece --- /dev/null +++ b/mock/py/mockbuild/scrub_all.py @@ -0,0 +1,107 @@ +""" +Logic for observing /var/lib/mock and /var/cache/mock, and try to cleanup as +much as possible sub-directories there. +""" + + +import os +from glob import glob +import subprocess +from mockbuild.constants import MOCKCONFDIR +from mockbuild.config import traverse_chroot_configs + + +def _do_scrub(configs, weird, chroot, suffix=None): + if suffix and chroot + "-" + suffix in weird: + print(f"skipping weird scrub: {chroot} {suffix}") + return + + # FIXME: use mockbuild.backend.Commands().scrub("all" instead + base_cmd = ["mock", "--scrub=all", "-r"] + cmd = base_cmd + ["eol/" + chroot if configs[chroot] == "eol" else chroot] + if suffix is not None: + cmd += ["--uniqueext", suffix] + + print("## Calling:", ' '.join(cmd), "##") + subprocess.call(cmd) + + +def scrub_all_chroots(): + """ + Traverse the important directories, and try to clean them up via + `--scrub=all` logic. + """ + + configs = {} + scrub = set() + scrub_bootstrap = set() + scrub_uniqueext = set() + scrub_uniqueext_bootstrap = set() + scrub_weird = set() + guessing_suffix = {} + + configs = {os.path.basename(f)[:-4]: ("eol" if eol else "normal") + for _, f, eol in traverse_chroot_configs(MOCKCONFDIR, + include_eol=True)} + for directory in glob("/var/lib/mock/*") + glob("/var/cache/mock/*"): + if not os.path.isdir(directory): + continue + + directory = os.path.basename(directory) + + if directory in configs: + scrub.add(directory) + continue + + if directory.endswith("-bootstrap"): + directory_no_bootstrap = directory[:-10] + if directory_no_bootstrap in configs: + scrub_bootstrap.add(directory_no_bootstrap) + continue + + guessing_suffix[directory] = None + + for config, _ in configs.items(): + for directory in list(guessing_suffix.keys()): + if guessing_suffix[directory]: + # already found the cleaning thing + continue + + if directory.startswith(config): + + suffix = directory[len(config) + 1:] + if suffix.endswith("-bootstrap"): + # See this: + # 1. alma+epel-8-x86_64-php-bootstrap + # 2. alma+epel-8-x86_64-bootstrap-php + # The 1. is weird, and we miss the corresponding + # configuration. The second could be a "php" uniqueext. + weird_chroot = directory[:-10] + scrub_weird.add(weird_chroot) + continue + + start = "bootstrap-" + if suffix.startswith(start): + suffix = suffix[len(start):] + scrub_uniqueext_bootstrap.add((config, suffix)) + else: + scrub_uniqueext.add((config, suffix)) + + guessing_suffix[directory] = "uniqueext" + + for sc, suffix in scrub_uniqueext_bootstrap - scrub_uniqueext: + _do_scrub(configs, scrub_weird, sc, suffix) + + for sc, suffix in scrub_uniqueext: + _do_scrub(configs, scrub_weird, sc, suffix) + + for only_bootstrap in scrub_bootstrap - scrub: + _do_scrub(configs, scrub_weird, only_bootstrap) + + for sc in scrub: + _do_scrub(configs, scrub_weird, sc) + + for directory, found in guessing_suffix.items(): + if found: + continue + print(f"Unknown directory: {directory}") diff --git a/mock/py/mockbuild/shadow_utils.py b/mock/py/mockbuild/shadow_utils.py new file mode 100644 index 0000000..7496758 --- /dev/null +++ b/mock/py/mockbuild/shadow_utils.py @@ -0,0 +1,92 @@ +""" +Create users/groups in chroot. Wrapping the useradd/groupadd utilities. +""" + +import grp +import pwd +from mockbuild.util import do_with_status + + +class ShadowUtils: + """ + Create a group + """ + def __init__(self, root): + self.root = root + + def _execute_command(self, command, can_fail=False): + with self.root.uid_manager.elevated_privileges(): + # Ordinarily we do not want to depend on shadow-utils in the buildroot, but + # configuring certain options (such as FreeIPA-provided subids) can make it + # impossible to create users in the buildroot using host shadow-utils so we + # provide this workaround. + # Tracking upstream bug https://github.com/shadow-maint/shadow/issues/897 + if self.root.config['use_host_shadow_utils']: + do_with_status(command + ['--prefix', self.root.make_chroot_path()], raiseExc=not can_fail) + else: + self.root.doChroot(command, raiseExc=not can_fail) + + def delete_user(self, username, can_fail=False): + """ + Delete user in self.root (/etc/passwd modified) + """ + command = ["userdel", "-f", username] + self._execute_command(command, can_fail=can_fail) + + def delete_group(self, groupname, can_fail=False): + """ + Delete group in self.root (/etc/group modified) + """ + command = ["groupdel", groupname] + self._execute_command(command, can_fail=can_fail) + + def create_group(self, groupname, gid=None): + """ + Create group in self.root (/etc/group modified) + """ + command = ["groupadd", groupname] + if gid is not None: + command += ["-g", str(gid)] + self._execute_command(command) + + def create_user(self, username, uid=None, gid=None, home=None): + """ + Create user in self.root (/etc/passwd modified) + """ + command = ["useradd", username] + if uid is not None: + command += ["-o", "-u", str(uid)] + if gid is not None: + command += ["-g", str(gid), "-N"] + if home is not None: + command += ["-d", str(home)] + self._execute_command(command) + + def copy_from_host(self, username): + """ + Copy user (with uid/gid/group_name) from Host into the self.root. + """ + try: + info = pwd.getpwnam(username) + uid = info.pw_uid + gid = info.pw_gid + except KeyError as err: + raise RuntimeError( + f"Can not find the requested user {username} " + "on host") from err + + try: + group_name = grp.getgrgid(gid).gr_name + except KeyError as err: + raise RuntimeError( + f"Can not find the requested GID {gid} " + "on host") from err + + self.delete_user(username, can_fail=True) + # This might fail because the group doesn't exist in the chroot (OK to + # ignore), or because there still are other users in the group (a + # serious error case, but OK to ignore because the subsequent + # 'crate_group' attempt will fail anyway). + self.delete_group(group_name, can_fail=True) + self.create_group(group_name, gid=gid) + self.create_user(group_name, uid=uid, gid=gid) diff --git a/mock/py/mockbuild/state.py b/mock/py/mockbuild/state.py new file mode 100644 index 0000000..68c440d --- /dev/null +++ b/mock/py/mockbuild/state.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# vim: noai:ts=4:sw=4:expandtab + +from .exception import StateError +from .trace_decorator import getLog + + +class State(object): + def __init__(self, bootstrap=None): + self._state = [] + # can be "unknown", "success" or "fail" + self.result = "unknown" + self.bootstrap = bootstrap + self.state_log = getLog("mockbuild.Root.state") + + def state(self): + if not len(self._state): + raise StateError("state called on empty state stack") + return self._state[-1] + + def start(self, state): + if state is None: + raise StateError("start called with None State") + self._state.append(state) + if self.bootstrap: + self.state_log.info("Start(bootstrap): %s", state) + else: + self.state_log.info("Start: %s", state) + + def finish(self, state): + if len(self._state) == 0: + raise StateError("finish called on empty state list") + current = self._state.pop() + if state != current: + raise StateError("state finish mismatch: current: %s, state: %s" % (current, state)) + if self.bootstrap: + self.state_log.info("Finish(bootstrap): %s", state) + else: + self.state_log.info("Finish: %s", state) + + def alldone(self): + if len(self._state) != 0: + raise StateError("alldone called with pending states: %s" % ",".join(self._state)) diff --git a/mock/py/mockbuild/text.py b/mock/py/mockbuild/text.py new file mode 100644 index 0000000..0e3fa61 --- /dev/null +++ b/mock/py/mockbuild/text.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: + +import locale + +from .trace_decorator import getLog + +encoding = locale.getpreferredencoding() + + +def compat_expand_string(string, conf_dict): + """ + Expand %(uid)s, etc., only if needed - and warn the user + that Jinja should be used instead. + """ + if '%(' not in string: + return string + getLog().warning("Obsoleted %(foo) config expansion in '{}', " + "use Jinja alternative {{foo}}".format(string)) + return string % conf_dict + + +def _to_text(obj, arg_encoding='utf-8', errors='strict', nonstring='strict'): + if isinstance(obj, str): + return obj + elif isinstance(obj, bytes): + return obj.decode(arg_encoding, errors) + else: + if nonstring == 'strict': + raise TypeError('First argument must be a string') + raise ValueError('nonstring must be one of: ["strict",]') + + +_to_native = _to_text diff --git a/mock/py/mockbuild/trace_decorator.py b/mock/py/mockbuild/trace_decorator.py new file mode 100644 index 0000000..ae5394c --- /dev/null +++ b/mock/py/mockbuild/trace_decorator.py @@ -0,0 +1,162 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Michael Brown +# Copyright (C) 2007 Michael E Brown + +import functools +import inspect +import logging +import os +import sys + + +# defaults to module verbose log +# does a late binding on log. Forwards all attributes to logger. +# works around problem where reconfiguring the logging module means loggers +# configured before reconfig dont output. +class getLog(object): + # pylint: disable=unused-argument,too-few-public-methods + def __init__(self, name=None, prefix="", *args, **kargs): + if name is None: + frame = inspect.getouterframes(inspect.currentframe())[1][0] + name = frame.f_globals["__name__"] + + self.name = prefix + name + + def __getattr__(self, name): + logger = logging.getLogger(self.name) + return getattr(logger, name) + + +# emulates logic in logging module to ensure we only log +# messages that logger is enabled to produce. +def doLog(logger, level, *args, **kargs): + if logger.manager.disable >= level: + return + if logger.isEnabledFor(level): + try: + logger.handle(logger.makeRecord(logger.name, level, *args, **kargs)) + except TypeError: + del(kargs["func"]) + logger.handle(logger.makeRecord(logger.name, level, *args, **kargs)) + + +def safe_repr(arg): + """ Generally repr() can fail when called before __init__(), we will workaround this case """ + try: + return repr(arg) + except AttributeError: + return str(type(arg)) + +def traceLog(logger=None): + def noop(func): + return func + + def decorator(func): + @functools.wraps(func) + def trace(*args, **kw): + # default to logger that was passed by module, but + # can override by passing logger=foo as function parameter. + # make sure this doesn't conflict with one of the parameters + # you are expecting + + filename = os.path.normcase(inspect.getsourcefile(func)) + func_name = func.__name__ + if hasattr(func, 'func_code'): + lineno = func.func_code.co_firstlineno + else: + lineno = func.__code__.co_firstlineno + + l2 = kw.get('logger', logger) + if l2 is None: + l2 = logging.getLogger("trace.%s" % func.__module__) + if isinstance(l2, str): + l2 = logging.getLogger(l2) + + message = "ENTER %s(" + message = message + ', '.join([safe_repr(arg) for arg in args]) + if args and kw: + message += ', ' + for k, v in list(kw.items()): + message = message + "%s=%s" % (k, safe_repr(v)) + message = message + ")" + + frame = inspect.getouterframes(inspect.currentframe())[1][0] + doLog(l2, logging.INFO, os.path.normcase(frame.f_code.co_filename), + frame.f_lineno, message, args=[func_name], exc_info=None, + func=frame.f_code.co_name) + try: + result = "Bad exception raised: Exception was not a derived "\ + "class of 'Exception'" + try: + result = func(*args, **kw) + except (KeyboardInterrupt, Exception) as e: + result = "EXCEPTION RAISED" + doLog(l2, logging.INFO, filename, lineno, + "EXCEPTION: %s\n", args=[e], + exc_info=sys.exc_info(), func=func_name) + raise + finally: + doLog(l2, logging.INFO, filename, lineno, + "LEAVE %s --> %s\n", args=[func_name, result], + exc_info=None, func=func_name) + + return result + return trace + #end of trace() + + if os.environ.get("MOCK_TRACE_LOG", "true") == "false": + return noop + + if logging.getLogger("trace").propagate: + return decorator + else: + return noop + + +# unit tests... +if __name__ == "__main__": + logging.basicConfig( + level=logging.WARNING, + format='%(name)s %(levelname)s %(filename)s, %(funcName)s, Line: %(lineno)d: %(message)s',) + log = getLog("foobar.bubble") + root = getLog(name="") + log.setLevel(logging.WARNING) + root.setLevel(logging.DEBUG) + + log.debug(" --> debug") + log.error(" --> error") + log.warning(" --> warning") + + @traceLog(log) + # pylint: disable=unused-argument + def testFunc(arg1, arg2="default", *args, **kargs): + return 42 + + testFunc("hello", "world", logger=root) + testFunc("happy", "joy", name="skippy") + testFunc("hi") + + @traceLog(root) + def testFunc22(): + return testFunc("archie", "bunker") + + testFunc22() + + @traceLog(root) + def testGen(): + yield 1 + yield 2 + + for j in testGen(): + log.debug("got: %s", j) + + @traceLog() + def anotherFunc(*args): + # pylint: disable=no-value-for-parameter + return testFunc(*args) + + anotherFunc("pretty") + + getLog() diff --git a/mock/py/mockbuild/uid.py b/mock/py/mockbuild/uid.py new file mode 100644 index 0000000..099fd23 --- /dev/null +++ b/mock/py/mockbuild/uid.py @@ -0,0 +1,229 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Michael Brown +# Copyright (C) 2007 Michael E Brown + +import atexit +import ctypes +import errno +import grp +import multiprocessing +import os +import pwd +import sys +from concurrent.futures import ProcessPoolExecutor +from contextlib import contextmanager + +from .trace_decorator import traceLog + +_libc = ctypes.CDLL(None, use_errno=True) + + +@traceLog() +def setup_uid_manager(): + mockgid = grp.getgrnam('mock').gr_gid + unprivUid = os.getuid() + unprivGid = mockgid + + # sudo + if os.environ.get("SUDO_UID") is not None: + unprivUid = int(os.environ['SUDO_UID']) + os.setgroups((mockgid,)) + + # consolehelper + if os.environ.get("USERHELPER_UID") is not None: + unprivUid = int(os.environ['USERHELPER_UID']) + unprivName = pwd.getpwuid(unprivUid).pw_name + secondary_groups = [g.gr_gid for g in grp.getgrall() if unprivName in g.gr_mem] + os.setgroups([mockgid] + secondary_groups) + + uidManager = UidManager(unprivUid, unprivGid) + return uidManager + +class UidManager(object): + @traceLog() + def __init__(self, unprivUid=-1, unprivGid=-1): + self.privStack = [] + self.privEnviron = [] + self.unprivUid = unprivUid + self.unprivGid = unprivGid + self.unprivEnviron = dict(os.environ) + self.unprivEnviron['HOME'] = pwd.getpwuid(unprivUid).pw_dir + self.mockgid = grp.getgrnam('mock').gr_gid + + @traceLog() + def __enter__(self): + self.dropPrivsTemp() + return self + + @traceLog() + def __exit__(self, exc_type, exc_val, exc_tb): + self.restorePrivs() + + @contextmanager + def elevated_privileges(self): + self._push() + self._elevatePrivs() + try: + yield + finally: + self.restorePrivs() + + @traceLog() + def becomeUser(self, uid, gid=-1): + # save current ruid, euid, rgid, egid + self._push() + self.become_user_without_push(uid, gid) + + @traceLog() + def dropPrivsTemp(self): + # save current ruid, euid, rgid, egid + self._push() + self.become_user_without_push(self.unprivUid, self.unprivGid) + os.environ.clear() + os.environ.update(self.unprivEnviron) + + @traceLog() + def restorePrivs(self): + # back to root first + self._elevatePrivs() + + # then set saved + privs = self.privStack.pop() + os.environ.clear() + os.environ.update(self.privEnviron.pop()) + os.setregid(privs['rgid'], privs['egid']) + setresuid(privs['ruid'], privs['euid']) + + @traceLog() + def dropPrivsForever(self): + self._elevatePrivs() + os.setregid(self.unprivGid, self.unprivGid) + os.setreuid(self.unprivUid, self.unprivUid) + + @traceLog() + def _push(self): + # save current ruid, euid, rgid, egid + self.privStack.append({ + "ruid": os.getuid(), + "euid": os.geteuid(), + "rgid": os.getgid(), + "egid": os.getegid(), + }) + self.privEnviron.append(dict(os.environ)) + + @traceLog() + # pylint: disable=no-self-use + def _elevatePrivs(self): + setresuid(0, 0, 0) + os.setregid(0, 0) + + @traceLog() + def become_user_without_push(self, uid, gid=None): + self._elevatePrivs() + if gid is not None: + os.setregid(gid, gid) + setresuid(uid, uid, 0) + + @traceLog() + def changeOwner(self, path, uid=None, gid=None, recursive=False): + self._elevatePrivs() + if uid is None: + uid = self.unprivUid + if gid is None: + gid = self.unprivGid + self._tolerant_chown(path, uid, gid) + if recursive: + for root, dirs, files in os.walk(path): + for d in dirs: + self._tolerant_chown(os.path.join(root, d), uid, gid) + for f in files: + self._tolerant_chown(os.path.join(root, f), uid, gid) + + @staticmethod + def _tolerant_chown(path, uid, gid): + """ chown() which does not raise error if file does not exist. """ + try: + os.lchown(path, uid, gid) + except OSError as e: + if e.errno == errno.ENOENT: + pass + else: + raise + + @traceLog() + def fix_different_chrootgid(self, config_opts): + """ Allow a different mock group to be specified. + This tries to solve chicken-egg problem. Because the uidManager + has to be initialized before reading config, but config_opts['chrootgid'] + is known only after reading config + """ + if config_opts['chrootgid'] != self.mockgid: + self.restorePrivs() + os.setgroups((self.mockgid, config_opts['chrootgid'])) + self.dropPrivsTemp() + + def drop_privs_forever_and_execute(self, method, *args, **kwargs): + """ + Assure that the process can not re-elevate privileges to root, and + execute the given method. + """ + self.dropPrivsForever() + atexit._clear() + return method(*args, **kwargs) + + def run_in_subprocess_without_privileges(self, method, *args, **kwargs): + """ + Execute the given method in a forked process that can not re-elevate + privileges to root (we drop the saved set-*IDs). The exceptions from + the child pops up to the parent. + """ + if sys.version_info >= (3, 14): + # RHEL 9+ supports this too + # Fedora 43+ needs this + pool_executor = ProcessPoolExecutor(max_workers=1, mp_context=multiprocessing.get_context("fork")) + else: + # Can be removed when we stop supporting RHEL8 + pool_executor = ProcessPoolExecutor(max_workers=1) + with pool_executor as executor: + future = executor.submit(self.drop_privs_forever_and_execute, + method, *args, **kwargs) + return future.result() + +def getresuid(): + ruid = ctypes.c_long() + euid = ctypes.c_long() + suid = ctypes.c_long() + res = _libc.getresuid(ctypes.byref(ruid), ctypes.byref(euid), ctypes.byref(suid)) + if res: + raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno())) + return (ruid.value, euid.value, suid.value) + + +def setresuid(ruid=-1, euid=-1, suid=-1): + ruid = ctypes.c_long(ruid) + euid = ctypes.c_long(euid) + suid = ctypes.c_long(suid) + res = _libc.setresuid(ruid, euid, suid) + if res: + raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno())) + + +def getresgid(): + rgid = ctypes.c_long() + egid = ctypes.c_long() + sgid = ctypes.c_long() + res = _libc.getresgid(ctypes.byref(rgid), ctypes.byref(egid), ctypes.byref(sgid)) + if res: + raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno())) + return (rgid.value, egid.value, sgid.value) + + +def setresgid(rgid=-1, egid=-1, sgid=-1): + rgid = ctypes.c_long(rgid) + egid = ctypes.c_long(egid) + sgid = ctypes.c_long(sgid) + res = _libc.setresgid(rgid, egid, sgid) + if res: + raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno())) diff --git a/mock/py/mockbuild/util.py b/mock/py/mockbuild/util.py new file mode 100644 index 0000000..27ada54 --- /dev/null +++ b/mock/py/mockbuild/util.py @@ -0,0 +1,1106 @@ +# -*- coding: utf-8 -*- +# vim:expandtab:autoindent:tabstop=4:shiftwidth=4:filetype=python:textwidth=0: +# License: GPL2 or later see COPYING +# Written by Michael Brown +# Sections by Seth Vidal +# Sections taken from Mach by Thomas Vander Stichele +# Copyright (C) 2007 Michael E Brown +from __future__ import print_function + +import atexit +import contextlib +import ctypes +import errno +import fcntl +from glob import glob +import logging +import os +import os.path +import re +import select +import signal +import shlex +import shutil +import struct +import subprocess +import sys +import tempfile +# pylint: disable=wrong-import-order +import termios +from textwrap import dedent +import time +import uuid + +import distro + +from . import exception +from . import file_util +from . import text +from .trace_decorator import getLog, traceLog +from .uid import setresuid +from pyroute2 import IPRoute + +_libc = ctypes.cdll.LoadLibrary(None) +_libc.personality.argtypes = [ctypes.c_ulong] +_libc.personality.restype = ctypes.c_int +_libc.unshare.argtypes = [ctypes.c_int] +_libc.unshare.restype = ctypes.c_int +_libc.sethostname.argtypes = [ctypes.c_char_p, ctypes.c_int] +_libc.sethostname.restype = ctypes.c_int + +# See linux/include/sched.h +CLONE_NEWNS = 0x00020000 +CLONE_NEWUTS = 0x04000000 +CLONE_NEWPID = 0x20000000 +CLONE_NEWNET = 0x40000000 +CLONE_NEWIPC = 0x08000000 + +# taken from sys/personality.h +PER_LINUX32 = 0x0008 +PER_LINUX = 0x0000 +personality_defs = { + 'x86_64': PER_LINUX, 'ppc64': PER_LINUX, 'sparc64': PER_LINUX, + 'i386': PER_LINUX32, 'i586': PER_LINUX32, 'i686': PER_LINUX32, + 'armv7': PER_LINUX32, 'armv7l': PER_LINUX32, 'armv7hl': PER_LINUX32, + 'armv7hnl': PER_LINUX32, 'armv7hcnl': PER_LINUX32, + 'armv7b': PER_LINUX32, 'armv7hb': PER_LINUX32, + 'armv7hnb': PER_LINUX32, 'armv7hcnb': PER_LINUX32, + 'armv8': PER_LINUX32, 'armv8l': PER_LINUX32, 'armv8hl': PER_LINUX32, + 'armv8hnl': PER_LINUX32, 'armv8hcnl': PER_LINUX32, + 'armv8b': PER_LINUX32, 'armv8hb': PER_LINUX32, + 'armv8hnb': PER_LINUX32, 'armv8hcnb': PER_LINUX32, + 'ppc': PER_LINUX32, 'sparc': PER_LINUX32, 'sparcv9': PER_LINUX32, + 'ia64': PER_LINUX, 'alpha': PER_LINUX, + 's390': PER_LINUX32, 's390x': PER_LINUX, + 'mips': PER_LINUX32, 'mipsel': PER_LINUX32, + 'mipsr6': PER_LINUX32, 'mipsr6el': PER_LINUX32, + 'mips64': PER_LINUX, 'mips64el': PER_LINUX, + 'mips64r6': PER_LINUX, 'mips64r6el': PER_LINUX, +} + +USE_NSPAWN = False +USE_NSPAWN_SECCOMP = False + +_NSPAWN_HELP_OUTPUT = None + +RHEL_CLONES = ['centos', 'deskos', 'ol', 'rhel', 'scientific'] + +_OPS_TIMEOUT = 0 + + +def cmd_pretty(cmd, env=None): + if isinstance(cmd, list): + return ' '.join(shlex.quote(arg) for arg in cmd) + if env: + variables = [f"{k}={shlex.quote(v)}" for k, v in env.items()] + cmd = " ".join(variables) + " " + cmd + return cmd + + +@traceLog() +def get_proxy_environment(config): + env = {} + for proto in ('http', 'https', 'ftp', 'no'): + key = '%s_proxy' % proto + value = config.get(key) + if value: + env[key] = value + elif os.getenv(key): + env[key] = os.getenv(key) + return env + + +def _safe_check_output(*args): + # this can be done in one call in python3, but python2 requires this hack + try: + output = subprocess.check_output(*args, shell=False, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + output = str(e.output) + return output + + +@traceLog() +def get_machinectl_uuid(chroot_path): + """ Get UUID from machinectl. This function does not check if NSPAWN is used """ + # we will ignore errors in machinectl, it sometimes fails for various errors (cannot find IP addr...) + # we do not care about exit code, we just want the output + vm_list = _safe_check_output(["/bin/machinectl", "list", "--no-legend", "--no-pager"]) + if (isinstance(vm_list, bytes)): + vm_list = vm_list.decode("utf-8") + for name in vm_list.split("\n"): + if len(name) > 0: + m_uuid = name.split()[0] + try: + vm_root = _safe_check_output(["/bin/machinectl", "show", "-pRootDirectory", m_uuid]) + if (isinstance(vm_root, bytes)): + vm_root = vm_root.decode("utf-8") + except subprocess.CalledProcessError: + continue + vm_root = '='.join(vm_root.rstrip().split('=')[1:]) + if vm_root == chroot_path: + return m_uuid + # we should never get here + return None + + +def compare_two_paths_cached(path1, path2, path_cache): + """ compare two files on dev/ino pairs """ + def file_dev_ino(path): + """ Return dev/ino pair for path, and cache results """ + if path in path_cache: + return path_cache[path] + stat_val = os.stat(os.path.realpath(path)) + ret = path_cache[path] = stat_val.st_dev, stat_val.st_ino + return ret + return file_dev_ino(path1) == file_dev_ino(path2) + + +def get_pid_cmdline(pid): + """ + For given PID return the command-line arguments from /proc/PID/cmdline + """ + cmd_file = f"/proc/{pid}/cmdline" + try: + with open(cmd_file, "rb") as cmdline_file: + cmdline = cmdline_file.read().decode('utf-8').split('\0') + cmdline.pop() # last string is always empty in 0-terminated file + return ' '.join([shlex.quote(x) for x in cmdline]) + except OSError as e: + return f"ERROR: Can not read {pid} file {e}" + + +@traceLog() +def orphansKill(rootToKill, manual_forced=False): + """ + Kill off anything that is still chrooted. + + When USE_NSPAWN==False, this method manually detects the running processes + in chroot by reading the /proc file-system. When USE_NSPAWN==True, it just + relies on '/bin/machinectl terminate' call. + + When manual_forced==True, the manual kill based on /proc is enforced. + """ + getLog().debug("kill orphans in chroot %s", rootToKill) + if USE_NSPAWN is False or manual_forced: + path_cache = {} + for killsig in [signal.SIGTERM, signal.SIGKILL]: + for fn in [d for d in os.listdir("/proc") if d.isdigit()]: + try: + root = os.readlink("/proc/%s/root" % fn) + if compare_two_paths_cached(root, rootToKill, path_cache): + pid = int(fn, 10) + getLog().warning("Leftover process %s is being killed with signal %s: %s", + pid, killsig, get_pid_cmdline(pid)) + os.kill(pid, killsig) + os.waitpid(pid, 0) + except OSError: + pass + else: + m_uuid = get_machinectl_uuid(rootToKill) + if m_uuid: + getLog().warning("Machine %s still running. Killing...", m_uuid) + os.system("/bin/machinectl terminate %s" % m_uuid) + + +@traceLog() +def yieldSrpmHeaders(srpms, plainRpmOk=0): + # pylint: disable=import-outside-toplevel + import rpm + ts = rpm.TransactionSet('/') + # When RPM > 4.14.90 is common we can use RPMVSF_MASK_NOSIGNATURES, RPMVSF_MASK_NODIGESTS + # pylint: disable=protected-access + flags = (rpm._RPMVSF_NOSIGNATURES | rpm._RPMVSF_NODIGESTS) + ts.setVSFlags(flags) + for srpm in srpms: + srpm = host_file(srpm) + try: + fd = os.open(srpm, os.O_RDONLY) + except OSError as e: + raise exception.Error("Cannot find/open srpm: %s. Error: %s" + % (srpm, e)) + try: + hdr = ts.hdrFromFdno(fd) + except rpm.error as e: + raise exception.Error( + "Cannot find/open srpm: %s. Error: %s" % (srpm, e)) + finally: + os.close(fd) + + if not plainRpmOk and hdr[rpm.RPMTAG_SOURCEPACKAGE] != 1: + raise exception.Error("File is not an srpm: %s." % srpm) + + yield hdr + + +@traceLog() +def checkSrpmHeaders(srpms, plainRpmOk=0): + for dummy in yieldSrpmHeaders(srpms, plainRpmOk): + pass + + +@traceLog() +def getNEVRA(hdr): + # pylint: disable=import-outside-toplevel + import rpm + name = hdr[rpm.RPMTAG_NAME] + ver = hdr[rpm.RPMTAG_VERSION] + rel = hdr[rpm.RPMTAG_RELEASE] + epoch = hdr[rpm.RPMTAG_EPOCH] + arch = hdr[rpm.RPMTAG_ARCH] + if epoch is None: + epoch = 0 + ret = (name, epoch, ver, rel, arch) + return tuple(text._to_text(x) if i != 1 else x for i, x in enumerate(ret)) + + +@traceLog() +def cmpKernelVer(str1, str2): + 'compare two kernel version strings and return -1, 0, 1 for less, equal, greater' + # pylint: disable=import-outside-toplevel + import rpm + return rpm.labelCompare(('', str1, ''), ('', str2, '')) + + +@traceLog() +def getAddtlReqs(hdr, conf): + # Add the 'more_buildreqs' for this SRPM (if defined in config file) + # pylint: disable=unused-variable + (name, epoch, ver, rel, arch) = getNEVRA(hdr) + reqlist = [] + for this_srpm in ['-'.join([name, ver, rel]), + '-'.join([name, ver]), + '-'.join([name])]: + if this_srpm in conf: + more_reqs = conf[this_srpm] + if isinstance(more_reqs, str): + reqlist.append(more_reqs) + else: + reqlist.extend(more_reqs) + break + + return set(reqlist) + + +@traceLog() +def unshare(flags): + #getLog().debug("Unsharing. Flags: %s", flags) + try: + res = _libc.unshare(flags) + if res: + raise exception.UnshareFailed(os.strerror(ctypes.get_errno())) + except AttributeError: + pass + + +def sethostname(hostname): + getLog().info("Setting hostname: %s", hostname) + hostname = hostname.encode('utf-8') + if _libc.sethostname(hostname, len(hostname)) != 0: + raise OSError('Failed to sethostname %s' % hostname) + + +# these are called in child process, so no logging +def condChroot(chrootPath): + if chrootPath is not None: + saved = {"ruid": os.getuid(), "euid": os.geteuid()} + setresuid(0, 0, 0) + os.chdir(chrootPath) + os.chroot(chrootPath) + setresuid(saved['ruid'], saved['euid']) + + +def condChdir(cwd): + if cwd is not None: + os.chdir(cwd) + + +def condDropPrivs(uid, gid): + if gid is not None: + os.setregid(gid, gid) + if uid is not None: + os.setreuid(uid, uid) + + +def condPersonality(per=None): + if per is None or per in ('noarch',): + return + if personality_defs.get(per, None) is None: + return + res = _libc.personality(personality_defs[per]) + if res == -1: + raise OSError(ctypes.get_errno(), os.strerror(ctypes.get_errno())) + + +def condEnvironment(env=None): + if not env: + return + os.environ.clear() + for k in list(env.keys()): + os.putenv(k, env[k]) + + +def condUnshareIPC(unshare_ipc=True): + if unshare_ipc: + try: + unshare(CLONE_NEWIPC) + except exception.UnshareFailed: + # IPC and UTS ns are supported since the same kernel version. If this + # fails, there had to be a warning already + pass + + +def condUnshareNet(unshare_net=True): + if USE_NSPAWN and unshare_net: + try: + unshare(CLONE_NEWNET) + # Set up loopback interface and add default route via loopback in the namespace. + # Missing default route may confuse some software, see + # https://github.com/rpm-software-management/mock/issues/113 + ipr = IPRoute() + dev = ipr.link_lookup(ifname='lo')[0] + + ipr.link('set', index=dev, state='up') + ipr.route("add", dst="default", gateway="127.0.0.1") + except exception.UnshareFailed: + # IPC and UTS ns are supported since the same kernel version. If this + # fails, there had to be a warning already + pass + except Exception as e: # pylint: disable=broad-except + getLog().warning("network namespace setup failed: %s", e) + + +def process_input(line): + out = [] + for char in line.rstrip('\r'): + if char == '\r': + out = [] + elif char == '\b': + out.pop() + else: + out.append(char) + return ''.join(out) + + +def logOutput(fdout, fderr, logger, returnOutput=1, start=0, timeout=0, printOutput=False, + child=None, chrootPath=None, pty=False, returnStderr=True): + output = "" + done = False + fds = [fdout, fderr] + + # set all fds to nonblocking + for fd in fds: + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + if not fd.closed: + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + + mockbuild_logger = logging.getLogger('mockbuild') + stored_propagate = mockbuild_logger.propagate + if printOutput: + # prevent output being printed twice when log propagates to stdout + mockbuild_logger.propagate = 0 + sys.stdout.flush() + try: + tail = "" + ansi_escape = re.compile(r'\x1b\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]\x0f?') + while not done: + if (time.time() - start) > timeout and timeout != 0: + done = True + break + + i_rdy, o_rdy, e_rdy = select.select(fds, [], [], 1) + + if not i_rdy and not o_rdy and not e_rdy: + if child and child.poll() is not None: + logger.info("Child pid '%s' is dead", child.pid) + done = True + if chrootPath: + logger.info("Child dead, killing orphans") + orphansKill(chrootPath) + continue + + for s in i_rdy: + # slurp as much input as is ready + raw = s.read() + if not raw: + done = True + break + if printOutput: + if hasattr(sys.stdout, 'buffer'): + # python3 would print binary strings ugly + # pylint: disable=no-member + sys.stdout.buffer.write(raw) + else: + print(raw, end='') + sys.stdout.flush() + + if returnStderr is False and s == fderr: + continue + + txt_input = raw.decode(text.encoding, 'replace') + lines = txt_input.split("\n") + if tail: + lines[0] = tail + lines[0] + # we may not have all of the last line + tail = lines.pop() + if not lines: + continue + if pty: + lines = [process_input(line) for line in lines] + processed_input = '\n'.join(lines) + '\n' + if "mock_stderr_line_prefix" in dir(mockbuild_logger): + mock_stderr_line_prefix = mockbuild_logger.mock_stderr_line_prefix + else: + mock_stderr_line_prefix = "" + if logger is not None: + for line in lines: + if line != '': + line = ansi_escape.sub('', line) + if fderr is s and not line.startswith('+ '): + logger.debug("%s%s", mock_stderr_line_prefix, line) + else: + logger.debug(line) + for h in logger.handlers: + h.flush() + if returnOutput: + output += processed_input + + if tail: + if pty: + tail = process_input(tail) + '\n' + if logger is not None: + logger.debug(tail) + if returnOutput: + output += tail + finally: + mockbuild_logger.propagate = stored_propagate + + return output + + +@traceLog() +def selinuxEnabled(): + """Check if SELinux is enabled (enforcing or permissive).""" + with open("/proc/mounts") as f: + for mount in f.readlines(): + (fstype, mountpoint, _) = mount.split(None, 2) + if fstype == "selinuxfs": + selinux_mountpoint = mountpoint + break + else: + selinux_mountpoint = "/selinux" + + try: + enforce_filename = os.path.join(selinux_mountpoint, "enforce") + with open(enforce_filename) as f: + if f.read().strip() in ("1", "0"): + return True + # pylint: disable=bare-except + except: + pass + return False + + +def resize_pty(pty): + try: + winsize = struct.pack('HHHH', 0, 0, 0, 0) + winsize = fcntl.ioctl(sys.stdout.fileno(), termios.TIOCGWINSZ, winsize) + fcntl.ioctl(pty, termios.TIOCSWINSZ, winsize) + except IOError: + # Nice to have, but not necessary + pass + + +def do(*args, **kargs): + """ returns output of the command. Arguments are the same as for do_with_status() """ + return do_with_status(*args, **kargs)[0] + +# logger = +# output = [1|0] +# chrootPath +# +# The "Not-as-complicated" version +# +@traceLog() +# pylint: disable=unused-argument +def do_with_status(command, shell=False, chrootPath=None, cwd=None, timeout=0, raiseExc=True, + returnOutput=0, uid=None, gid=None, user=None, personality=None, + printOutput=False, env=None, pty=False, nspawn_args=None, unshare_net=False, + returnStderr=True, *_, **kargs): + logger = kargs.get("logger", getLog()) + if timeout == 0: + timeout = _OPS_TIMEOUT + output = "" + start = time.time() + if pty: + lead_pty, sub_pty = os.openpty() + resize_pty(sub_pty) + reader = os.fdopen(lead_pty, 'rb') + preexec = ChildPreExec(personality, chrootPath, cwd, uid, gid, + unshare_ipc=bool(chrootPath), unshare_net=unshare_net) + if env is None: + env = clean_env() + stdout = None + + if isinstance(command, list): + # convert int args to strings + command = [str(x) for x in command] + + try: + child = None + if chrootPath and USE_NSPAWN: + logger.debug("Using nspawn with args %s", nspawn_args) + command = _prepare_nspawn_command(chrootPath, user, command, + nspawn_args=nspawn_args, + env=env, cwd=cwd, shell=shell) + shell = False + logger.debug("Executing command: %s with env %s and shell %s", command, env, shell) + with open(os.devnull, "r") as stdin: + child = subprocess.Popen( + command, + shell=shell, + env=env, + bufsize=0, close_fds=True, + stdin=stdin, + stdout=sub_pty if pty else subprocess.PIPE, + stderr=subprocess.PIPE, + preexec_fn=preexec, + ) + if not pty: + stdout = child.stdout + with child.stderr: + # use select() to poll for output so we dont block + output = logOutput( + reader if pty else child.stdout, child.stderr, + logger, returnOutput, start, timeout, pty=pty, + printOutput=printOutput, child=child, + chrootPath=chrootPath, returnStderr=returnStderr) + except: + # kill children if they arent done + if child is not None and child.returncode is None: + os.killpg(child.pid, 9) + try: + if child is not None: + os.waitpid(child.pid, 0) + except: # pylint: disable=bare-except + pass + raise + finally: + if pty: + os.close(sub_pty) + reader.close() + if stdout: + stdout.close() + + # wait until child is done, kill it if it passes timeout + niceExit = 1 + while child.poll() is None: + if (time.time() - start) > timeout and timeout != 0: + niceExit = 0 + os.killpg(child.pid, 15) + if (time.time() - start) > (timeout + 1) and timeout != 0: + niceExit = 0 + os.killpg(child.pid, 9) + + if not niceExit: + raise exception.commandTimeoutExpired("Timeout(%s) expired for command:\n # %s\n%s" % + (timeout, cmd_pretty(command, env), output)) + + logger.debug("Child return code was: %s", child.returncode) + if raiseExc and child.returncode: + raise exception.Error("Command failed: \n # %s\n%s" % (cmd_pretty(command, env), output), child.returncode) + + return (output, child.returncode) + + +class ChildPreExec(object): + def __init__(self, personality, chrootPath, cwd, uid, gid, env=None, + shell=False, unshare_ipc=False, unshare_net=False, + no_setsid=False): + """ + Params: + - no_setsid - assure we don't call os.setsid(), as the process we run + calls that itself + """ + self.personality = personality + self.chrootPath = chrootPath + self.cwd = cwd + self.uid = uid + self.gid = gid + self.env = env + self.shell = shell + self.unshare_ipc = unshare_ipc + self.unshare_net = unshare_net + self.no_setsid = no_setsid + getLog().debug("child environment: %s", env) + + def __call__(self, *args, **kargs): + if not self.shell and not self.no_setsid: + os.setsid() + os.umask(0o02) + condUnshareNet(self.unshare_net) + condPersonality(self.personality) + condEnvironment(self.env) + # Even if nspawn is allowed to be used, it won't be used unless there + # is a chrootPath set + if not USE_NSPAWN or not self.chrootPath: + condChroot(self.chrootPath) + condDropPrivs(self.uid, self.gid) + condChdir(self.cwd) + condUnshareIPC(self.unshare_ipc) + reset_sigpipe() + + +def setup_operations_timeout(config_opts): + global _OPS_TIMEOUT + _OPS_TIMEOUT = config_opts.get('opstimeout', 0) + + +def set_use_nspawn(value, config_opts): + global USE_NSPAWN + global USE_NSPAWN_SECCOMP + USE_NSPAWN = value + USE_NSPAWN_SECCOMP = config_opts["seccomp"] + + +class BindMountedFile(str): + 'see host_file() doc' + def __new__(cls, value, on_host=None): + the_string = str.__new__(cls, value) + the_string.on_host = on_host if on_host else value + return the_string + + +def host_file(file): + """ + Some functions accept arguments which may be either str() or + BindMountedFile(); we use this helper to work with those transparently. + TODO: all the code parts which need this should be fixed so they + are executed _inside_ bootstrap chroot, not on host. + """ + return file.on_host if hasattr(file, 'on_host') else file + + +def reset_sigpipe(): + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + +def _nspawnTempResolvAtExit(path): + """Remove nspawn temporary resolv.conf from host.""" + try: + os.remove(path) + except OSError as e: + if e.errno not in [errno.ENOENT, errno.EPERM]: + getLog().warning("unable to delete temporary resolv.conf (%s): %s", path, e) + + +def systemd_nspawn_help_output(): + """ Get (cached, so we don't re-run) systemd-nspawn --help output. """ + global _NSPAWN_HELP_OUTPUT # pylint: disable=global-statement + if _NSPAWN_HELP_OUTPUT is not None: + return _NSPAWN_HELP_OUTPUT + + _NSPAWN_HELP_OUTPUT = subprocess.check_output( + 'systemd-nspawn --help || true', + shell=True) + _NSPAWN_HELP_OUTPUT = _NSPAWN_HELP_OUTPUT.decode('utf-8', errors='ignore') + return _NSPAWN_HELP_OUTPUT + + +def _check_nspawn_pipe_option(): + """ + Detect whether host's systemd-nspawn supports --pipe argument and if we can + use it for non-interactive commands. Before --pipe was implemented in + nspawn the default behavior was to detect tty => and use 'interactive' vs. + 'pipe'. Later the default was changed to 'interactive' vs. 'read-only' + (systemd commit de40a3037). + """ + output = systemd_nspawn_help_output() + return '--pipe' in output and '--console' in output + + +def _check_nspawn_resolv_conf(): + """ + Detect that --resolv-conf= option is supported in systemd-nspawn, and if + yes - switch the default value 'auto' to 'off' so nspawn doesn't override + our pre-generated resolv.conf file. + """ + return '--resolv-conf' in systemd_nspawn_help_output() + + +def check_nspawn_has_chdir_option(): + """ + Older systemd-nspawn versions don't have --chdir option, and sometimes we + need to know we work with such version. + """ + return '--chdir' in systemd_nspawn_help_output() + + +def _prepare_nspawn_command(chrootPath, user, cmd, nspawn_args=None, env=None, + cwd=None, interactive=False, shell=False): + nspawn_argv = ['/usr/bin/systemd-nspawn', '-q', '-M', uuid.uuid4().hex, '-D', chrootPath] + distro_label = distro.id() + try: + distro_version = float(distro.version() or 0) + except ValueError: + distro_version = 0 + if distro_label not in RHEL_CLONES or distro_version >= 7.5: + # EL < 7.5 does not support the nspawn -a option. See BZ 1417387 + nspawn_argv += ['-a'] + + if user: + # user can be either id or name + nspawn_argv += ['-u', str(user)] + + if nspawn_args: + nspawn_argv.extend(nspawn_args) + + if _check_nspawn_pipe_option(): + if not interactive or not (sys.stdin.isatty() and sys.stdout.isatty()): + nspawn_argv += ['--console=pipe'] + + if cwd: + nspawn_argv.append('--chdir={0}'.format(cwd)) + + assert env is not None + + # Those variables are expected to be set _inside_ the container + for k, v in env.items(): + nspawn_argv.append('--setenv={0}={1}'.format(k, v)) + + # And these need to be set outside the container (processed by nspawn) + env['SYSTEMD_NSPAWN_TMPFS_TMP'] = '0' + if not USE_NSPAWN_SECCOMP: + env['SYSTEMD_SECCOMP'] = '0' + + if _check_nspawn_resolv_conf(): + nspawn_argv.append("--resolv-conf=off") + + # The '/bin/sh -c' wrapper is explicitly requested (--shell). In this case + # we shrink the list of arguments into one shell command, so the command is + # completely shell-expanded. + if shell and isinstance(cmd, list): + cmd = ' '.join(cmd) + + # HACK! No matter if --shell/--chroot is used, we have documented that we + # shell-expand the CMD if there are no ARGS. This is historical + # requirement that other people probably depend on. + if isinstance(cmd, str): + cmd = ['/bin/sh', '-c', cmd] + + return nspawn_argv + cmd + +def doshell(chrootPath=None, environ=None, uid=None, gid=None, cmd=None, + cwd=None, + nspawn_args=None, + unshare_ipc=True, + unshare_net=False): + log = getLog() + log.debug("doshell: chrootPath:%s, uid:%d, gid:%d", chrootPath, uid, gid) + if environ is None: + environ = clean_env() + if 'PROMPT_COMMAND' not in environ: + environ['PROMPT_COMMAND'] = r'printf "\033]0;\007"' + if 'PS1' not in environ: + environ['PS1'] = r' \s-\v\$ ' + if 'SHELL' not in environ: + environ['SHELL'] = '/bin/sh' + log.debug("doshell environment: %s", environ) + + no_setsid = False + shell = True + if not cmd: + cmd = ["/bin/sh", "-i", "-l"] + shell = False + no_setsid = True + elif isinstance(cmd, list): + cmd = ' '.join(cmd) + + preexec = ChildPreExec(personality=None, chrootPath=chrootPath, cwd=cwd, + uid=uid, gid=gid, env=environ, shell=shell, + unshare_ipc=unshare_ipc, unshare_net=unshare_net, + no_setsid=no_setsid) + + if USE_NSPAWN: + # nspawn cannot set gid + log.debug("Using nspawn with args %s", nspawn_args) + cmd = _prepare_nspawn_command(chrootPath, uid, cmd, nspawn_args=nspawn_args, env=environ, + interactive=True, cwd=cwd) + shell = False + + log.debug("doshell: command: %s", cmd_pretty(cmd)) + return subprocess.call(cmd, preexec_fn=preexec, env=environ, shell=shell) + + +def run(cmd, isShell=True): + log = getLog() + log.debug("run: cmd = %s", cmd_pretty(cmd)) + return subprocess.call(cmd, shell=isShell) + + +def clean_env(): + return { + 'TERM': 'vt100', + 'SHELL': '/bin/sh', + 'HOME': '/builddir', + 'HOSTNAME': 'mock', + 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin', + 'LANG': 'C.UTF-8', + } + + +@traceLog() +def setup_host_resolv(config_opts): + if not config_opts['use_host_resolv']: + # If we don't copy host's resolv.conf, we at least want to resolve + # our own hostname. See commit 28027fc26d. + if 'etc/hosts' not in config_opts['files']: + config_opts['files']['etc/hosts'] = dedent('''\ + 127.0.0.1 localhost localhost.localdomain + ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + ''') + + if config_opts['isolation'] == 'simple': + # Not using nspawn -> don't touch /etc/resolv.conf; we already have + # a valid file prepared by Buildroot._init() (if user requested). + return + + if config_opts['rpmbuild_networking'] and not config_opts['use_host_resolv']: + # keep the default systemd-nspawn's /etc/resolv.conf + return + + # Either we want to have empty resolv.conf to speedup name resolution + # failure (rpmbuild_networking is off, see commit 3f939785bb), or we want + # to copy hosts resolv.conf file. + + resolv_path = (tempfile.mkstemp(prefix="mock-resolv."))[1] + atexit.register(_nspawnTempResolvAtExit, resolv_path) + + # make sure that anyone in container can read resolv.conf file + os.chmod(resolv_path, 0o644) + + if config_opts['use_host_resolv']: + shutil.copyfile('/etc/resolv.conf', resolv_path) + + config_opts['nspawn_args'] += ['--bind={0}:/etc/resolv.conf'.format(resolv_path)] + + +def pretty_getcwd(): + try: + return os.getcwd() + except OSError: + if ORIGINAL_CWD is not None: + return ORIGINAL_CWD + else: + return file_util.find_non_nfs_dir() + + +ORIGINAL_CWD = None +ORIGINAL_CWD = pretty_getcwd() + + +@traceLog() +def find_btrfs_in_chroot(mockdir, chroot_path): + """ + Find a btrfs subvolume inside the chroot. + + Example btrfs output: + ID 258 gen 32689 top level 5 path root + ID 493 gen 32682 top level 258 path var/lib/mock/fedora-rawhide-x86_64/root/var/lib/machines + + The subvolume's path will always be the 9th field of the output and + will not contain a leading '/'. The output will also contain additional + newline at the end, which should not be parsed. + """ + + try: + output = do(["btrfs", "subv", "list", mockdir], returnOutput=1, printOutput=False) + except OSError as e: + # btrfs utility does not exist, nothing we can do about it + if e.errno == errno.ENOENT: + return None + raise e + except Exception as e: # pylint: disable=broad-except + # it is not btrfs volume + log = getLog() + log.debug("Please ignore the error above about btrfs.") + return None + + for l in output[:-1].splitlines(): + subv = l.split()[8] + if subv.startswith(chroot_path[1:]): + return subv + return None + + +@traceLog() +def createrepo(config_opts, path): + """ Create repository in given path. """ + cmd = shlex.split(config_opts["createrepo_command"]) + if os.path.exists(os.path.join(path, 'repodata/repomd.xml')): + cmd.append('--update') + cmd.append(path) + return do(cmd) + + +REPOS_ID = [] + + +@traceLog() +def generate_repo_id(baseurl): + """ generate repository id for yum.conf out of baseurl """ + repoid = baseurl + + # drop proto:// suffix + proto_split = baseurl.split('://') + if len(proto_split) > 1: + repoid = "/".join(proto_split[1:]) + else: + repoid = baseurl + + repoid = repoid.replace('/', '_') + repoid = re.sub(r'[^a-zA-Z0-9_]', '', repoid) + suffix = '' + i = 1 + while repoid + suffix in REPOS_ID: + suffix = str(i) + i += 1 + repoid = repoid + suffix + REPOS_ID.append(repoid) + return repoid + + +@traceLog() +def add_local_repo(config_opts, baseurl, repoid=None, bootstrap=None): + if not repoid: + repoid = generate_repo_id(baseurl) + else: + REPOS_ID.append(repoid) + localyumrepo = """ + +[{repoid}] +name={baseurl} +baseurl={baseurl} +enabled=1 +skip_if_unavailable=0 +metadata_expire=0 +gpgcheck=0 +cost=1 +best=1 +""".format(repoid=repoid, baseurl=baseurl) + + def _fix_cfg(cfg): + cfg['dnf.conf'] += localyumrepo + + _fix_cfg(config_opts) + + if bootstrap is None: + return + + _fix_cfg(bootstrap.config) + + +def subscription_redhat_init(opts, uidManager): + if not opts['redhat_subscription_required']: + return + + if 'redhat_subscription_key_id' in opts: + return + + ent_path = '/etc/pki/entitlement' + if not os.path.isdir(ent_path): + raise exception.ConfigError(ent_path + " is not a directory " + "is subscription-manager installed?") + + key_pattern = os.path.join(ent_path, "*-key.pem") + keys = glob(key_pattern) + if not keys: + hostdir = "/etc/pki/entitlement-host" + if os.path.isdir(hostdir): + # running in a Podman container + host_keys = glob(os.path.join(hostdir, "*-key.pem")) + if host_keys: + with uidManager.elevated_privileges(): + for file in glob(os.path.join(hostdir, "*.pem")): + basename = os.path.basename(file) + target = os.path.join(ent_path, basename) + shutil.copy2(file, target) + + keys = glob(key_pattern) + if not keys: + raise exception.ConfigError( + "No key found in /etc/pki/entitlement directory. It means " + "this machine is not subscribed. Please use \n" + " 1. subscription-manager register\n" + " 2. subscription-manager list --all --available " + "(available pool IDs)\n" + " 3. subscription-manager attach --pool \n" + "If you don't have Red Hat subscription yet, consider " + "getting subscription:\n" + " https://access.redhat.com/solutions/253273\n" + "You can have a free developer subscription:\n" + " https://developers.redhat.com/faq/" + ) + + # Use the first available key. + key_file_name = os.path.basename(keys[0]) + opts['redhat_subscription_key_id'] = key_file_name.split('-')[0] + + +def is_host_rh_family(): + distro_name = distro.id() + return distro_name in RHEL_CLONES + ['fedora'] + +def mock_host_environment_type(): + """ + Detect if we run in Docker. + """ + if hasattr(mock_host_environment_type, "cached_retval"): + return mock_host_environment_type.cached_retval + + def _cache(retval): + mock_host_environment_type.cached_retval = retval + getLog().info("Guessed host environment type: %s", retval) + return retval + + # Docker container has different cgroup than PID 1 of host. + # And have "docker" in that tree. + with open('/proc/self/cgroup', encoding="utf8") as f: + for line in f: + items = line.split(':') + if 'docker' in items[2]: + return _cache("docker") + # For containers with cgroupv2 + with open('/proc/self/mountinfo', encoding='utf8') as f: + for line in f: + if '/docker/containers/' in line and "/etc/hosts" in line: + return _cache("docker") + + return _cache("unknown") + + +@contextlib.contextmanager +def nullcontext(): + """ + contextlib.nullcontext is not available in Python 3.6, but we are still + Python 3.6+ compatible because of EL 8 + """ + yield None + +@contextlib.contextmanager +def env_var_override(name, value): + """ + Temporary set environment variable NAME to VALUE, revert to the previous + state. + """ + + oldval = None + if name in os.environ: + oldval = os.environ[name] + + os.environ[name] = value + yield + + if oldval is None: + del os.environ[name] + else: + os.environ[name] = oldval diff --git a/mock/pylintrc b/mock/pylintrc new file mode 100644 index 0000000..2ef8a28 --- /dev/null +++ b/mock/pylintrc @@ -0,0 +1,146 @@ +# mock pylint configuration + +[MAIN] + +# Pickle collected data for later comparisons. +persistent=no + +[MESSAGES CONTROL] + +# Reasoning for wide warning ignore +# --------------------------------- +# import-error +# This is here to silence Pylint in CI where we do not have all the +# build/runtime dependencies installed. +# cyclic-import +# Seems like cyclic-import is just a style check which is not going to be +# fixed: https://github.com/PyCQA/pylint/issues/6983 +disable=import-error,cyclic-import + +# list of disabled messages: +#I0011: 62: Locally disabling R0201 +#C0302: 1: Too many lines in module (2425) +#C0111: 1: Missing docstring +#R0902: 19:RequestedChannels: Too many instance attributes (9/7) +#R0904: 26:Transport: Too many public methods (22/20) +#R0912:171:set_slots_from_cert: Too many branches (59/20) +#R0913:101:GETServer.__init__: Too many arguments (11/10) +#R0914:171:set_slots_from_cert: Too many local variables (38/20) +#R0915:171:set_slots_from_cert: Too many statements (169/50) +#W0142:228:MPM_Package.write: Used * or ** magic +#W0403: 28: Relative import 'rhnLog', should be 'backend.common.rhnLog' +#W0603: 72:initLOG: Using the global statement +# for pylint-1.0 we also disable +#C1001: 46, 0: Old-style class defined. (old-style-class) +#W0121: 33,16: Use raise ErrorClass(args) instead of raise ErrorClass, args. (old-raise-syntax) +#W:243, 8: Else clause on loop without a break statement (useless-else-on-loop) +# pylint-1.1 checks +#C:334, 0: No space allowed after bracket (bad-whitespace) +#W:162, 8: Attempting to unpack a non-sequence defined at line 6 of (unpacking-non-sequence) +#C: 37, 0: Unnecessary parens after 'not' keyword (superfluous-parens) +#C:301, 0: Unnecessary parens after 'if' keyword (superfluous-parens) + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html +output-format=text + +# Tells whether to display a full report or only the messages +reports=no + + +[VARIABLES] + +# A regular expression matching names used for dummy variables (i.e. not used). +dummy-variables-rgx=_|dummy + + +[BASIC] + +# Regular expression which should only match correct module names +#module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ +module-rgx=([a-zA-Z_][a-zA-Z0-9_]+)$ + +# Regular expression which should only match correct module level names +const-rgx=(([a-zA-Z_][a-zA-Z0-9_]*)|(__.*__))$ + +# Regular expression which should only match correct class names +class-rgx=[a-zA-Z_][a-zA-Z0-9_]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-zA-Z0-9_]{,42}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-zA-Z0-9_]{,42}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-zA-Z0-9_]{,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-zA-Z0-9_]{,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-zA-Z0-9_]{,30}$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Regular expression which should only match correct class sttribute names +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,42}|(__.*__))$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=10 + +# Maximum number of locals for function / method body +max-locals=20 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=20 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=1 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module +max-module-lines=2000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes= diff --git a/mock/requirements.txt b/mock/requirements.txt new file mode 100644 index 0000000..6102064 --- /dev/null +++ b/mock/requirements.txt @@ -0,0 +1,8 @@ +backoff +distro +jinja2 +pyroute2==0.5.3; python_version < "3.9" +pyroute2; python_version >= "3.9" +requests +rpmautospec-core +templated-dictionary diff --git a/mock/run-tests.sh b/mock/run-tests.sh new file mode 100755 index 0000000..968291f --- /dev/null +++ b/mock/run-tests.sh @@ -0,0 +1,38 @@ +#! /bin/bash + +script=$(readlink -f "$0") +testdir=$(dirname "$script")/tests +sourcedir=$(readlink -f "$testdir/../py") + +: "${PYTHON=/usr/bin/python3}" + +if test -n "$PYTHONPATH"; then + PYTHONPATH=$PYTHONPATH:$sourcedir +else + PYTHONPATH=$sourcedir +fi +export PYTHONPATH + +debug() { echo >&2 " * $*" ; } + +debug "sourcedir: $sourcedir" +debug "testdir: $testdir" +debug "PYTHON: $PYTHON" +debug "PYTHONPATH: $PYTHONPATH" + +args=() +cov=true +for arg; do + case $arg in + --no-cov) cov=false ;; + *) args+=( "$arg" ) ;; + esac +done + +cov_args=() +if $cov; then + cov_args=( --cov-report term-missing --cov "$sourcedir" ) +fi + +set -x +"$PYTHON" -B -m pytest "${cov_args[@]}" "$testdir" "${args[@]}" diff --git a/mock/scripts/test-cfgs.py b/mock/scripts/test-cfgs.py new file mode 100644 index 0000000..82605f0 --- /dev/null +++ b/mock/scripts/test-cfgs.py @@ -0,0 +1,96 @@ +#!/usr/bin/python3 -tt +# +# Script to check validity of mock config URLs +# + +import glob +import os +import os.path +import urllib + + +class Config(object): + def __init__(self, file): + self.path = file + self.cfg = os.path.basename(self.path)[0:-4] + self.stanzas = [] + self.map = {} + current_key = '' + with open(self.path) as f: + for l in f: + l = l.strip() + if l.startswith('#'): + continue + if l.startswith('['): + key = l[1:l.rindex(']')] + current_key = key + if key == 'main' or key == 'local': + continue + self.stanzas.append(current_key) + self.map[current_key] = {} + continue + if 'http://' in l or 'https://' in l: + if current_key == 'main' or current_key == 'local': + continue + key, url = l.split('=', 1) + self.map[current_key][key.strip()] = url.strip() + continue + + def __str__(self): + return self.cfg + + def check_urls(self): + print(self.cfg) + total_sites = 0 + for s in self.stanzas: + for k in list(self.map[s].keys()): + if k == 'mirrorlist': + num = self.check_mirrorlist(self.map[s][k]) + if num == 0: + print("\t[%s] Error: no mirror sites\t<-------" % s) + else: + print("\t[%s] Ok (%d sites)" % (s, num)) + total_sites += num + elif k == 'baseurl': + if self.check_baseurl(self.map[s][k]) == 0: + print("\t[%s] Error: no files for baseurl\t<-------" % s) + else: + print("\t[%s] baseurl Ok" % s) + total_sites += 1 + elif k == 'metalink': + print("\t[%s] Warning: metalink check not implemented yet" % s) + else: + raise RuntimeError("Unknown URL type in %s: %s" % (s, k)) + if total_sites == 0: + print(" %s has no valid URLs" % self.cfg) + + def check_mirrorlist(self, url): + # print("checking mirrorlist at %s" % url) + try: + lines = [l for l in urllib.request.urlopen(url).readlines() + if not l.startswith(b'#') and len(l.strip()) != 0] + if len(lines) == 1 and lines[0].startswith(b'Bad arch'): + return 0 + return len(lines) + except urllib.error.URLError: + pass + return 0 + + def check_baseurl(self, url): + # print("checking baseurl at %s" % url) + try: + data = urllib.request.urlopen(url).readlines() + except urllib.error.HTTPError: + return 0 + except urllib.error.URLError: + return 0 + return len(data) + + +if __name__ == '__main__': + configs = glob.glob('etc/mock/*.cfg') + configs.sort() + for c in configs: + if os.path.basename(c).startswith('site-defaults'): + continue + Config(c).check_urls() diff --git a/mock/setup.cfg b/mock/setup.cfg new file mode 100644 index 0000000..8a7ec9a --- /dev/null +++ b/mock/setup.cfg @@ -0,0 +1,8 @@ +[flake8] +max-line-length = 120 +# flake8-import-order: +import-order-style = google +application-import-names = mockbuild + +[pep8] +max-line-length = 120 diff --git a/mock/tests/__init__.py b/mock/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mock/tests/conftest.py b/mock/tests/conftest.py new file mode 100644 index 0000000..2d70b28 --- /dev/null +++ b/mock/tests/conftest.py @@ -0,0 +1,14 @@ +"""Common pytest fixtures.""" + +import pytest + + +pytest_version_tuple = tuple(int(piece) for piece in pytest.__version__.split(".")) +if pytest_version_tuple < (3, 9): + # Old versions of pytest don’t have the tmp_path fixture, fill it in here. + from pathlib import Path + + @pytest.fixture + def tmp_path(tmpdir): + """Return temporary directory path object.""" + return Path(tmpdir) diff --git a/mock/tests/data/config-001/fedora-rawhide-x86_64.cfg b/mock/tests/data/config-001/fedora-rawhide-x86_64.cfg new file mode 100644 index 0000000..c9d47a6 --- /dev/null +++ b/mock/tests/data/config-001/fedora-rawhide-x86_64.cfg @@ -0,0 +1,7 @@ +include("templates/fedora-rawhide.tpl") + +config_opts["target_arch"] = "x86_64" + +config_opts["override_wins_conf"] = "conf" +config_opts["override_wins_home_site"] = "conf" +config_opts["override_wins_home_conf"] = "conf" diff --git a/mock/tests/data/config-001/site-defaults.cfg b/mock/tests/data/config-001/site-defaults.cfg new file mode 100644 index 0000000..94c70d7 --- /dev/null +++ b/mock/tests/data/config-001/site-defaults.cfg @@ -0,0 +1,7 @@ +config_opts["default"] = True + +config_opts["override_wins_site"] = "site" +config_opts["override_wins_conf"] = "site" +config_opts["override_wins_home_site"] = "site" +config_opts["override_wins_home_conf"] = "site" +config_opts["override_wins_template"] = "site" diff --git a/mock/tests/data/config-001/templates/fedora-rawhide.tpl b/mock/tests/data/config-001/templates/fedora-rawhide.tpl new file mode 100644 index 0000000..70c354f --- /dev/null +++ b/mock/tests/data/config-001/templates/fedora-rawhide.tpl @@ -0,0 +1,6 @@ +config_opts["template"] = "templated_value" + +config_opts["override_wins_conf"] = "template" +config_opts["override_wins_home_site"] = "template" +config_opts["override_wins_home_conf"] = "template" +config_opts["override_wins_template"] = "template" diff --git a/mock/tests/data/home-001/.config/mock.cfg b/mock/tests/data/home-001/.config/mock.cfg new file mode 100644 index 0000000..e8303e3 --- /dev/null +++ b/mock/tests/data/home-001/.config/mock.cfg @@ -0,0 +1,8 @@ +config_opts["home_default"] = (1, 2) + +# A bit tricky. The order of evaluation is: +# 1. site-defaults.cfg +# 2. configuration file => fedora-rawhide-x86_64.cfg in HOME! +# 3. configuration file included +# 4. home mock.cfg +config_opts["override_wins_home_site"] = "home_site" diff --git a/mock/tests/data/home-001/.config/mock/fedora-rawhide-x86_64.cfg b/mock/tests/data/home-001/.config/mock/fedora-rawhide-x86_64.cfg new file mode 100644 index 0000000..815f5f1 --- /dev/null +++ b/mock/tests/data/home-001/.config/mock/fedora-rawhide-x86_64.cfg @@ -0,0 +1,5 @@ +include("fedora-rawhide-x86_64.cfg") +config_opts["home_override"] = "10" + +config_opts["override_wins_home_conf"] = "home_conf" +config_opts["override_wins_home_site"] = "home_conf" diff --git a/mock/tests/plugins/__init__.py b/mock/tests/plugins/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/mock/tests/plugins/test_rpmautospec.py b/mock/tests/plugins/test_rpmautospec.py new file mode 100644 index 0000000..04a8166 --- /dev/null +++ b/mock/tests/plugins/test_rpmautospec.py @@ -0,0 +1,192 @@ +"""Test the rpmautospec plugin.""" + +from copy import deepcopy +from pathlib import Path +from unittest import mock + +import pytest + +from mockbuild.exception import ConfigError, PkgError +from mockbuild.plugins import rpmautospec +from mockbuild.util import nullcontext + +UNSET = object() + + +@mock.patch("mockbuild.plugins.rpmautospec.RpmautospecPlugin") +def test_init(RpmautospecPlugin): # pylint: disable=invalid-name + """Test the function which registers the plugin.""" + plugins = object() + conf = object() + buildroot = object() + + rpmautospec.init(plugins, conf, buildroot) + + RpmautospecPlugin.assert_called_once_with(plugins, conf, buildroot) + + +class TestRpmautospecPlugin: + """Test the RpmautospecPlugin class.""" + + DEFAULT_OPTS = { + "requires": ["rpmautospec"], + "cmd_base": ["rpmautospec", "process-distgit"], + } + + def create_plugin(self, plugins=UNSET, conf=UNSET, buildroot=UNSET): + """Create a plugin object and prepare it for testing.""" + if plugins is UNSET: + plugins = mock.Mock() + if conf is UNSET: + conf = deepcopy(self.DEFAULT_OPTS) + if buildroot is UNSET: + buildroot = mock.Mock() + + return rpmautospec.RpmautospecPlugin(plugins, conf, buildroot) + + @pytest.mark.parametrize( + "with_cmd_base", (True, False), ids=("with-cmd_base", "without-cmd_base") + ) + @mock.patch("mockbuild.plugins.rpmautospec.getLog") + def test___init__(self, getLog, with_cmd_base): + """Test the constructor.""" + plugins = mock.Mock() + + conf = deepcopy(self.DEFAULT_OPTS) + if with_cmd_base: + expectation = nullcontext() + else: + expectation = pytest.raises(ConfigError) + del conf["cmd_base"] + + buildroot = mock.Mock() + + with expectation as exc_info: + logger = getLog.return_value + plugin = self.create_plugin(plugins=plugins, conf=conf, buildroot=buildroot) + + if with_cmd_base: + assert plugin.buildroot is buildroot + assert plugin.config is buildroot.config + assert plugin.opts is conf + assert plugin.log is logger + plugins.add_hook.assert_called_once_with("pre_srpm_build", plugin.attempt_process_distgit) + logger.info.assert_called_once_with("rpmautospec: initialized") + else: + assert "rpmautospec_opts.cmd_base" in str(exc_info.value) + + @pytest.mark.parametrize( + "testcase", + ( + "happy-path", + "happy-path-no-requires", + "without-sources", + "sources-not-dir", + "sources-not-repo", + "sources-no-specfile", + "spec-files-different", + "specfile-no-rpmautospec", + "broken-requires", + ), + ) + def test_attempt_process_distgit( + self, testcase, tmp_path + ): # pylint: disable=too-many-branches disable=too-many-statements disable=too-many-locals + """Test the attempt_process_distgit() method.""" + # Set the stage + plugin = self.create_plugin() + plugin.log = log = mock.Mock() + plugin.buildroot.make_chroot_path.return_value = str(tmp_path) + if "no-requires" in testcase: + plugin.opts["requires"] = [] + + spec_dir = tmp_path / "SPECS" + spec_dir.mkdir() + sources_dir = tmp_path / "SOURCES" + sources_dir.mkdir() + + host_chroot_spec = spec_dir / "pkg.spec" + host_chroot_sources = sources_dir / "pkg" + host_chroot_sources.mkdir() + host_chroot_sources_git = host_chroot_sources / ".git" + host_chroot_sources_spec = host_chroot_sources / "pkg.spec" + + if "no-rpmautospec" not in testcase: + spec_contents = ( + "Release: %autorelease", + "%changelog", + "%autochangelog", + ) + else: + spec_contents = ( + "Release: 1", + "%changelog", + ) + + with host_chroot_spec.open("w") as fp: + for line in spec_contents: + print(line, file=fp) + + if "without-sources" in testcase: + host_chroot_sources = None + elif "sources-not-dir" not in testcase: + if "sources-no-specfile" not in testcase: + with host_chroot_sources_spec.open("w") as fp: + if "spec-files-different" in testcase: + print("# BOO", file=fp) + for line in spec_contents: + print(line, file=fp) + if "sources-not-repo" not in testcase: + host_chroot_sources_git.mkdir() + else: + host_chroot_sources = tmp_path / "pkg.tar" + host_chroot_sources.touch() + + if "broken-requires" in testcase: + plugin.buildroot.install_as_root.side_effect = RuntimeError("FAIL") + expect_exception = pytest.raises(PkgError) + else: + expect_exception = nullcontext() + + with expect_exception as excinfo: + plugin.attempt_process_distgit(host_chroot_spec, host_chroot_sources) + + if "happy-path" in testcase: + chroot_spec = Path("/") / host_chroot_spec.relative_to(tmp_path) + chroot_sources = Path("/") / host_chroot_sources.relative_to(tmp_path) + chroot_sources_spec = Path("/") / host_chroot_sources_spec.relative_to(tmp_path) + + expected_command = plugin.opts["cmd_base"] + [chroot_sources_spec, chroot_spec] + + plugin.buildroot.doChrootPlugin.assert_called_once_with( + expected_command, + cwd=chroot_sources, + ) + else: + plugin.buildroot.doChrootPlugin.assert_not_called() + if "broken-requires" not in testcase: + if "spec-files-different" in testcase: + log_method = log.warning + else: + log_method = log.debug + log_method.assert_called_once() + log_string = log_method.call_args[0][0] + assert "skipping rpmautospec preprocessing" in log_string + + if "without-sources" in testcase: + assert "Sources not specified" in log_string + elif "sources-not-dir" in testcase: + assert "Sources not a directory" in log_string + elif "sources-not-repo" in testcase: + assert "Sources is not a git repository" in log_string + elif "sources-no-specfile" in testcase: + assert "Sources doesn’t contain spec file" in log_string + elif "spec-files-different" in testcase: + assert "Spec file inside and outside sources are different" in log_string + elif "specfile-no-rpmautospec" in testcase: + assert "Spec file doesn’t use rpmautospec" in log_string + else: + assert str(excinfo.value) == ( + "Can’t install rpmautospec dependencies into chroot: " + + ", ".join(self.DEFAULT_OPTS["requires"]) + ) diff --git a/mock/tests/test_buildroot.py b/mock/tests/test_buildroot.py new file mode 100644 index 0000000..3df8dad --- /dev/null +++ b/mock/tests/test_buildroot.py @@ -0,0 +1,24 @@ +""" Tests for buildroot.py """ + +from unittest.mock import MagicMock +from mockbuild import util, buildroot + + +def test_module_config(): + """ test _module_commands_from_config method """ + def _check(config, output): + assert buildroot.Buildroot._module_commands_from_config(config) \ + == output + _check([("enable", "module:stream")], + [["module", "enable", "module:stream"]]) + _check([("enable", "module:stream, module2:stream2")], + [["module", "enable", "module:stream", "module2:stream2"]]) + _check([("disable", "module:*,module2:*"), + ("enable", "module:stream, module2:stream2"), + ("install", "pg:12")], + [["module", "disable", "module:*", "module2:*"], + ["module", "enable", "module:stream", "module2:stream2"], + ["module", "install", "pg:12"]]) + + _check([("info", "")], [["module", "info"]]) + _check([("info", None)], [["module", "info"]]) diff --git a/mock/tests/test_buildroot_lock.py b/mock/tests/test_buildroot_lock.py new file mode 100644 index 0000000..f6ea492 --- /dev/null +++ b/mock/tests/test_buildroot_lock.py @@ -0,0 +1,162 @@ +""" +Test the methods that generate buildroot_lock.json +""" + +import copy +import json +import os +import tempfile +from unittest import TestCase +from unittest.mock import MagicMock, patch +import jsonschema + +from mockbuild.plugins.buildroot_lock import init +import mockbuild.exception + +# gpg-pubkey packages stay ignored +# fedora-release is normal package with simple license +RPM_OUTPUT = """\ +gpg-pubkey|/@(none)|/@pubkey|/@105ef944|/@65ca83d1|/@(none)|/@(none)|/@(none) +gpg-pubkey|/@(none)|/@pubkey|/@e99d6ad1|/@64d2612c|/@(none)|/@(none)|/@(none) +fedora-release|/@noarch|/@MIT|/@42|/@0.3|/@(none)|/@cf31d87e5e3eac97ff32a98e7e073f37|/@(none) +bash|/@x86_64|/@GPLv3+|/@5.1.8|/@9.el9|/@(none)|/@57e93b1739cc3512f9f29dcaa8a38055|/@RSA/SHA256, Sun Mar 31 12:41:20 2024, Key ID 199e2f91fd431d51 +""" # noqa: E501 + +# cyrus-sasl-lib is not present in the REPOQUERY_OUTPUT +RPM_ONLY_RPM = """\ +cyrus-sasl-lib|/@x86_64|/@BSD with advertising|/@2.1.27|/@21.el9|/@(none)|/@9e1caba09fac94568419b9dfd14fb4c5|/@RSA/SHA256, Mon Sep 12 23:24:22 2022, Key ID 199e2f91fd431d51 +""" # noqa: E501 + +REPOQUERY_OUTPUT = """\ +http://ftp.fi.muni.cz/pub/linux/fedora/linux/development/rawhide/Everything/x86_64/os/Packages/f/fedora-release-42-0.3.noarch.rpm +https://cdn.redhat.com/content/dist/rhel9/9/x86_64/baseos/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm +""" + +EXPECTED_OUTPUT = { + 'version': '1.1.0', + 'buildroot': { + 'rpms': [{ + 'arch': 'x86_64', + 'epoch': None, + 'license': 'GPLv3+', + 'name': 'bash', + 'release': '9.el9', + 'sigmd5': '57e93b1739cc3512f9f29dcaa8a38055', + 'signature': 'fd431d51', + 'url': 'https://cdn.redhat.com/content/dist/rhel9/9/x86_64' + '/baseos/os/Packages/b/bash-5.1.8-9.el9.x86_64.rpm', + 'version': '5.1.8' + }, { + 'arch': 'noarch', + 'epoch': None, + 'license': 'MIT', + 'name': 'fedora-release', + 'release': '0.3', + 'sigmd5': 'cf31d87e5e3eac97ff32a98e7e073f37', + 'signature': None, + 'url': 'http://ftp.fi.muni.cz/pub/linux/fedora/linux' + '/development/rawhide/Everything/x86_64/os' + '/Packages/f/fedora-release-42-0.3.noarch.rpm', + 'version': '42', + }] + }, + "bootstrap": { + "image_digest": "sha256:ba1067bef190fbe88f085bd019464a8c0803b7cd1e3f", + "pull_digest": "sha256:1d9f0eaec60b59a669b285d1e775d970061b9694d4998b5bdb9626c9f33685cd", + "id": "b222730c2ba32385173fe3351026c316c9a294fa8d8c3c0f80d8afdb1b1aeef7", + "architecture": "amd64", + }, + 'config': { + 'bootstrap_image': 'foo', + 'bootstrap_image_ready': True, + "legal_host_arches": ["x86_64"], + "target_arch": "x86_64", + "dist": ".f42", + }, +} + + +def _exp_output_bootstrap(exp_data): + data = copy.deepcopy(exp_data["bootstrap"]) + del data["image_digest"] + return data + + +def _mock_vars(rpm_out, repoquery_out): + tc = TestCase() + tc.maxDiff = None + buildroot = MagicMock() + buildroot.state = MagicMock() + buildroot.uid_manager = MagicMock() + buildroot.doOutChroot = MagicMock( + side_effect=iter([ + (rpm_out, None), + (repoquery_out, None), + ]) + ) + buildroot.config = EXPECTED_OUTPUT['config'] + buildroot.resultdir = tempfile.mkdtemp(prefix="mock-test-buildroot-lock") + plugins = MagicMock() + plugins.add_hook = MagicMock() + return tc, buildroot, plugins + + +def _call_method(plugins, buildroot): + # initialize the plugin + init(plugins, {}, buildroot) + # obtain the hook method, and call it + plugins.add_hook.assert_called_once() + _, method = plugins.add_hook.call_args[0] + + podman_obj = MagicMock() + podman_obj.get_oci_digest.return_value = EXPECTED_OUTPUT["bootstrap"]["image_digest"] + podman_obj.inspect_hermetic_metadata.return_value = _exp_output_bootstrap(EXPECTED_OUTPUT) + podman_cls = MagicMock(return_value=podman_obj) + with patch("mockbuild.plugins.buildroot_lock.Podman", side_effect=podman_cls): + method() + + +def test_nonexisting_file_in_repo(): + """ + Test the situation when RPM is installed, and no longer available in + repository. + """ + _, buildroot, plugins = _mock_vars( + RPM_OUTPUT + RPM_ONLY_RPM, + REPOQUERY_OUTPUT, + ) + raised = False + try: + _call_method(plugins, buildroot) + except mockbuild.exception.Error as e: + assert e.msg == "Can't get location for cyrus-sasl-lib-2.1.27-21.el9.x86_64" + raised = True + assert raised + + +def _get_json_schema(): + testdir = os.path.dirname(__file__) + basename = "buildroot-lock-schema-" + EXPECTED_OUTPUT["version"] + ".json" + with open(os.path.join(testdir, "..", "docs", basename), + "r", encoding="utf-8") as fd: + return json.load(fd) + + +def test_buildroot_lock_output(): + """ test the buildroot_lock.json file format """ + tc, buildroot, plugins = _mock_vars(RPM_OUTPUT, REPOQUERY_OUTPUT) + _call_method(plugins, buildroot) + with open(os.path.join(buildroot.resultdir, "buildroot_lock.json"), "r", + encoding="utf-8") as fd: + data = json.load(fd) + tc.assertDictEqual(data, EXPECTED_OUTPUT) + schema = _get_json_schema() + jsonschema.validate(EXPECTED_OUTPUT, schema) + + +def test_json_schema_metadata(): + """ Test basic format of the json schema """ + schema = _get_json_schema() + version = EXPECTED_OUTPUT["version"] + assert "Version " + version + ";" in schema["description"] + assert "schema-" + version + ".json" in schema["$id"] diff --git a/mock/tests/test_config_loader.py b/mock/tests/test_config_loader.py new file mode 100644 index 0000000..7c6ea32 --- /dev/null +++ b/mock/tests/test_config_loader.py @@ -0,0 +1,52 @@ +""" +Tests for mockbuild.config +""" + +# pylint: disable=missing-class-docstring +# pylint: disable=missing-function-docstring +# pylint: disable=attribute-defined-outside-init + +import pwd +import os +import shutil +import tempfile + +from unittest import mock +from mockbuild.config import simple_load_config + + +class TestConfigLoader: + def setup_method(self): + testdir = os.path.dirname(os.path.realpath(__file__)) + self.configdir = os.path.join(testdir, "data", "config-001") + self.homedir = tempfile.mkdtemp(prefix='mock-test-home') + homedata = os.path.join(testdir, "data", "home-001") + self.username = pwd.getpwuid(os.getuid())[0] + userdir = os.path.join(self.homedir, self.username) + shutil.copytree(homedata, userdir) + + def test_config_paths(self): + with mock.patch("mockbuild.config.os.path.expanduser") as patch: + patch.side_effect = lambda x: x.replace("~", self.homedir + "/") + config = simple_load_config('fedora-rawhide-x86_64', self.configdir) + assert set(config["config_paths"]) == { + os.path.join(self.configdir, "site-defaults.cfg"), + os.path.join(self.configdir, "fedora-rawhide-x86_64.cfg"), + os.path.join(self.configdir, "templates", "fedora-rawhide.tpl"), + os.path.join(self.homedir, self.username, ".config", "mock.cfg"), + os.path.join(self.homedir, self.username, ".config", "mock", "fedora-rawhide-x86_64.cfg"), + } + + assert config["default"] is True + assert config["target_arch"] == 'x86_64' + assert config["template"] == "templated_value" + assert config["home_default"] == (1, 2) + + assert config["override_wins_site"] == "site" + assert config["override_wins_conf"] == "conf" + assert config["override_wins_home_site"] == "home_site" + assert config["override_wins_home_conf"] == "home_conf" + assert config["override_wins_template"] == "template" + + def teardown_method(self): + shutil.rmtree(self.homedir) diff --git a/mock/tests/test_config_templates.py b/mock/tests/test_config_templates.py new file mode 100644 index 0000000..f9f525b --- /dev/null +++ b/mock/tests/test_config_templates.py @@ -0,0 +1,127 @@ +import pytest +from templated_dictionary import TemplatedDictionary + + +def test_transitive_expand(): + config = TemplatedDictionary() + config['a'] = 'test' + config['b'] = '{{ a }} {{ a }}' + config['c'] = '{{ b + " " + b }}' + assert config['c'] == '{{ b + " " + b }}' + config['__jinja_expand'] = True + assert config['c'] == 'test test test test' + + +@pytest.mark.parametrize('setup', [ + # host target bootstrap expected_repo + ('x86_64', 'arm7hl', True, 'x86_64'), + ('x86_64', 'arm7hl', False, 'armhfp'), + ('ppc64le', 'arm7hl', True, 'ppc64le'), + ('ppc64le', 'arm7hl', False, 'armhfp'), +]) +def test_nested_access(setup): + """ + Check that we can access config_opts["foo"]["bar"] items in jinja. + """ + host, target, bootstrap, result = setup + config = TemplatedDictionary() + config["archmap"] = {"i386": "i686", "arm7hl": "armhfp", "x86_64": "x86_64"} + config["host_arch"] = host + config["target_arch"] = target + config["root"] = "foo-bootstrap" if bootstrap else "foo" + config["repo_arch"] = ( + "{% set desired = host_arch if root.endswith('bootstrap') else target_arch %}" + "{{ archmap[desired] if desired in archmap else desired }}" + ) + config['__jinja_expand'] = True + assert config['repo_arch'] == result + + +def test_aliases(): + config = TemplatedDictionary( + alias_spec={ + 'dnf.conf': ['yum.conf', 'package_manager.conf'], + }, + ) + + config['dnf.conf'] = "initial" + config['yum.conf'] += " appended" + + config['__jinja_expand'] = True + assert config['package_manager.conf'] == "initial appended" + config['__jinja_expand'] = False + + config['package_manager.conf'] = "replaced" + + config['__jinja_expand'] = True + assert config['dnf.conf'] == config['yum.conf'] == 'replaced' + + config['variable'] = "content" + config['package_manager.conf'] += " {{ variable }}" + + assert config['dnf.conf'] == config['yum.conf'] == 'replaced content' + + +@pytest.mark.xfail +def test_that_access_doesnt_affect_value(): + config = TemplatedDictionary() + config['a'] = {} + config['a']['b'] = '{{ b }}' + config['__jinja_expand'] = True + + # access it, and and destroy 'a' (shouldn't happen) + assert '' == config['a']['b'] + + # we set b, but it is not propagated to a.b because a.b was already + # accessed - and that rewrote a.b to ''. So even after setting b properly, + # a.b stays empty. + config['b'] = 'b' + assert 'b' == config['a']['b'] + + +def test_not_detected_recursion(): + config = TemplatedDictionary() + config['a'] = '{{ a }}' + config['b'] = '{{ a }}' + config['__jinja_expand'] = True + + # TODO: should this throw exception? We might hypotetically use this + # "problem" to assure that some values are unexpanded. + assert config['a'] == '{{ a }}' + assert config['b'] == '{{ a }}' + + +def test_too_deep_recursion(): + config = TemplatedDictionary() + config['a'] = '{{ b }}' + config['b'] = '[ {{ a }} ]' + config['__jinja_expand'] = True + with pytest.raises(ValueError): + # infinite recursion + config['a'] + + config = TemplatedDictionary() + config['a'] = '{{ b }}' + config['b'] = '{{ c }}' + config['c'] = '{{ d }}' + config['d'] = '{{ e }}' + config['e'] = '{{ f }}' + config['f'] = 'f' + config['g'] = 11 + config['__jinja_expand'] = True + + # this is not yet too deep + assert config['b'] == 'f' + + # but this is too deep + with pytest.raises(ValueError): + config['a'] + + +def test_many_newlines(): + # rhbz#1806482 + config = TemplatedDictionary() + string = "\n\n\n\n\n\na\n\n\n\n\n\n" + config['a'] = string + config['__jinja_expand'] = True + assert config['a'] == string diff --git a/mock/tests/test_installed_packages.py b/mock/tests/test_installed_packages.py new file mode 100644 index 0000000..ba9eb87 --- /dev/null +++ b/mock/tests/test_installed_packages.py @@ -0,0 +1,14 @@ +""" +Test the "installed_packages.py" file +""" + +from mockbuild.installed_packages import _subprocess_executor + + +def test_the_default_executor(): + """ + The expected executor output is just stderr + """ + assert "stdout\n" == _subprocess_executor([ + "/bin/sh", "-c", "echo stdout ; echo >&2 stderr" + ]) diff --git a/mock/tests/test_package_manager.py b/mock/tests/test_package_manager.py new file mode 100644 index 0000000..5eaed9f --- /dev/null +++ b/mock/tests/test_package_manager.py @@ -0,0 +1,192 @@ +import os +import tempfile +import shutil + +import pytest +from unittest import mock +from unittest.mock import MagicMock + +from templated_dictionary import TemplatedDictionary +from mockbuild.config import setup_default_config_opts +from mockbuild.constants import PKGPYTHONDIR +from mockbuild.buildroot import Buildroot +from mockbuild.package_manager import _PackageManager + + +class TestPackageManager: + + def setup_method(self, method): + self.workdir = tempfile.mkdtemp(prefix='mock-test') + + testdir = os.path.dirname(os.path.realpath(__file__)) + plugindir = os.path.join(testdir, '..', 'py', 'mockbuild') + plugindir = os.path.realpath(plugindir) + PKGPYTHONDIR = plugindir + + self.config_opts = setup_default_config_opts() + self.config_opts['root'] = 'distro-version-arch' + self.config_opts['basedir'] = self.workdir + self.config_opts["resultdir"] = "{{basedir}}/{{root}}/result" + self.config_opts['chroothome'] = '/builddir' + self.config_opts['chrootgid'] = '135' + self.config_opts['package_manager'] = 'dnf' + self.config_opts['releasever'] = '1' + self.config_opts['target_arch'] = 'fakearch' + self.config_opts['dnf_vars'] = {'test': 'testval'} + + with mock.patch('mockbuild.buildroot.package_manager'): + with mock.patch('mockbuild.util.cmpKernelVer') as kv: + kv.return_value = True + self.bootstrap_buildroot = Buildroot( + self.config_opts.copy(), + None, # state + MagicMock(), # state + MagicMock(), # plugins + None, # bootstrap_buildroot + True, # is_bootstrap + ) + + self.buildroot = Buildroot( + self.config_opts, + None, # uidManager + MagicMock(), # state + MagicMock(), # plugins + self.bootstrap_buildroot, + False, # is_bootstrap + ) + + self.package_manager = _PackageManager( + self.buildroot.config, + self.buildroot, + self.buildroot.plugins, + self.bootstrap_buildroot, + ) + + self.package_manager_bootstrap = _PackageManager( + self.bootstrap_buildroot.config, + self.bootstrap_buildroot, + self.bootstrap_buildroot.plugins, + self.buildroot.plugins, + ) + + def teardown_method(self, method): + shutil.rmtree(self.workdir) + + def get_user_bind_mounts_from_config(self, config): + pm = self.package_manager_bootstrap + pm.pkg_manager_config = config + pm.initialize_config() + pm._bind_mount_repos_to_bootstrap() + return self.bootstrap_buildroot.mounts.managed_mounts + + def test_absolute_path_name_in_baseurl(self): + repo_directory = os.path.join(self.workdir, 'repo') + os.mkdir(repo_directory) + config = """ + [main] + something = 1 + + [external] + baseurl = http://exmaple.com/test/ + + [external-urlencoded] + baseurl = http://example.com/results/%40fedora-llvm-team/ + + [fedora] + baseurl = {} + """.format(repo_directory) + mounts = self.get_user_bind_mounts_from_config(config) + assert len(mounts) == 1 + assert mounts[0].srcpath == repo_directory + assert mounts[0].bindpath.startswith(self.workdir) + assert mounts[0].bindpath.endswith(repo_directory) + + def test_file_colon_slash_path_name_in_baseurl(self): + repo_directory = os.path.join(self.workdir, 'repo') + os.mkdir(repo_directory) + config = """ + [main] + something = 1 + + [fedora] + baseurl = file://{} + """.format(repo_directory) + mounts = self.get_user_bind_mounts_from_config(config) + assert len(mounts) == 1 + assert mounts[0].srcpath == repo_directory + assert mounts[0].bindpath.startswith(self.workdir) + assert mounts[0].bindpath.endswith(repo_directory) + + def test_dir_doesnt_exist(self): + repo_directory = os.path.join(self.workdir, 'repo') + config = """ + [main] + something = 1 + + [fedora] + baseurl = file://{} + """.format(repo_directory) + mounts = self.get_user_bind_mounts_from_config(config) + assert len(mounts) == 0 + + @pytest.mark.parametrize('option', ['metalink', 'mirrorlist']) + @pytest.mark.parametrize('exists', [True, False]) + def test_local_metalink_mirrorlist(self, option, exists): + repo_directory = os.path.join(self.workdir, 'repo') + os.mkdir(repo_directory) + path = os.path.join(repo_directory, 'testfile') + + if exists: + with open(path, 'w') as f: + f.write("line\n") + + config = """ + [main] + metalink = file://{} + """.format(path) + mounts = self.get_user_bind_mounts_from_config(config) + + if not exists: + assert len(mounts) == 0 + return + + assert len(mounts) == 1 + assert mounts[0].srcpath == path + assert mounts[0].bindpath.startswith(self.workdir) + assert mounts[0].bindpath.endswith(path) + + def test_bindmount_baseurl_list_with_dupes(self): + for repo in ['alt1', 'alt2', 'alt3']: + repo_directory = os.path.join(self.workdir, repo) + os.mkdir(repo_directory) + + config = ( + "[main]\n" + "baseurl = file://{0}/alt2, {0}/alt1\n" + " file://{0}/alt3\n" + " file://{0}/alt2\n" # 2nd time + ).format(self.workdir) + + mounts = self.get_user_bind_mounts_from_config(config) + assert len(mounts) == 3 + assert {x.srcpath for x in mounts} == set([ + os.path.join(self.workdir, 'alt1'), + os.path.join(self.workdir, 'alt2'), + os.path.join(self.workdir, 'alt3'), + ]) + + def test_bindmount_expand_vars(self): + repo_directory = os.path.join(self.workdir, + self.config_opts['target_arch'], + self.config_opts['releasever'], + self.config_opts['dnf_vars']['test']) + os.makedirs(repo_directory) + config = ( + "[main]\n" + "baseurl = file://{0}/$basearch/${{releasever}}/$test\n" + ).format(self.workdir) + + mounts = self.get_user_bind_mounts_from_config(config) + assert len(mounts) == 1 + mount = mounts[0] + assert mount.srcpath == repo_directory diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..61632bd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,14 @@ +[tool.towncrier] +# we post-process this file with releng/generate-release-notes +filename = ".Release-Notes-Next.md" +directory = "releng/release-notes-next" +underlines = ["", "", ""] +template = "releng/release-notes-next/template.jinja" +title_format = "## [Release {version}](https://rpm-software-management.github.io/mock/Release-Notes-{version}) - {project_date}" +issue_format = "" +type = [ + { name = "Breaking changes", directory = "breaking", showcontent = true }, + { name = "New features", directory = "feature", showcontent = true }, + { name = "Bugfixes", directory = "bugfix", showcontent = true }, + { name = "Mock Core Configs changes", directory = "config", showcontent = true }, +] diff --git a/releng/.vcs-diff-lint.yml b/releng/.vcs-diff-lint.yml new file mode 100644 index 0000000..e69de29 diff --git a/releng/generate-release-notes b/releng/generate-release-notes new file mode 100755 index 0000000..af2ec2c --- /dev/null +++ b/releng/generate-release-notes @@ -0,0 +1,115 @@ +#! /usr/bin/python3 + +import argparse +import os +import re +import subprocess +import tempfile + +TRANSFORM = { + "rhbz": "https://bugzilla.redhat.com/{id}", + "issue": "https://github.com/rpm-software-management/mock/issues/{id}", + "PR": "https://github.com/rpm-software-management/mock/pull/{id}", + "commit": "https://github.com/rpm-software-management/mock/commit/{id}", + "copr_issue": "https://github.com/fedora-copr/copr/issues/{id}", +} + +TEMP_FILE = ".Release-Notes-Next.md" + +def _arg_parser(): + parser = argparse.ArgumentParser() + parser.add_argument("--use-version", dest="version") + group = parser.add_mutually_exclusive_group() + group.add_argument("--config-only", action="store_true") + group.add_argument("--mock-only", action="store_true") + return parser + +def _limited_config(opts): + if not opts.config_only and not opts.mock_only: + return [] + with tempfile.NamedTemporaryFile(delete=False, dir=".") as config_fd: + opts.temp_config = config_fd.name + with open("pyproject.toml", "rb") as fd: + for line in fd.readlines(): + if b'{ name' not in line: + config_fd.write(line) + continue + # skip non-config files + if opts.config_only and b"config" not in line: + continue + # skip config files + if opts.mock_only and b"config" in line: + continue + config_fd.write(line) + return ["--config", opts.temp_config] + + +def _main(): + options = _arg_parser().parse_args() + options.temp_config = None + tc_opts = _limited_config(options) + if not options.version: + options.version = "Next" + tc_opts.append("--keep") + else: + tc_opts.append("--yes") + + output_file = os.path.join(f"docs/Release-Notes-{options.version}.md") + + found = set() + pattern = "|".join(TRANSFORM.keys()) + issues_re = re.compile(r"\[(" + pattern + r")#([0-9a-f]+)\]") + + + cmd = ["towncrier", "build", "--version", options.version] + tc_opts + print(f"Calling {cmd}") + subprocess.check_call(cmd) + + with open(output_file, "w", encoding="utf8") as fdout: + + fdout.write(f"""--- +layout: default +title: Release Notes - Mock {options.version} +--- + +""") + + if options.version == "Next": + fdout.write("""\ +# Pre-release Mock changelog + +Every change in Mock or `mock-core-configs` that deserves some promotion needs +to be mentioned in release notes. This generated page provides a set of changes +that will land the future version of Mock. See [how to add +entries](Release-Notes-New-Entry). +""") + + + with open(TEMP_FILE, encoding="utf8") as fdorig: + for line in fdorig: + fdout.write(line) + findings = re.findall(issues_re, line) + if not findings: + continue + for finding in findings: + found.add(finding) + + fdout.write("\n") + for (key, value) in found: + fdout.write(f"[{key}#{value}]: {TRANSFORM[key].format(id=value)}\n") + + if options.version != "Next": + subprocess.check_call(["git", "add", output_file]) + + +if __name__ == "__main__": + # go top-level + os.chdir(os.path.join(os.path.dirname(__file__), '..')) + try: + _main() + finally: + try: + os.unlink(TEMP_FILE) + subprocess.check_call(["git", "rm", "-f", TEMP_FILE]) + except FileNotFoundError: + pass diff --git a/releng/rawhide-branching.sh b/releng/rawhide-branching.sh new file mode 100755 index 0000000..86314eb --- /dev/null +++ b/releng/rawhide-branching.sh @@ -0,0 +1,80 @@ +#! /bin/bash + +## Prepare templates for the next branched Fedora verasion +## +## E.g. when the Fedora Rawhide is going to be branched into Fedora 35, +## execute this script as './releng/rawhide-branching.sh' (without +## arguments). + +set -e +topdir=$(git rev-parse --show-toplevel) + +cd "$topdir/mock-core-configs/etc/mock" + +for config in fedora-??-x86_64.cfg; do + prev_version=$version + version=$(echo "$config" | sed -e 's/fedora-//' -e 's/-x86_64.*//') + next_version=$(( version + 1 )) +done + +rawhide_plus_one_version=$(( next_version + 1 )) +next_gpg=/usr/share/distribution-gpg-keys/fedora/RPM-GPG-KEY-fedora-$rawhide_plus_one_version-primary +test -e "$next_gpg" || { + cat >&2 < $version" +echo "Arches: ${architectures[*]}" + +for arch in "${architectures[@]}"; do + # drop the old rawhide symlink + rm "fedora-$version-$arch.cfg" + # copy old branched config to new branched config, and update releasever + cp fedora-{"$prev_version","$version"}-"$arch.cfg" + sed -i "s|'$prev_version'|'$version'|" "fedora-$version-$arch.cfg" + # create updated rawhide symlink + ln -s "fedora-rawhide-$arch.cfg" "fedora-$next_version-$arch.cfg" + # stash those updated configs + git add "fedora-$next_version-$arch.cfg" "fedora-$version-$arch.cfg" +done + +towncrier_file=$topdir/releng/release-notes-next/fedora-$version-branching.config +cat > "$towncrier_file" <=6.0 click>=8.0.0 -pyyaml>=6.0 -jinja2>=3.0.0 -requests>=2.25.0 \ No newline at end of file +rich>=13.0.0 + +# Testing dependencies +pytest>=7.0.0 +pytest-cov>=4.0.0 +pytest-mock>=3.10.0 + +# Development dependencies +black>=23.0.0 +flake8>=6.0.0 +mypy>=1.0.0 + +# Optional dependencies for advanced features +# These may be added as development progresses +# docker>=6.0.0 +# kubernetes>=26.0.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 99bdcc6..a5ba472 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 """ -Setup script for mock package +Setup script for deb-mock +Debian's equivalent to Fedora's Mock build environment manager """ from setuptools import setup, find_packages @@ -11,43 +12,57 @@ def read_readme(): with open("README.md", "r", encoding="utf-8") as fh: return fh.read() +# Read requirements from requirements.txt +def read_requirements(): + with open("requirements.txt", "r", encoding="utf-8") as fh: + return [line.strip() for line in fh if line.strip() and not line.startswith("#")] + setup( name="deb-mock", version="0.1.0", - author="Mock Team", - author_email="team@mock.org", - description="A low-level utility to create clean, isolated build environments for Debian packages", + author="Debian Bootc Ecosystem Team", + author_email="debian-bootc@lists.debian.org", + description="Debian's equivalent to Fedora's Mock build environment manager", long_description=read_readme(), long_description_content_type="text/markdown", - url="https://github.com/deb-mock/deb-mock", - packages=find_packages(), + url="https://github.com/debian/deb-bootc-compose", + project_urls={ + "Bug Tracker": "https://github.com/debian/deb-bootc-compose/issues", + "Documentation": "https://github.com/debian/deb-bootc-compose/wiki", + "Source Code": "https://github.com/debian/deb-bootc-compose", + }, classifiers=[ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", - "License :: OSI Approved :: MIT License", + "License :: OSI Approved :: GNU General Public License v2 or later (GPLv2+)", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Topic :: Software Development :: Build Tools", - "Topic :: System :: Archiving :: Packaging", - ], - python_requires=">=3.8", - install_requires=[ - "click>=8.0.0", - "pyyaml>=6.0", - "jinja2>=3.0.0", - "requests>=2.25.0", + "Topic :: System :: Systems Administration", ], + packages=find_packages(), + python_requires=">=3.9", + install_requires=read_requirements(), + extras_require={ + "dev": [ + "black>=23.0.0", + "flake8>=6.0.0", + "mypy>=1.0.0", + "pytest>=7.0.0", + "pytest-cov>=4.0.0", + "pytest-mock>=3.10.0", + ], + }, entry_points={ "console_scripts": [ - "mock=deb_mock.cli:main", + "deb-mock=deb_mock.cli:main", ], }, include_package_data=True, - package_data={ - "deb_mock": ["templates/*", "config/*"], - }, + zip_safe=False, + keywords="debian, mock, chroot, build, environment, packaging", + platforms=["Linux"], ) \ No newline at end of file diff --git a/testing-farm/plans/behave.fmf b/testing-farm/plans/behave.fmf new file mode 100644 index 0000000..8cadf02 --- /dev/null +++ b/testing-farm/plans/behave.fmf @@ -0,0 +1,12 @@ +--- +summary: run the behave tests in Fedora Testing Farm +discover: + - how: fmf + filter: "tag: behave" + +prepare: + - how: ansible + playbook: mock/integration-tests/setup-playbook/play-tf.yml + +execute: + - how: tmt diff --git a/testing-farm/plans/old-testsuite.fmf b/testing-farm/plans/old-testsuite.fmf new file mode 100644 index 0000000..7595c62 --- /dev/null +++ b/testing-farm/plans/old-testsuite.fmf @@ -0,0 +1,12 @@ +--- +summary: run the old testsuite in Fedora Testing Farm +discover: + - how: fmf + filter: "tag: old_testsuite" + +prepare: + - how: ansible + playbook: mock/integration-tests/setup-playbook/play-tf.yml + +execute: + - how: tmt diff --git a/testing-farm/tests/behave/main.fmf b/testing-farm/tests/behave/main.fmf new file mode 100644 index 0000000..9096918 --- /dev/null +++ b/testing-farm/tests/behave/main.fmf @@ -0,0 +1,9 @@ +--- +summary: Execute the behave test +test: ./test.sh +tag: behave +duration: 2h + +require: + - type: file + pattern: /behave diff --git a/testing-farm/tests/behave/test.sh b/testing-farm/tests/behave/test.sh new file mode 100755 index 0000000..c9a6824 --- /dev/null +++ b/testing-farm/tests/behave/test.sh @@ -0,0 +1,5 @@ +#!/bin/sh -eux + +cd ../../../behave +install-mock-packages-built-by-packit mock-core-configs mock +behave diff --git a/testing-farm/tests/old-testsuite/main.fmf b/testing-farm/tests/old-testsuite/main.fmf new file mode 100644 index 0000000..3291e83 --- /dev/null +++ b/testing-farm/tests/old-testsuite/main.fmf @@ -0,0 +1,11 @@ +--- +summary: Execute the old `make check` test-suite +test: ./test.sh +tag: old_testsuite +duration: 3h +tty: true + +require: + - type: file + # copy-paste the whole directory + pattern: / diff --git a/testing-farm/tests/old-testsuite/test.sh b/testing-farm/tests/old-testsuite/test.sh new file mode 100755 index 0000000..84b2e66 --- /dev/null +++ b/testing-farm/tests/old-testsuite/test.sh @@ -0,0 +1,28 @@ +#!/bin/bash -eux + +# The Mock's test-suite is designed for the mockbuild users. Copy files to a +# separate directory where the 'mockbuild' user has a full access. +workdir=$(mktemp -d --suffix=-mock-old-tests) +rsync -ra ../../../ "$workdir" +chown -R mockbuild:mockbuild "$workdir" + +# TODO: Mock should work with 'rw-------' files too. +# https://github.com/rpm-software-management/mock/issues/1300 +chmod a+r "$workdir/mock/integration-tests"/test-* + +# Install the tested RPMs +install-mock-packages-built-by-packit mock-core-configs mock + +# Download the tested SRPM +SRPM_DOWNLOAD_DIR=/tmp/mock-test-srpms install-mock-packages-built-by-packit mock + +cd "$workdir/mock" + +# We want to provide rather short live logs ASAP, hence that grep. Full logs +# are provided later no matter the exit status. +exit_status=0 +sudo -E -u mockbuild make check |& tee the-log | grep -e FAILED: -e PASSED: || { + exit_status=$? +} +cat the-log +exit $exit_status diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..a9eb0b2 --- /dev/null +++ b/tox.ini @@ -0,0 +1,17 @@ +[tox] +# sync with /.github/workflows/fedora-tox.yml +envlist = py{36,39,310,311,312,313} +skipsdist = True + +[testenv] +deps = + -rmock/requirements.txt + coverage + jsonschema + pytest + pytest-cov +setenv = + PYTHONPATH = ./mock/py +commands = + python -m pytest -v {posargs} --cov-report term-missing --cov-branch --cov mock/py mock/tests + python -m coverage report --fail-under=100 -m mock/py/mockbuild/plugins/rpmautospec.py