diff --git a/osbuild/util/linux.py b/osbuild/util/linux.py index 01d241bd..ce106b9a 100644 --- a/osbuild/util/linux.py +++ b/osbuild/util/linux.py @@ -18,9 +18,11 @@ import ctypes.util import fcntl import os import platform +import struct import threading __all__ = [ + "fcntl_flock", "ioctl_get_immutable", "ioctl_toggle_immutable", ] @@ -274,3 +276,142 @@ def cap_mask_to_set(mask: int) -> set: } return res + + +def fcntl_flock(fd: int, lock_type: int, wait: bool = False): + """Perform File-locking Operation + + This function performs a linux file-locking operation on the specified + file-descriptor. The specific type of lock must be specified by the caller. + This function does not allow to specify the byte-range of the file to lock. + Instead, it always applies the lock operations to the entire file. + + For system-level documentation, see the `fcntl(2)` man-page, especially the + section about `struct flock` and the locking commands. + + This function always uses the open-file-description locks provided by + modern linux kernels. This means, locks are tied to the + open-file-description. That is, they are shared between duplicated + file-descriptors. Furthermore, acquiring a lock while already holding a + lock will update the lock to the new specified lock type, rather than + acquiring a new lock. + + If `wait` is `False` a non-blocking operation is performed. In case the lock + is contested a `BlockingIOError` is raised by the python standard library. + If `Wait` is `True`, the kernel will suspend execution until the lock is + acquired. + + If a synchronous exception is raised, the operation will be canceled and the + exception is forwarded. + + Parameters + ---------- + fd + The file-descriptor to use for the locking operation. + lock_type + The type of lock to use. This can be one of: `fcntl.F_RDLCK`, + `fcntl.F_WRLCK`, `fcntl.F_UNLCK`. + wait + Whether to suspend execution until the lock is acquired in case of + contested locks. + + Raises + ------ + OSError + If the underlying `fcntl(2)` syscall fails, a matching `OSError` is + raised. In particular, `BlockingIOError` signals contested locks. The + POSIX error code is `EAGAIN`. + """ + + valid_types = [fcntl.F_RDLCK, fcntl.F_WRLCK, fcntl.F_UNLCK] + if lock_type not in valid_types: + raise ValueError("Unknown lock type") + if not isinstance(fd, int): + raise ValueError("File-descriptor is not an integer") + if fd < 0: + raise ValueError("File-descriptor is negative") + + # + # The `OFD` constants are not available through the `fcntl` module, so we + # need to use their integer representations directly. They are the same + # across all linux ABIs: + # + # F_OFD_GETLK = 36 + # F_OFD_SETLK = 37 + # F_OFD_SETLKW = 38 + # + + if wait: + lock_cmd = 38 + else: + lock_cmd = 37 + + # + # We use the linux open-file-descriptor (OFD) version of the POSIX file + # locking operations. They attach locks to an open file description, rather + # than to a process. They have clear, useful semantics. + # This means, we need to use the `fcntl(2)` operation with `struct flock`, + # which is rather unfortunate, since it varies depending on compiler + # arguments used for the python library, as well as depends on the host + # architecture, etc. + # + # The structure layout of the locking argument is: + # + # struct flock { + # short int l_type; + # short int l_whence; + # off_t l_start; + # off_t l_len; + # pid_t int l_pid; + # } + # + # The possible options for `l_whence` are `SEEK_SET`, `SEEK_CUR`, and + # `SEEK_END`. All are provided by the `fcntl` module. Same for the possible + # options for `l_type`, which are `L_RDLCK`, `L_WRLCK`, and `L_UNLCK`. + # + # Depending on which architecture you run on, but also depending on whether + # large-file mode was enabled to compile the python library, the values of + # the constants as well as the sizes of `off_t` can change. What we know is + # that `short int` is always 16-bit on linux, and we know that `fcntl(2)` + # does not take a `size` parameter. Therefore, the kernel will just fetch + # the structure from user-space with the correct size. The python wrapper + # `fcntl.fcntl()` always uses a 1024-bytes buffer and thus we can just pad + # our argument with trailing zeros to provide a valid argument to the + # kernel. Note that your libc might also do automatic translation to + # `fcntl64(2)` and `struct flock64` (if you run on 32bit machines with + # large-file support enabled). Also, random architectures change trailing + # padding of the structure (MIPS-ABI32 adds 128-byte trailing padding, + # SPARC adds 16?). + # + # To avoid all this mess, we use the fact that we only care for `l_type`. + # Everything else is always set to 0 in all our needed locking calls. + # Therefore, we simply use the largest possible `struct flock` for your + # libc and set everything to 0. The `l_type` field is guaranteed to be + # 16-bit, so it will have the correct offset, alignment, and endianness + # without us doing anything. Downside of all this is that all our locks + # always affect the entire file. However, we do not need locks for specific + # sub-regions of a file, so we should be fine. Eventually, what we end up + # with passing to libc is: + # + # struct flock { + # uint16_t l_type; + # uint16_t l_whence; + # uint32_t pad0; + # uint64_t pad1; + # uint64_t pad2; + # uint32_t pad3; + # uint32_t pad4; + # } + # + + type_flock64 = struct.Struct('=HHIQQII') + arg_flock64 = type_flock64.pack(lock_type, 0, 0, 0, 0, 0, 0) + + # + # Since python-3.5 (PEP475) the standard library guards around `EINTR` and + # automatically retries the operation. Hence, there is no need to retry + # waiting calls. If a python signal handler raises an exception, the + # operation is not retried and the exception is forwarded. + # + + fcntl.fcntl(fd, lock_cmd, arg_flock64) diff --git a/test/mod/test_util_linux.py b/test/mod/test_util_linux.py index 2838dc66..9f8a29c4 100644 --- a/test/mod/test_util_linux.py +++ b/test/mod/test_util_linux.py @@ -95,3 +95,73 @@ def test_capabilities(): assert not linux.cap_is_supported("CAP_GICMO") with pytest.raises(OSError): lib.from_name("CAP_GICMO") + + +def test_fcntl_flock(): + # + # This tests the `linux.fcntl_flock()` file-locking helper. Note + # that file-locks are on the open-file-description, so they are shared + # between dupped file-descriptors. We explicitly create a separate + # file-description via `/proc/self/fd/`. + # + + with tempfile.TemporaryFile() as f: + fd1 = f.fileno() + fd2 = os.open(os.path.join("/proc/self/fd/", str(fd1)), os.O_RDWR | os.O_CLOEXEC) + + # Test: unlock + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + + # Test: write-lock + unlock + linux.fcntl_flock(fd1, linux.fcntl.F_WRLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + + # Test: read-lock1 + read-lock2 + unlock1 + unlock2 + linux.fcntl_flock(fd1, linux.fcntl.F_RDLCK) + linux.fcntl_flock(fd2, linux.fcntl.F_RDLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + linux.fcntl_flock(fd2, linux.fcntl.F_UNLCK) + + # Test: write-lock1 + write-lock2 + unlock + linux.fcntl_flock(fd1, linux.fcntl.F_WRLCK) + with pytest.raises(BlockingIOError): + linux.fcntl_flock(fd2, linux.fcntl.F_WRLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + + # Test: write-lock1 + read-lock2 + unlock + linux.fcntl_flock(fd1, linux.fcntl.F_WRLCK) + with pytest.raises(BlockingIOError): + linux.fcntl_flock(fd2, linux.fcntl.F_RDLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + + # Test: read-lock1 + write-lock2 + unlock + linux.fcntl_flock(fd1, linux.fcntl.F_RDLCK) + with pytest.raises(BlockingIOError): + linux.fcntl_flock(fd2, linux.fcntl.F_WRLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + + # Test: write-lock1 + read-lock1 + read-lock2 + unlock + linux.fcntl_flock(fd1, linux.fcntl.F_WRLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_RDLCK) + linux.fcntl_flock(fd2, linux.fcntl.F_RDLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + + # Test: read-lock1 + read-lock2 + write-lock1 + unlock1 + unlock2 + linux.fcntl_flock(fd1, linux.fcntl.F_RDLCK) + linux.fcntl_flock(fd2, linux.fcntl.F_RDLCK) + with pytest.raises(BlockingIOError): + linux.fcntl_flock(fd1, linux.fcntl.F_WRLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + linux.fcntl_flock(fd2, linux.fcntl.F_UNLCK) + + # Test: write-lock3 + write-lock1 + close3 + write-lock1 + unlock1 + fd3 = os.open(os.path.join("/proc/self/fd/", str(fd1)), os.O_RDWR | os.O_CLOEXEC) + linux.fcntl_flock(fd3, linux.fcntl.F_WRLCK) + with pytest.raises(BlockingIOError): + linux.fcntl_flock(fd1, linux.fcntl.F_WRLCK) + os.close(fd3) + linux.fcntl_flock(fd1, linux.fcntl.F_WRLCK) + linux.fcntl_flock(fd1, linux.fcntl.F_UNLCK) + + # Cleanup + os.close(fd2)