particle-os-tools/docs/apt-layer/rpm-ostree/contributing/debug.md

Debugging rpm-ostree

Overview

This guide covers debugging techniques and tools for troubleshooting rpm-ostree issues. It includes setting up debug environments, using debugging tools, and solving common problems.

Debug Environment Setup

Enabling Debug Output

Environment Variables

# Enable all debug messages
export G_MESSAGES_DEBUG=all
export RUST_LOG=debug
export RPMOSTREE_DEBUG=1

# Enable specific debug domains
export G_MESSAGES_DEBUG=rpmostree
export RUST_LOG=rpmostree=debug

# Enable verbose output
export RPMOSTREE_VERBOSE=1

Command Line Options

# Enable verbose output
rpm-ostree --verbose status

# Enable debug output
rpm-ostree --debug status

# Enable trace output
rpm-ostree --trace status

Debug Configuration

Daemon Debug Configuration

# /etc/rpm-ostree/rpm-ostreed.conf
[daemon]
debug=true
verbose=true
log-level=debug

Client Debug Configuration

# ~/.config/rpm-ostree/config
[client]
debug=true
verbose=true
log-level=debug

Debugging Tools

GDB (GNU Debugger)

Basic GDB Usage

# Debug rpm-ostree binary
gdb --args rpm-ostree status

# Debug daemon process
gdb -p $(pidof rpm-ostreed)

# Attach to running process
gdb attach $(pidof rpm-ostreed)

GDB Commands

# Set breakpoints
(gdb) break rpmostree_core_new
(gdb) break rpmostree_daemon_handle_upgrade

# Run program
(gdb) run

# Continue execution
(gdb) continue

# Step through code
(gdb) next
(gdb) step

# Examine variables
(gdb) print variable_name
(gdb) print *pointer_variable

# Examine call stack
(gdb) bt
(gdb) bt full

# Examine memory
(gdb) x/10x pointer_address
(gdb) x/s string_address

GDB Scripts

# Create GDB script
cat > debug_rpm_ostree.gdb << EOF
set pagination off
set logging file debug.log
set logging on

break rpmostree_core_new
commands
  print *core
  continue
end

run status
EOF

# Run with script
gdb -x debug_rpm_ostree.gdb --args rpm-ostree status

Valgrind

Memory Leak Detection

# Check for memory leaks
valgrind --leak-check=full \
         --show-leak-kinds=all \
         --track-origins=yes \
         --verbose \
         --log-file=valgrind.log \
         rpm-ostree status

Memory Error Detection

# Check for memory errors
valgrind --tool=memcheck \
         --track-origins=yes \
         --verbose \
         --log-file=valgrind.log \
         rpm-ostree status

Callgrind Profiling

# Profile function calls
valgrind --tool=callgrind \
         --callgrind-out-file=callgrind.out \
         rpm-ostree upgrade

# Analyze results
callgrind_annotate callgrind.out

strace

System Call Tracing

# Trace all system calls
strace -f -o strace.log rpm-ostree status

# Trace specific system calls
strace -e trace=file,network -f rpm-ostree status

# Trace with timestamps
strace -t -f -o strace.log rpm-ostree status

# Trace with relative timestamps
strace -r -f -o strace.log rpm-ostree status

Network Call Tracing

# Trace network operations
strace -e trace=network -f rpm-ostree upgrade

# Trace specific network calls
strace -e trace=connect,accept,sendto,recvfrom -f rpm-ostree upgrade

perf

Performance Profiling

# Profile CPU usage
perf record -g rpm-ostree upgrade
perf report

# Profile specific process
perf record -g -p $(pidof rpm-ostreed)
perf report

# Profile with call graph
perf record -g --call-graph=dwarf rpm-ostree upgrade
perf report --call-graph

Event Analysis

# Analyze system events
perf record -e sched:* rpm-ostree upgrade
perf report

# Analyze cache misses
perf record -e cache-misses rpm-ostree upgrade
perf report

Debugging Specific Components

Daemon Debugging

Daemon Logs

# View daemon logs
journalctl -u rpm-ostreed -f

# View daemon logs with debug
journalctl -u rpm-ostreed -f --grep="DEBUG"

# View recent daemon logs
journalctl -u rpm-ostreed --since="10 minutes ago"

Daemon Process Debugging

# Debug daemon startup
gdb --args /usr/bin/rpm-ostreed --debug

# Attach to running daemon
gdb -p $(pidof rpm-ostreed)

# Check daemon status
systemctl status rpm-ostreed

D-Bus Debugging

# Monitor D-Bus messages
dbus-monitor --system

# Monitor specific D-Bus interface
dbus-monitor --system "interface=org.projectatomic.rpmostree1"

# Test D-Bus methods
gdbus call --system \
    --dest=org.projectatomic.rpmostree1 \
    --object-path=/org/projectatomic/rpmostree1 \
    --method=org.projectatomic.rpmostree1.OS.Status

Package Management Debugging

Package Resolution

# Debug package resolution
rpm-ostree --debug db diff $commit1 $commit2

# Check package conflicts
rpm-ostree --debug install --dry-run package-name

# Debug dependency resolution
rpm-ostree --debug override replace package-name

Package Installation

# Debug package installation
rpm-ostree --debug install package-name

# Check installation logs
journalctl -u rpm-ostreed --grep="package"

# Debug package scripts
rpm-ostree --debug install --apply-live package-name

OSTree Debugging

Repository Debugging

# Check repository status
ostree fsck --repo=/ostree/repo

# Debug repository operations
ostree --verbose log $commit

# Check repository objects
ostree ls $commit

# Debug repository pull
ostree --verbose pull $remote $ref

Commit Debugging

# Examine commit metadata
ostree show $commit

# Debug commit differences
ostree diff $commit1 $commit2

# Check commit files
ostree ls -R $commit

Common Debugging Scenarios

Upgrade Failures

Debug Upgrade Process

# Enable debug output for upgrade
export G_MESSAGES_DEBUG=all
export RUST_LOG=debug
rpm-ostree --debug upgrade

# Check upgrade logs
journalctl -u rpm-ostreed --since="5 minutes ago"

# Debug specific upgrade step
rpm-ostree --debug upgrade --dry-run

Network Issues

# Check network connectivity
curl -I https://ostree.example.com/repo/

# Debug network operations
strace -e trace=network -f rpm-ostree upgrade

# Check DNS resolution
nslookup ostree.example.com

Package Installation Issues

Dependency Conflicts

# Check package conflicts
rpm-ostree --debug install --dry-run package-name

# Debug dependency resolution
rpm-ostree --debug db diff $current $pending

# Check package metadata
rpm-ostree --debug db list --user

Script Execution Issues

# Debug package scripts
rpm-ostree --debug install --apply-live package-name

# Check script logs
journalctl -u rpm-ostreed --grep="script"

# Debug script environment
rpm-ostree --debug install --verbose package-name

Boot Issues

Bootloader Problems

# Check bootloader configuration
ls -la /boot/loader/entries/

# Debug bootloader update
rpm-ostree --debug kargs --append="debug"

# Check bootloader logs
journalctl -u systemd-boot

Kernel Issues

# Debug kernel arguments
rpm-ostree --debug kargs

# Check kernel modules
rpm-ostree --debug db list --user | grep kernel

# Debug initramfs
rpm-ostree --debug reload

Debug Output Analysis

Log Analysis

Parsing Debug Logs

# Extract error messages
grep -i "error" debug.log

# Extract warning messages
grep -i "warning" debug.log

# Extract debug messages
grep -i "debug" debug.log

# Extract specific component logs
grep "rpmostree" debug.log

Performance Analysis

# Analyze timing information
grep "time" debug.log | awk '{print $1, $2, $NF}'

# Analyze memory usage
grep "memory" debug.log

# Analyze network operations
grep "network" debug.log

Core Dump Analysis

Enabling Core Dumps

# Enable core dumps
ulimit -c unlimited

# Set core dump location
echo "/tmp/core.%e.%p" | sudo tee /proc/sys/kernel/core_pattern

# Run command that might crash
rpm-ostree status

Analyzing Core Dumps

# Analyze core dump
gdb rpm-ostree core.rpm-ostree.12345

# Get backtrace
(gdb) bt

# Get full backtrace
(gdb) bt full

# Examine registers
(gdb) info registers

Debugging Best Practices

Systematic Approach

1. Reproduce the issue: Ensure consistent reproduction
2. Isolate the problem: Narrow down the scope
3. Gather information: Collect logs and debug output
4. Analyze the data: Look for patterns and clues
5. Test solutions: Verify fixes work

Documentation

1. Record steps: Document reproduction steps
2. Save logs: Preserve debug output and logs
3. Note environment: Document system configuration
4. Track changes: Record what was tried and results

Tools Selection

1. Start simple: Use basic tools first
2. Add complexity: Use advanced tools as needed
3. Combine tools: Use multiple tools together
4. Validate results: Cross-check with different tools

---

Effective debugging requires systematic approach, appropriate tools, and thorough analysis. This guide provides the foundation for troubleshooting rpm-ostree issues.