commit f0dba7cc0a36276f932559b487bf8957656b2217 Author: robojerk Date: Wed Jun 18 21:54:06 2025 -0700 Initial commit: ComposeSync - Automated Docker Compose update agent diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e38b264 --- /dev/null +++ b/.gitignore @@ -0,0 +1,50 @@ +# OS generated files +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Editor files +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# Logs +*.log +logs/ + +# Environment files (user-specific) +.env +.env.local +.env.*.local + +# Temporary files +*.tmp +*.temp +*.bak + +# Backup directories +backups/ + +# Lock files +.lock/ + +# Test files +test/ +tests/ + +# Documentation builds +docs/_build/ +site/ + +# Package files +*.tar.gz +*.zip + +# Docker +.dockerignore \ No newline at end of file diff --git a/Docs/configuration.md b/Docs/configuration.md new file mode 100644 index 0000000..8371692 --- /dev/null +++ b/Docs/configuration.md @@ -0,0 +1,79 @@ +# Configuration Reference + +This guide covers all configuration options available in ComposeSync. + +## Configuration Files + +The service can be configured through: + +1. **Systemd Service File** (`/etc/systemd/system/composesync.service`): + - Basic service configuration + - User and group settings + - Working directory + +2. **Environment File** (`/opt/composesync/.env`): + - Base directory for stacks + - Global settings + - Stack-specific configurations + +3. **Stack Directories** (`/opt/composesync/stacks//`): + - `docker-compose.yml` (managed by agent) + - `docker-compose.override.yml` (your customizations) + - Versioned copies with `.bak` extension + +## Global Configuration Options + +```env +# Base directory for all stacks +COMPOSESYNC_BASE_DIR=/opt/composesync/stacks + +# Number of versions to keep per stack (default: 10) +KEEP_VERSIONS=10 + +# Update interval in seconds (default: 3600) +UPDATE_INTERVAL_SECONDS=3600 + +# Update mode: notify_only or notify_and_apply (default: notify_and_apply) +UPDATE_MODE=notify_and_apply + +# Enable dry-run mode (true/false, default: false) +DRY_RUN=false + +# Number of stacks to manage +STACKS=1 + +# Optional webhook URL for notifications +NOTIFICATION_WEBHOOK_URL=https://your-webhook-url.com/endpoint +``` + +## Stack Configuration Options + +For each stack, you can configure: + +```env +# Basic stack configuration +STACK_N_NAME=stack_name # Required: Name of the stack +STACK_N_URL=https://example.com/compose.yml # Required: URL to download from +STACK_N_PATH=/path/to/stack # Required: Local path for the stack +STACK_N_TOOL=wget # Required: Download tool (wget or git) +STACK_N_INTERVAL=86400 # Optional: Stack-specific interval (overrides global) +STACK_N_KEEP_VERSIONS=10 # Optional: Stack-specific version count (overrides global) + +# Git-specific options (only when STACK_N_TOOL=git) +STACK_N_GIT_SUBPATH=docker/compose.yml # Optional: Path to compose file in repo +STACK_N_GIT_REF=main # Optional: Branch or tag to checkout + +# Multiple compose files (numbered) +STACK_N_EXTRA_FILES_1=https://example.com/file1.yml +STACK_N_EXTRA_FILES_2=https://example.com/file2.yml +# ... continue numbering as needed + +# Custom file ordering (optional, comma-separated list of numbers) +STACK_N_EXTRA_FILES_ORDER=2,1,3 +``` + +If `STACK_N_EXTRA_FILES_ORDER` is set, extra files will be processed in the specified order (e.g., 2,1,3). Otherwise, files are processed in numeric order (1,2,3,...). + +## Update Modes + +- **`notify_only` \ No newline at end of file diff --git a/Docs/dry-run.md b/Docs/dry-run.md new file mode 100644 index 0000000..8ec6f1e --- /dev/null +++ b/Docs/dry-run.md @@ -0,0 +1,204 @@ +# Dry-Run Mode + +This guide covers how to use dry-run mode to test your ComposeSync configuration safely. + +## Overview + +Dry-run mode allows you to test your configuration without actually applying changes. This is useful for: +- Testing new stack configurations +- Verifying download URLs work correctly +- Previewing what changes would be applied +- Debugging configuration issues +- Testing webhook notifications + +## Enabling Dry-Run Mode + +To enable dry-run mode, add this to your `.env` file: + +```env +DRY_RUN=true +``` + +## What Dry-Run Mode Does + +In dry-run mode, ComposeSync will: +- Download and process all files normally +- Show what actions would be taken +- Display a preview of changes (diff output) +- Not create backups or versioned files +- Not apply any changes to your stacks +- Prefix all log messages with `[DRY-RUN]` +- Send webhook notifications (if configured) + +## Example Dry-Run Output + +``` +[2024-01-15 10:30:00] [DRY-RUN] Processing stack immich +[2024-01-15 10:30:01] [DRY-RUN] Downloading https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml to /opt/composesync/stacks/immich/docker-compose.yml +[2024-01-15 10:30:02] [DRY-RUN] Changes detected in immich +[2024-01-15 10:30:02] [DRY-RUN] DRY-RUN: Would apply changes to immich +[2024-01-15 10:30:02] [DRY-RUN] DRY-RUN: Would run: docker compose -f /opt/composesync/stacks/immich/docker-compose.yml -f /opt/composesync/stacks/immich/hwaccel.ml.yml -f /opt/composesync/stacks/immich/docker-compose.override.yml up -d +[2024-01-15 10:30:02] [DRY-RUN] DRY-RUN: Changes that would be applied: +[2024-01-15 10:30:02] [DRY-RUN] --- compose-20240115103000.yml.bak +[2024-01-15 10:30:02] [DRY-RUN] +++ docker-compose.yml +[2024-01-15 10:30:02] [DRY-RUN] @@ -1,6 +1,6 @@ +[2024-01-15 10:30:02] [DRY-RUN] version: '3.8' +[2024-01-15 10:30:02] [DRY-RUN] +[2024-01-15 10:30:02] [DRY-RUN] services: +[2024-01-15 10:30:02] [DRY-RUN] immich-server: +[2024-01-15 10:30:02] [DRY-RUN] - image: ghcr.io/immich-app/immich-server:release +[2024-01-15 10:30:02] [DRY-RUN] + image: ghcr.io/immich-app/immich-server:release-1.91.0 +``` + +## Testing New Configurations + +### 1. Test a New Stack + +1. Add a new stack configuration to your `.env` file +2. Enable dry-run mode: + ```env + DRY_RUN=true + ``` +3. Restart the service: + ```bash + sudo systemctl restart composesync + ``` +4. Check the logs to see what would happen: + ```bash + sudo journalctl -u composesync -f + ``` + +### 2. Test URL Changes + +1. Modify a stack URL in your `.env` file +2. Enable dry-run mode +3. Restart the service +4. Verify the new URL works and downloads correctly + +### 3. Test Multiple Compose Files + +1. Add extra compose files to a stack configuration +2. Enable dry-run mode +3. Restart the service +4. Verify all files are downloaded and processed correctly + +## Disabling Dry-Run Mode + +To disable dry-run mode, set `DRY_RUN=false` or remove the line from your `.env` file: + +```env +DRY_RUN=false +``` + +Then restart the service: +```bash +sudo systemctl restart composesync +``` + +## Webhook Testing + +Dry-run mode is particularly useful for testing webhook notifications because: +- Webhooks are still sent in dry-run mode +- You can verify your webhook configuration works +- No actual changes are applied to your stacks +- You can test different webhook services safely + +## Best Practices + +### 1. Always Test New Configurations + +Before adding a new stack or changing existing configurations: +1. Enable dry-run mode +2. Test the configuration +3. Verify everything works as expected +4. Disable dry-run mode when ready + +### 2. Use for Debugging + +When troubleshooting issues: +1. Enable dry-run mode +2. Check the logs for detailed information +3. Verify URLs and configurations +4. Fix any issues before disabling dry-run mode + +### 3. Test Webhook Integration + +Use dry-run mode to test webhook notifications: +1. Configure your webhook URL +2. Enable dry-run mode +3. Trigger an update cycle +4. Verify webhook notifications are received + +### 4. Preview Changes + +Use dry-run mode to preview what changes would be applied: +1. Enable dry-run mode +2. Let the service run a cycle +3. Review the diff output +4. Decide if you want to apply the changes + +## Troubleshooting + +### No Changes Detected + +If dry-run mode shows "No changes detected": +- The remote file is identical to your current file +- This is normal and expected behavior +- No action would be taken in normal mode either + +### Download Failures + +If you see download errors in dry-run mode: +- Check the URL is correct and accessible +- Verify network connectivity +- Check for authentication requirements +- Fix the URL before disabling dry-run mode + +### Configuration Errors + +If you see configuration errors: +- Check your `.env` file syntax +- Verify all required fields are present +- Check file permissions +- Fix configuration issues before proceeding + +## Example Workflow + +Here's a typical workflow for testing a new stack: + +1. **Add Configuration** + ```env + STACKS=2 + STACK_2_NAME=test-app + STACK_2_URL=https://example.com/docker-compose.yml + STACK_2_PATH=/opt/composesync/stacks/test-app + STACK_2_TOOL=wget + ``` + +2. **Enable Dry-Run Mode** + ```env + DRY_RUN=true + ``` + +3. **Restart Service** + ```bash + sudo systemctl restart composesync + ``` + +4. **Check Logs** + ```bash + sudo journalctl -u composesync -f + ``` + +5. **Verify Results** + - Check that files are downloaded + - Verify no errors occur + - Review any changes that would be applied + +6. **Disable Dry-Run Mode** + ```env + DRY_RUN=false + sudo systemctl restart composesync + ``` + +This workflow ensures your configuration is correct before applying any changes to your running stacks. \ No newline at end of file diff --git a/Docs/git-repositories.md b/Docs/git-repositories.md new file mode 100644 index 0000000..7c85e68 --- /dev/null +++ b/Docs/git-repositories.md @@ -0,0 +1,193 @@ +# Git Repository Setup + +This guide covers how to use Git repositories as sources for your Docker Compose files. + +## Basic Git Configuration + +To use a Git repository as your source, set the tool to `git` and provide the repository URL: + +```env +STACK_1_NAME=myapp +STACK_1_URL=https://github.com/org/repo.git +STACK_1_PATH=/opt/composesync/stacks/myapp +STACK_1_TOOL=git +``` + +## Git-Specific Options + +### Subpath Configuration + +If your `docker-compose.yml` file is not in the root of the repository, specify the subpath: + +```env +STACK_1_GIT_SUBPATH=docker/docker-compose.yml +``` + +This is useful for repositories that organize their Docker Compose files in subdirectories. + +### Branch and Tag Support + +You can specify a specific branch or tag to checkout: + +```env +STACK_1_GIT_REF=main +``` + +Examples: +- `STACK_1_GIT_REF=main` - Use the main branch +- `STACK_1_GIT_REF=v1.0.0` - Use a specific tag +- `STACK_1_GIT_REF=develop` - Use a development branch + +## Complete Example + +Here's a complete example of configuring a Git repository: + +```env +STACK_1_NAME=portainer +STACK_1_URL=https://github.com/portainer/portainer-compose.git +STACK_1_PATH=/opt/composesync/stacks/portainer +STACK_1_TOOL=git +STACK_1_GIT_SUBPATH=docker-compose.yml +STACK_1_GIT_REF=main +STACK_1_INTERVAL=43200 +STACK_1_KEEP_VERSIONS=5 +``` + +## How Git Sources Work + +When using Git as a source, ComposeSync: + +1. **Clones the repository** (if not already present) +2. **Fetches updates** from the remote repository +3. **Checks out the specified branch/tag** (if configured) +4. **Extracts the compose file** from the specified subpath (or root) +5. **Uses Git commit hash** as the version identifier + +## Version Identifiers + +For Git sources, ComposeSync uses the Git commit hash as the version identifier: + +``` +compose-a1b2c3d.yml.bak # Git commit hash +``` + +This provides precise version tracking and makes it easy to identify which commit a version came from. + +## Repository Management + +### Local Repository Storage + +Git repositories are stored locally in `.git` directories: + +``` +/opt/composesync/stacks/myapp/ +├── docker-compose.yml +├── docker-compose.override.yml +├── compose-a1b2c3d.yml.bak +└── myapp.git/ # Git repository +``` + +### Repository Updates + +ComposeSync automatically: +- Clones new repositories when first encountered +- Fetches updates from existing repositories +- Handles branch/tag changes +- Maintains local repository state + +## Best Practices + +### 1. Use Specific Branches/Tags + +For production environments, pin to specific branches or tags for stability: + +```env +STACK_1_GIT_REF=v2.1.0 # Pin to specific version +``` + +### 2. Use Subpaths for Organization + +Organize your repositories with subpaths for better structure: + +``` +repository/ +├── docker/ +│ ├── docker-compose.yml +│ └── docker-compose.prod.yml +├── docs/ +└── README.md +``` + +### 3. Monitor Repository Changes + +Use webhook notifications to monitor when repositories are updated: + +```env +NOTIFICATION_WEBHOOK_URL=https://your-webhook-url.com/endpoint +``` + +### 4. Test with Dry-Run Mode + +Always test new Git configurations with dry-run mode: + +```env +DRY_RUN=true +``` + +## Troubleshooting Git Issues + +### Repository Not Found + +If you get "Failed to clone" errors: +- Verify the repository URL is correct +- Ensure the repository is public or you have access +- Check network connectivity + +### Subpath Not Found + +If you get "Subpath not found" errors: +- Verify the subpath exists in the repository +- Check the branch/tag contains the file +- Use the correct path separator (forward slashes) + +### Branch/Tag Issues + +If you get "Failed to checkout" errors: +- Verify the branch/tag exists +- Check for typos in the reference name +- Ensure the reference is accessible + +## Example Configurations + +### Simple Repository (Root Compose File) + +```env +STACK_1_NAME=simple-app +STACK_1_URL=https://github.com/user/simple-app.git +STACK_1_PATH=/opt/composesync/stacks/simple-app +STACK_1_TOOL=git +STACK_1_GIT_REF=main +``` + +### Complex Repository (Subpath + Tag) + +```env +STACK_1_NAME=complex-app +STACK_1_URL=https://github.com/org/complex-app.git +STACK_1_PATH=/opt/composesync/stacks/complex-app +STACK_1_TOOL=git +STACK_1_GIT_SUBPATH=docker/compose/production.yml +STACK_1_GIT_REF=v1.2.3 +``` + +### Development Branch + +```env +STACK_1_NAME=dev-app +STACK_1_URL=https://github.com/user/dev-app.git +STACK_1_PATH=/opt/composesync/stacks/dev-app +STACK_1_TOOL=git +STACK_1_GIT_SUBPATH=docker/docker-compose.yml +STACK_1_GIT_REF=develop +STACK_1_INTERVAL=1800 # Check every 30 minutes for dev +``` \ No newline at end of file diff --git a/Docs/installation.md b/Docs/installation.md new file mode 100644 index 0000000..7c022d8 --- /dev/null +++ b/Docs/installation.md @@ -0,0 +1,132 @@ +# Installation Guide + +This guide covers the complete installation process for ComposeSync. + +## Deployment Model + +ComposeSync is designed to run **directly on the host system** as a systemd service, not in a container. This provides: +- Direct access to the Docker daemon via `/var/run/docker.sock` +- Better performance and reliability +- Standard systemd service management +- Native file system access for backups and versioning + +## Security Considerations + +Since ComposeSync runs with Docker group privileges, ensure: +- Only trusted users have access to the ComposeSync configuration +- The service user is properly restricted +- Regular security updates are applied +- Monitor logs for any suspicious activity + +## Prerequisites + +Make sure these are installed on your host system: + +* **Docker Engine** and **Docker Compose Plugin** (ensure `docker` and `docker compose` commands work) +* **Basic Utilities:** `wget`, `git`, `bash`, `curl`, `diffutils` + ```bash + # On Debian/Ubuntu: + sudo apt install wget git bash curl diffutils + ``` +* **User in Docker Group:** The user you specify during installation must be in the `docker` group + ```bash + sudo usermod -aG docker YOUR_USERNAME + # Then log out and back in + ``` + +## Installation Steps + +1. **Clone this repository:** + ```bash + git clone + cd composesync + ``` + +2. **Run the installation script as root:** + ```bash + sudo ./install.sh + ``` + The script will: + - Ask for the username to run ComposeSync + - Verify the user exists and is in the docker group + - Create necessary directories and set permissions + - Set up the systemd service + - Create a default configuration + +3. **Configure your stacks in `/opt/composesync/.env`:** + ```bash + sudo nano /opt/composesync/.env + ``` + Example configuration: + ```env + # Base directory for stacks + COMPOSESYNC_BASE_DIR=/opt/composesync/stacks + + # Number of versions to keep (default: 10) + KEEP_VERSIONS=10 + + # Update interval in seconds (default: 3600) + UPDATE_INTERVAL_SECONDS=3600 + + # Update mode (notify_only or notify_and_apply) + UPDATE_MODE=notify_and_apply + + # Stack configurations + STACKS=1 + + STACK_1_NAME=immich + STACK_1_URL=https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml + STACK_1_PATH=/opt/composesync/stacks/immich + STACK_1_TOOL=wget + STACK_1_INTERVAL=86400 + STACK_1_KEEP_VERSIONS=10 + ``` + +4. **Create stack directories:** + ```bash + sudo mkdir -p /opt/composesync/stacks/immich + sudo chown YOUR_USERNAME:docker /opt/composesync/stacks/immich + ``` + +5. **Restart the service to apply changes:** + ```bash + sudo systemctl restart composesync + ``` + +## How It Works + +ComposeSync runs as a systemd service and monitors your configured stacks for updates. When changes are detected, it: + +1. Creates a backup of the current configuration +2. Downloads the new configuration +3. Applies your custom overrides +4. Updates the stack if in `notify_and_apply` mode +5. Maintains versioned copies of configurations + +## Directory Structure + +``` +/opt/composesync/ +├── update-agent.sh +├── .env +└── stacks/ + ├── immich/ + │ ├── docker-compose.yml + │ ├── docker-compose.override.yml + │ ├── compose-*.yml.bak + │ └── backups/ + └── portainer/ + ├── docker-compose.yml + ├── docker-compose.override.yml + ├── compose-*.yml.bak + └── backups/ +``` + +## Next Steps + +After installation, you should: + +1. **[Configure your stacks](configuration.md)** - Set up your Docker Compose stacks +2. **[Create override files](multi-stack.md#creating-override-files)** - Add your customizations +3. **[Test with dry-run mode](dry-run.md)** - Verify your configuration +4. **[Set up monitoring](monitoring.md)** - Monitor the service logs \ No newline at end of file diff --git a/Docs/monitoring.md b/Docs/monitoring.md new file mode 100644 index 0000000..c42ea97 --- /dev/null +++ b/Docs/monitoring.md @@ -0,0 +1,351 @@ +# Monitoring Guide + +This guide covers how to monitor and manage your ComposeSync service. + +## Viewing Logs + +### Real-time Log Monitoring + +Watch logs in real-time: +```bash +sudo journalctl -u composesync -f +``` + +### Recent Logs + +View recent log entries: +```bash +# Last 50 lines +sudo journalctl -u composesync -n 50 + +# Last hour +sudo journalctl -u composesync --since "1 hour ago" + +# Today's logs +sudo journalctl -u composesync --since "today" +``` + +### Log Filtering + +Filter logs by specific criteria: +```bash +# Only error messages +sudo journalctl -u composesync -p err + +# Only messages containing "immich" +sudo journalctl -u composesync | grep immich + +# Dry-run messages only +sudo journalctl -u composesync | grep "DRY-RUN" +``` + +### Log Export + +Export logs to a file: +```bash +# Export today's logs +sudo journalctl -u composesync --since "today" > composesync-logs.txt + +# Export all logs +sudo journalctl -u composesync > composesync-all-logs.txt +``` + +## Service Control + +### Service Status + +Check service status: +```bash +sudo systemctl status composesync +``` + +### Start/Stop/Restart + +Control the service: +```bash +# Start the service +sudo systemctl start composesync + +# Stop the service +sudo systemctl stop composesync + +# Restart the service +sudo systemctl restart composesync + +# Reload configuration (if supported) +sudo systemctl reload composesync +``` + +### Enable/Disable + +Manage service startup: +```bash +# Enable service to start on boot +sudo systemctl enable composesync + +# Disable service from starting on boot +sudo systemctl disable composesync +``` + +### Service Information + +Get detailed service information: +```bash +# Show service configuration +sudo systemctl show composesync + +# Show service dependencies +sudo systemctl list-dependencies composesync +``` + +## Stack Monitoring + +### Check Stack Status + +Monitor individual stacks: +```bash +# Check if a specific stack is running +docker compose -f /opt/composesync/stacks/immich/docker-compose.yml ps + +# Check all stacks +for stack in /opt/composesync/stacks/*/; do + echo "=== $(basename $stack) ===" + docker compose -f "$stack/docker-compose.yml" ps +done +``` + +### Stack Health + +Check stack health and logs: +```bash +# Check stack logs +docker compose -f /opt/composesync/stacks/immich/docker-compose.yml logs + +# Check specific service logs +docker compose -f /opt/composesync/stacks/immich/docker-compose.yml logs immich-server + +# Follow logs in real-time +docker compose -f /opt/composesync/stacks/immich/docker-compose.yml logs -f +``` + +### Version History + +Check versioned files: +```bash +# List versioned files for a stack +ls -la /opt/composesync/stacks/immich/compose-*.yml.bak + +# Check backup directories +ls -la /opt/composesync/stacks/immich/backups/ +``` + +## Performance Monitoring + +### Resource Usage + +Monitor system resources: +```bash +# Check CPU and memory usage +top -p $(pgrep -f update-agent.sh) + +# Check disk usage +du -sh /opt/composesync/stacks/*/ + +# Check for large backup directories +find /opt/composesync/stacks/ -name "backups" -type d -exec du -sh {} \; +``` + +### Network Monitoring + +Monitor download activity: +```bash +# Check network connections +netstat -tulpn | grep wget +netstat -tulpn | grep git + +# Monitor bandwidth usage +iftop -i eth0 +``` + +## Alerting and Notifications + +### Webhook Monitoring + +Test webhook notifications: +```bash +# Test webhook manually +curl -X POST -H "Content-Type: application/json" \ + -d '{"event": "test", "message": "Test notification"}' \ + $NOTIFICATION_WEBHOOK_URL +``` + +### Email Notifications + +Set up email alerts (if your webhook supports it): +```bash +# Example: Send email via webhook +curl -X POST -H "Content-Type: application/json" \ + -d '{"to": "admin@example.com", "subject": "ComposeSync Alert", "body": "Update failed"}' \ + https://your-webhook-service.com/email +``` + +## Monitoring Best Practices + +### Regular Health Checks + +Set up regular monitoring: +```bash +# Create a monitoring script +cat > /usr/local/bin/composesync-health-check.sh << 'EOF' +#!/bin/bash + +# Check if service is running +if ! systemctl is-active --quiet composesync; then + echo "ComposeSync service is not running!" + exit 1 +fi + +# Check for recent errors +if journalctl -u composesync --since "1 hour ago" | grep -q "ERROR"; then + echo "ComposeSync has errors in the last hour" + exit 1 +fi + +# Check disk usage +if [ $(df /opt/composesync | tail -1 | awk '{print $5}' | sed 's/%//') -gt 90 ]; then + echo "ComposeSync disk usage is high" + exit 1 +fi + +echo "ComposeSync is healthy" +exit 0 +EOF + +chmod +x /usr/local/bin/composesync-health-check.sh +``` + +### Automated Monitoring + +Set up automated monitoring with cron: +```bash +# Add to crontab +echo "*/15 * * * * /usr/local/bin/composesync-health-check.sh" | sudo crontab - +``` + +### Log Rotation + +Configure log rotation to prevent disk space issues: +```bash +# Create logrotate configuration +sudo tee /etc/logrotate.d/composesync << EOF +/var/log/composesync/*.log { + daily + missingok + rotate 7 + compress + delaycompress + notifempty + create 644 composesync composesync +} +EOF +``` + +## Troubleshooting Monitoring + +### Service Not Starting + +If the service won't start: +```bash +# Check for configuration errors +sudo systemctl status composesync + +# Check logs for specific errors +sudo journalctl -u composesync -n 20 + +# Verify file permissions +ls -la /opt/composesync/ +``` + +### High Resource Usage + +If ComposeSync is using too many resources: +```bash +# Check what's consuming resources +ps aux | grep update-agent + +# Check for stuck processes +pgrep -f update-agent + +# Restart the service +sudo systemctl restart composesync +``` + +### Missing Logs + +If logs are missing: +```bash +# Check if journald is working +sudo journalctl --verify + +# Check journald status +sudo systemctl status systemd-journald + +# Check log storage +sudo journalctl --disk-usage +``` + +## Integration with External Monitoring + +### Prometheus/Grafana + +For advanced monitoring, you can integrate with Prometheus: +```bash +# Example: Export metrics via webhook +curl -X POST -H "Content-Type: application/json" \ + -d '{"metric": "composesync_updates_total", "value": 1, "labels": {"stack": "immich"}}' \ + http://prometheus:9090/api/v1/write +``` + +### Nagios/Icinga + +Create custom checks for monitoring systems: +```bash +# Example Nagios check +#!/bin/bash +if systemctl is-active --quiet composesync; then + echo "OK: ComposeSync is running" + exit 0 +else + echo "CRITICAL: ComposeSync is not running" + exit 2 +fi +``` + +## Security Monitoring + +### Access Monitoring + +Monitor who accesses ComposeSync: +```bash +# Check who modified the configuration +ls -la /opt/composesync/.env + +# Check for unauthorized changes +find /opt/composesync -mtime -1 -ls + +# Monitor Docker group membership +getent group docker +``` + +### Audit Logging + +Enable audit logging: +```bash +# Monitor file access +auditctl -w /opt/composesync/.env -p wa -k composesync_config + +# Monitor Docker socket access +auditctl -w /var/run/docker.sock -p wa -k docker_access +``` + +This comprehensive monitoring setup will help you keep track of ComposeSync's health and performance. \ No newline at end of file diff --git a/Docs/multi-stack.md b/Docs/multi-stack.md new file mode 100644 index 0000000..09ef47c --- /dev/null +++ b/Docs/multi-stack.md @@ -0,0 +1,254 @@ +# Multi-Stack Configuration + +This guide covers how to configure and manage multiple Docker Compose stacks with ComposeSync. + +## Basic Multi-Stack Setup + +You can manage multiple stacks by adding configurations to your `.env` file. Each stack is configured with a numbered prefix: + +```env +# Number of stacks to manage +STACKS=2 + +# Stack 1: Immich +STACK_1_NAME=immich +STACK_1_URL=https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml +STACK_1_PATH=/opt/composesync/stacks/immich +STACK_1_TOOL=wget +STACK_1_INTERVAL=86400 + +# Stack 2: Portainer +STACK_2_NAME=portainer +STACK_2_URL=https://github.com/portainer/portainer-compose.git +STACK_2_PATH=/opt/composesync/stacks/portainer +STACK_2_TOOL=git +STACK_2_INTERVAL=43200 +``` + +## Multiple Compose Files + +For stacks that use multiple compose files (like Immich), you can specify them in the configuration: + +```env +STACKS=1 + +# Immich stack with multiple compose files +STACK_1_NAME=immich +STACK_1_URL=https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml +STACK_1_PATH=/opt/composesync/stacks/immich +STACK_1_TOOL=wget +STACK_1_INTERVAL=86400 +STACK_1_KEEP_VERSIONS=10 + +# Additional compose files (numbered) +STACK_1_EXTRA_FILES_1=https://raw.githubusercontent.com/immich-app/immich/refs/heads/main/docker/hwaccel.ml.yml +STACK_1_EXTRA_FILES_2=https://raw.githubusercontent.com/immich-app/immich/refs/heads/main/docker/hwaccel.transcoding.yml +STACK_1_EXTRA_FILES_3=https://raw.githubusercontent.com/immich-app/immich/refs/heads/main/docker/prometheus.yml + +# Custom file ordering (optional) +STACK_1_EXTRA_FILES_ORDER=3,1,2 +``` + +If `STACK_1_EXTRA_FILES_ORDER` is set, extra files will be processed in the specified order (e.g., 3,1,2). Otherwise, files are processed in numeric order (1,2,3,...). + +## Complete Example: Immich Stack + +Here's a complete example of setting up Immich with all its compose files: + +### 1. Configure the stack in `.env` + +```env +STACKS=1 + +# Main compose file +STACK_1_NAME=immich +STACK_1_URL=https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml +STACK_1_PATH=/opt/composesync/stacks/immich +STACK_1_TOOL=wget +STACK_1_INTERVAL=86400 +STACK_1_KEEP_VERSIONS=10 + +# Additional compose files (one per line, numbered) +# 1. Hardware acceleration for machine learning +STACK_1_EXTRA_FILES_1=https://raw.githubusercontent.com/immich-app/immich/refs/heads/main/docker/hwaccel.ml.yml +# 2. Hardware acceleration for video transcoding +STACK_1_EXTRA_FILES_2=https://raw.githubusercontent.com/immich-app/immich/refs/heads/main/docker/hwaccel.transcoding.yml +# 3. Prometheus monitoring +STACK_1_EXTRA_FILES_3=https://raw.githubusercontent.com/immich-app/immich/refs/heads/main/docker/prometheus.yml +``` + +The script will process these files in order (1, 2, 3) when running `docker compose`. + +### 2. Create your override file + +```bash +sudo mkdir -p /opt/composesync/stacks/immich +sudo nano /opt/composesync/stacks/immich/docker-compose.override.yml +``` + +**Note:** You must create the `docker-compose.override.yml` file manually. ComposeSync will not create it for you. This file should contain your customizations and will be preserved during updates. + +### 3. Add your customizations + +```yaml +# docker-compose.override.yml +services: + immich-server: + environment: + - IMMICH_SERVER_URL=http://immich-server:2283 + - IMMICH_API_URL_EXTERNAL=https://immich.raines.xyz/api + - IMMICH_WEB_URL=https://immich.raines.xyz + networks: + - npm_network + - immich-backend + devices: + - /dev/dri:/dev/dri + + redis: + networks: + - immich-backend + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 30s + timeout: 5s + retries: 3 + + database: + networks: + - immich-backend + healthcheck: + test: ["CMD-SHELL", "pg_isready -d ${DB_DATABASE_NAME} -U ${DB_USERNAME}"] + interval: 30s + timeout: 5s + retries: 3 + start_interval: 30s + + immich-machine-learning: + image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release}-openvino + environment: + - OPENVINO_DEVICE=GPU + - OPENVINO_GPU_DEVICE_ID=0 + devices: + - /dev/dri:/dev/dri + device_cgroup_rules: + - 'c 189:* rmw' + volumes: + - /dev/bus/usb:/dev/bus/usb + networks: + - immich-backend + +volumes: + cifs_immich: + external: true + +networks: + npm_network: + external: true + immich-backend: + name: immich-backend +``` + +### 4. Set permissions + +```bash +sudo chown -R YOUR_USERNAME:docker /opt/composesync/stacks/immich +``` + +### 5. Restart the service + +```bash +sudo systemctl restart composesync +``` + +## How Multiple Files Work + +The service will now: +1. Download the main `docker-compose.yml` +2. Download all additional compose files specified in `STACK_1_EXTRA_FILES` +3. Apply your `docker-compose.override.yml` +4. Use all files when running `docker compose up` + +When running commands manually, you'll need to specify all the files: +```bash +docker compose -f docker-compose.yml \ + -f hwaccel.ml.yml \ + -f hwaccel.transcoding.yml \ + -f prometheus.yml \ + -f docker-compose.override.yml \ + up -d +``` + +## Stack-Specific Settings + +Each stack can have its own settings that override the global configuration: + +```env +# Global settings +UPDATE_INTERVAL_SECONDS=3600 +KEEP_VERSIONS=10 + +# Stack 1: Check every 12 hours, keep 15 versions +STACK_1_INTERVAL=43200 +STACK_1_KEEP_VERSIONS=15 + +# Stack 2: Check every 6 hours, keep 5 versions +STACK_2_INTERVAL=21600 +STACK_2_KEEP_VERSIONS=5 +``` + +## Creating Override Files + +For each stack, you should create a `docker-compose.override.yml` file in the stack directory. This file contains your customizations and will be preserved during updates. + +Example override file structure: +```yaml +services: + your-service: + environment: + - CUSTOM_VAR=value + networks: + - your-network + volumes: + - /host/path:/container/path + +networks: + your-network: + external: true +``` + +## Managing Multiple Stacks + +### Viewing All Stacks +```bash +ls -la /opt/composesync/stacks/ +``` + +### Checking Stack Status +```bash +# Check if a specific stack is running +docker compose -f /opt/composesync/stacks/immich/docker-compose.yml ps + +# Check all stacks +for stack in /opt/composesync/stacks/*/; do + echo "=== $(basename $stack) ===" + docker compose -f "$stack/docker-compose.yml" ps +done +``` + +### Manual Updates +```bash +# Update a specific stack manually +sudo systemctl restart composesync + +# Or run the update script directly +sudo -u composesync /opt/composesync/update-agent.sh +``` + +## Best Practices + +1. **Use descriptive stack names** - Make them easy to identify +2. **Group related stacks** - Keep similar applications together +3. **Set appropriate intervals** - More critical stacks can check more frequently +4. **Use stack-specific settings** - Override global settings when needed +5. **Test with dry-run mode** - Verify configurations before applying +6. **Monitor logs** - Keep an eye on update activities \ No newline at end of file diff --git a/Docs/safety-features.md b/Docs/safety-features.md new file mode 100644 index 0000000..85bc2f2 --- /dev/null +++ b/Docs/safety-features.md @@ -0,0 +1,272 @@ +# Safety Features + +This guide covers the safety features built into ComposeSync to protect your Docker Compose stacks. + +## Automatic Rollback + +ComposeSync includes automatic rollback functionality to handle failed updates: + +### How Rollback Works + +1. **Pre-Update Backup**: Before applying changes, all current files are backed up +2. **Update Attempt**: Changes are applied using `docker compose up -d` +3. **Failure Detection**: If the Docker Compose command fails, rollback is triggered +4. **Automatic Restoration**: All files are restored from the backup +5. **Stack Restart**: The stack is restarted with the original configuration + +### Rollback Process + +When a rollback occurs, ComposeSync will: + +```bash +# 1. Restore main compose file +cp /opt/composesync/stacks/myapp/backups/backup-20240115103000/docker-compose.yml /opt/composesync/stacks/myapp/docker-compose.yml + +# 2. Restore extra files (if any) +cp /opt/composesync/stacks/myapp/backups/backup-20240115103000/hwaccel.ml.yml /opt/composesync/stacks/myapp/hwaccel.ml.yml + +# 3. Restore override file (if it existed) +cp /opt/composesync/stacks/myapp/backups/backup-20240115103000/docker-compose.override.yml /opt/composesync/stacks/myapp/docker-compose.override.yml + +# 4. Restart stack with original configuration +docker compose -f /opt/composesync/stacks/myapp/docker-compose.yml up -d +``` + +### Rollback Benefits + +This ensures that: +- Your stack never gets stuck in a broken state +- Failed updates don't leave your services down +- You can quickly recover from problematic updates +- The system remains stable even with network issues or invalid configurations + +## Lock File Protection + +ComposeSync uses lock files to prevent concurrent updates to the same stack. + +### How Lock Files Work + +Each stack has a `.lock` file in its directory that: +- Is created when an update starts +- Is automatically removed when the update completes (success or failure) +- Has a 5-minute timeout to handle stale locks from crashed processes +- Uses directory-based locking for atomicity + +### Lock File Location + +``` +/opt/composesync/stacks/myapp/ +├── docker-compose.yml +├── docker-compose.override.yml +├── .lock/ # Lock directory +└── backups/ +``` + +### Stale Lock Detection + +If a lock file exists and is older than 5 minutes, ComposeSync will automatically remove it and proceed with the update. This handles cases where: +- The previous update process crashed +- The system was rebooted during an update +- Network issues interrupted the update process + +### Lock File Benefits + +Lock files prevent: +- Race conditions when multiple instances run simultaneously +- Corruption of compose files during updates +- Multiple update processes running on the same stack + +## Versioned History + +ComposeSync maintains a versioned history of your compose files for easy rollback and audit trails. + +### Version Identifiers + +- **For Git sources**: Uses the Git commit hash as the version identifier +- **For other sources**: Uses a timestamp (YYYYMMDDHHMMSS format) + +### Versioned File Structure + +``` +stack/ +├── docker-compose.yml # Current active compose file +├── docker-compose.override.yml # Your customizations +├── compose-20240315123456.yml.bak # Versioned copy (timestamp) +├── compose-a1b2c3d.yml.bak # Versioned copy (git hash) +└── backups/ # Backup directory for rollbacks +``` + +### Manual Rollback + +To roll back to a specific version: + +```bash +# Example: Roll back to a specific version +cp /opt/composesync/stacks/immich/compose-20240315123456.yml.bak /opt/composesync/stacks/immich/docker-compose.yml + +# Restart the stack with the rolled back configuration +docker compose -f /opt/composesync/stacks/immich/docker-compose.yml \ + -f /opt/composesync/stacks/immich/docker-compose.override.yml \ + up -d --remove-orphans +``` + +### Version Cleanup + +ComposeSync automatically maintains a configurable number of versioned files (default: 10) to prevent disk space issues. You can configure this per stack using the `STACK_N_KEEP_VERSIONS` environment variable. + +## Backup Management + +ComposeSync creates comprehensive backups before applying any changes. + +### Backup Structure + +Each update creates a timestamped backup directory: + +``` +/opt/composesync/stacks/myapp/backups/ +├── backup-20240115103000/ +│ ├── docker-compose.yml +│ ├── docker-compose.override.yml +│ ├── hwaccel.ml.yml +│ └── hwaccel.transcoding.yml +├── backup-20240115100000/ +└── backup-20240115093000/ +``` + +### Backup Contents + +Each backup includes: +- Main `docker-compose.yml` file +- `docker-compose.override.yml` (if it exists) +- All extra compose files +- Complete state before the update + +### Backup Cleanup + +Backup directories are automatically cleaned up based on the `KEEP_VERSIONS` setting: +- Default: Keep 10 backup directories +- Configurable per stack with `STACK_N_KEEP_VERSIONS` +- Oldest backups are removed first + +## Error Handling + +ComposeSync includes comprehensive error handling to ensure system stability. + +### Error Types Handled + +1. **Download Failures**: Network issues, invalid URLs, authentication problems +2. **File Validation**: Empty files, corrupted downloads, missing files +3. **Docker Compose Failures**: Invalid configurations, service startup issues +4. **Permission Issues**: File access problems, directory creation failures +5. **Lock File Issues**: Stale locks, concurrent access attempts + +### Error Recovery + +When errors occur, ComposeSync will: +- Log detailed error messages +- Attempt automatic recovery where possible +- Trigger rollback for critical failures +- Continue processing other stacks +- Send webhook notifications (if configured) + +### Error Logging + +All errors are logged with timestamps and context: + +``` +[2024-01-15 10:30:00] ERROR: Failed to download https://example.com/compose.yml +[2024-01-15 10:30:01] ERROR: Downloaded file is empty +[2024-01-15 10:30:02] ERROR: Failed to update stack myapp, attempting rollback... +[2024-01-15 10:30:03] Successfully rolled back stack myapp +``` + +## File Validation + +ComposeSync validates downloaded files before processing them. + +### Validation Checks + +- **File Existence**: Ensures files were downloaded successfully +- **File Size**: Verifies files are not empty +- **File Format**: Basic YAML validation +- **Content Integrity**: Checks for corrupted downloads + +### Validation Failures + +If validation fails: +- The file is not used +- An error is logged +- Rollback is triggered if necessary +- The process continues with other files + +## Update Modes + +ComposeSync supports different update modes for different safety levels. + +### Notify Only Mode + +```env +UPDATE_MODE=notify_only +``` + +In this mode: +- Files are downloaded and processed +- Changes are detected and logged +- No updates are applied to running stacks +- Perfect for testing and monitoring + +### Notify and Apply Mode + +```env +UPDATE_MODE=notify_and_apply +``` + +In this mode: +- Files are downloaded and processed +- Changes are automatically applied +- Rollback occurs on failures +- Full automation with safety features + +## Best Practices + +### 1. Test with Dry-Run Mode + +Always test new configurations with dry-run mode: +```env +DRY_RUN=true +``` + +### 2. Use Appropriate Update Intervals + +Set reasonable update intervals based on your needs: +- Production: 6-24 hours +- Development: 30 minutes - 2 hours +- Testing: Use dry-run mode + +### 3. Monitor Logs + +Regularly check the service logs: +```bash +sudo journalctl -u composesync -f +``` + +### 4. Configure Webhook Notifications + +Set up webhook notifications to monitor updates: +```env +NOTIFICATION_WEBHOOK_URL=https://your-webhook-url.com/endpoint +``` + +### 5. Regular Backup Verification + +Periodically verify your backups are working: +```bash +ls -la /opt/composesync/stacks/*/backups/ +``` + +### 6. Version Management + +Keep an appropriate number of versions: +- More versions = more rollback options +- Fewer versions = less disk space usage +- Balance based on your needs and storage \ No newline at end of file diff --git a/Docs/troubleshooting.md b/Docs/troubleshooting.md new file mode 100644 index 0000000..ae284f3 --- /dev/null +++ b/Docs/troubleshooting.md @@ -0,0 +1,342 @@ +# Troubleshooting Guide + +This guide covers common issues and their solutions when using ComposeSync. + +## Service Issues + +### Service Won't Start + +**Problem:** The ComposeSync service fails to start. + +**Solutions:** +1. Check service status: + ```bash + sudo systemctl status composesync + ``` + +2. Check logs for errors: + ```bash + sudo journalctl -u composesync -n 50 + ``` + +3. Verify user permissions: + ```bash + # Ensure the service user is in the docker group + groups YOUR_USERNAME + ``` + +4. Check file permissions: + ```bash + # Ensure the service user owns the ComposeSync directory + ls -la /opt/composesync/ + sudo chown -R YOUR_USERNAME:docker /opt/composesync/ + ``` + +### Service Crashes or Stops Unexpectedly + +**Problem:** The service runs but crashes or stops unexpectedly. + +**Solutions:** +1. Check for configuration errors: + ```bash + sudo journalctl -u composesync -f + ``` + +2. Verify your `.env` file syntax: + ```bash + # Check for syntax errors + source /opt/composesync/.env + ``` + +3. Test with dry-run mode: + ```env + DRY_RUN=true + ``` + +## Download Issues + +### Failed Downloads + +**Problem:** ComposeSync fails to download compose files. + +**Solutions:** +1. Check network connectivity: + ```bash + # Test if the URL is accessible + wget -q --spider https://your-url.com/docker-compose.yml + echo $? + ``` + +2. Verify URLs in your configuration: + ```bash + # Check your .env file + grep STACK_.*_URL /opt/composesync/.env + ``` + +3. Check for authentication requirements: + - Some URLs may require authentication + - Consider using Git repositories instead + +### Git Repository Issues + +**Problem:** Git operations fail. + +**Solutions:** +1. Verify repository access: + ```bash + # Test git clone manually + git clone --quiet https://github.com/user/repo.git /tmp/test + ``` + +2. Check Git subpath configuration: + ```bash + # Ensure the subpath exists in the repository + git ls-tree -r --name-only HEAD | grep docker-compose.yml + ``` + +3. Verify branch/tag exists: + ```bash + # List available branches/tags + git ls-remote --heads https://github.com/user/repo.git + git ls-remote --tags https://github.com/user/repo.git + ``` + +## Docker Compose Issues + +### Update Failures + +**Problem:** Docker Compose updates fail and trigger rollback. + +**Solutions:** +1. Check Docker Compose syntax: + ```bash + # Validate compose file manually + docker compose -f /path/to/docker-compose.yml config + ``` + +2. Check for port conflicts: + ```bash + # Check what's using the ports + netstat -tulpn | grep :80 + ``` + +3. Verify override file syntax: + ```bash + # Test with override file + docker compose -f docker-compose.yml -f docker-compose.override.yml config + ``` + +### Rollback Failures + +**Problem:** Both the update and rollback fail. + +**Solutions:** +1. Check backup files: + ```bash + # Verify backups exist + ls -la /opt/composesync/stacks/*/backups/ + ``` + +2. Manual rollback: + ```bash + # Manually restore from backup + cp /opt/composesync/stacks/stackname/backups/backup-*/docker-compose.yml /opt/composesync/stacks/stackname/ + ``` + +3. Check Docker daemon: + ```bash + # Ensure Docker is running + sudo systemctl status docker + ``` + +## Configuration Issues + +### Missing Environment Variables + +**Problem:** Required configuration is missing. + +**Solutions:** +1. Check your `.env` file: + ```bash + # Verify all required variables are set + grep -E "STACK_.*_(NAME|URL|PATH|TOOL)" /opt/composesync/.env + ``` + +2. Check variable syntax: + ```bash + # Look for syntax errors + cat -n /opt/composesync/.env + ``` + +### Invalid Paths + +**Problem:** Stack paths don't exist or are inaccessible. + +**Solutions:** +1. Create missing directories: + ```bash + # Create stack directories + sudo mkdir -p /opt/composesync/stacks/stackname + sudo chown YOUR_USERNAME:docker /opt/composesync/stacks/stackname + ``` + +2. Check permissions: + ```bash + # Verify directory permissions + ls -la /opt/composesync/stacks/ + ``` + +## Webhook Issues + +### Webhook Notifications Not Sent + +**Problem:** Webhook notifications aren't being sent. + +**Solutions:** +1. Check webhook URL: + ```bash + # Verify URL is set + grep NOTIFICATION_WEBHOOK_URL /opt/composesync/.env + ``` + +2. Test webhook manually: + ```bash + # Test webhook endpoint + curl -X POST -H "Content-Type: application/json" \ + -d '{"test": "message"}' \ + https://your-webhook-url.com/endpoint + ``` + +3. Check network connectivity: + ```bash + # Test if webhook URL is accessible + wget -q --spider https://your-webhook-url.com/endpoint + ``` + +## Performance Issues + +### High Resource Usage + +**Problem:** ComposeSync uses too much CPU or memory. + +**Solutions:** +1. Increase update intervals: + ```env + UPDATE_INTERVAL_SECONDS=7200 # Check every 2 hours instead of 1 + ``` + +2. Reduce version history: + ```env + KEEP_VERSIONS=5 # Keep fewer versions + ``` + +3. Use dry-run mode for testing: + ```env + DRY_RUN=true + ``` + +### Slow Downloads + +**Problem:** Downloads are taking too long. + +**Solutions:** +1. Check network connectivity: + ```bash + # Test download speed + wget -O /dev/null https://your-url.com/docker-compose.yml + ``` + +2. Consider using Git instead of wget: + ```env + STACK_1_TOOL=git + ``` + +## Lock File Issues + +### Stale Lock Files + +**Problem:** Lock files prevent updates. + +**Solutions:** +1. Check for stale locks: + ```bash + # Look for lock files + find /opt/composesync/stacks/ -name ".lock" -type d + ``` + +2. Remove stale locks manually: + ```bash + # Remove lock file (be careful!) + rm -rf /opt/composesync/stacks/stackname/.lock + ``` + +3. Restart the service: + ```bash + sudo systemctl restart composesync + ``` + +## Debugging Tips + +### Enable Verbose Logging + +For detailed debugging, you can temporarily modify the log function: + +```bash +# Edit the update-agent.sh file +sudo nano /opt/composesync/update-agent.sh + +# Add more verbose logging +log() { + local prefix="" + if [ "$DRY_RUN" = "true" ]; then + prefix="[DRY-RUN] " + fi + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ${prefix}$1" | tee -a /tmp/composesync-debug.log +} +``` + +### Test Individual Components + +1. **Test download function:** + ```bash + # Test wget download + wget -q -O /tmp/test.yml https://your-url.com/docker-compose.yml + ``` + +2. **Test Docker Compose:** + ```bash + # Test compose file manually + docker compose -f /path/to/docker-compose.yml config + ``` + +3. **Test webhook:** + ```bash + # Test webhook manually + curl -X POST -H "Content-Type: application/json" \ + -d '{"event": "test"}' \ + $NOTIFICATION_WEBHOOK_URL + ``` + +## Getting Help + +If you're still experiencing issues: + +1. **Check the logs:** + ```bash + sudo journalctl -u composesync -f + ``` + +2. **Enable dry-run mode** to test without making changes: + ```env + DRY_RUN=true + ``` + +3. **Verify your configuration** step by step + +4. **Check the documentation** for your specific use case + +5. **Submit an issue** with: + - Your configuration (with sensitive data removed) + - Relevant log output + - Steps to reproduce the issue + - Expected vs actual behavior \ No newline at end of file diff --git a/Docs/watchtower.md b/Docs/watchtower.md new file mode 100644 index 0000000..9104f9e --- /dev/null +++ b/Docs/watchtower.md @@ -0,0 +1,263 @@ +# Watchtower Integration + +This guide covers how to use ComposeSync alongside Watchtower and how to configure them to work together effectively. + +## Understanding the Tools + +### Watchtower +- Monitors running Docker images and updates them when new versions are available in the registry +- Focuses on updating container images, not compose file configurations +- Works at the container level + +### ComposeSync +- Monitors remote sources for changes to `docker-compose.yml` files and applies those changes +- Focuses on updating compose file configurations, not just images +- Works at the compose file level + +## Recommended Configuration + +For stacks managed by ComposeSync, it's recommended to disable Watchtower to prevent conflicts and race conditions. + +### Disabling Watchtower for ComposeSync Stacks + +Add the following to your `docker-compose.override.yml`: + +```yaml +services: + your-service: + labels: + - "com.centurylinklabs.watchtower.enable=false" +``` + +### Example Configuration + +```yaml +# docker-compose.override.yml for a ComposeSync-managed stack +services: + immich-server: + labels: + - "com.centurylinklabs.watchtower.enable=false" + immich-microservices: + labels: + - "com.centurylinklabs.watchtower.enable=false" + immich-machine-learning: + labels: + - "com.centurylinklabs.watchtower.enable=false" +``` + +## Best Practices + +### 1. Use ComposeSync for: +- Applications where the `docker-compose.yml` configuration evolves +- Complex stacks with multiple services +- Applications that require specific version management +- Stacks with custom configurations and overrides + +### 2. Use Watchtower for: +- Simple, self-contained applications +- Stacks with static `docker-compose.yml` files +- Applications where only image updates are needed +- Stacks without complex configurations + +### 3. Notification Integration: +- Configure both tools to send notifications to the same webhook +- This provides a unified view of all Docker updates +- Helps track what's being updated and when + +## Configuration Examples + +### ComposeSync-Managed Stack (Immich) + +```yaml +# docker-compose.override.yml +services: + immich-server: + labels: + - "com.centurylinklabs.watchtower.enable=false" + environment: + - IMMICH_SERVER_URL=http://immich-server:2283 + networks: + - npm_network + - immich-backend + + redis: + labels: + - "com.centurylinklabs.watchtower.enable=false" + networks: + - immich-backend + + database: + labels: + - "com.centurylinklabs.watchtower.enable=false" + networks: + - immich-backend + +networks: + npm_network: + external: true + immich-backend: + name: immich-backend +``` + +### Watchtower-Managed Stack (Simple App) + +```yaml +# docker-compose.yml +services: + simple-app: + image: nginx:latest + ports: + - "8080:80" + labels: + - "com.centurylinklabs.watchtower.enable=true" +``` + +## Potential Conflicts + +### Race Conditions +- Both tools might try to update the same stack simultaneously +- Can lead to inconsistent states +- May cause service interruptions + +### Configuration Conflicts +- Watchtower might update images while ComposeSync is updating compose files +- Can result in version mismatches +- May break service dependencies + +### Resource Contention +- Both tools competing for Docker resources +- Can slow down update processes +- May cause timeouts or failures + +## Monitoring Both Tools + +### Unified Logging +Monitor both services together: +```bash +# Watch ComposeSync logs +sudo journalctl -u composesync -f + +# Watch Watchtower logs (if running as container) +docker logs -f watchtower + +# Or if running as systemd service +sudo journalctl -u watchtower -f +``` + +### Webhook Integration +Configure both tools to use the same webhook: +```env +# ComposeSync webhook +NOTIFICATION_WEBHOOK_URL=https://your-webhook-url.com/endpoint + +# Watchtower webhook (in docker-compose.yml) +environment: + - WATCHTOWER_NOTIFICATIONS=webhook + - WATCHTOWER_NOTIFICATION_WEBHOOK_URL=https://your-webhook-url.com/endpoint +``` + +## Migration Strategy + +### From Watchtower to ComposeSync + +1. **Identify stacks to migrate** + - Choose stacks with evolving configurations + - Select stacks that need custom overrides + +2. **Configure ComposeSync** + - Add stack configuration to `.env` + - Set up override files + - Test with dry-run mode + +3. **Disable Watchtower** + - Add labels to disable Watchtower for migrated stacks + - Keep Watchtower for simple stacks + +4. **Monitor and verify** + - Check that updates work correctly + - Verify no conflicts occur + +### From ComposeSync to Watchtower + +1. **Identify simple stacks** + - Choose stacks with static configurations + - Select stacks that only need image updates + +2. **Remove from ComposeSync** + - Remove stack configuration from `.env` + - Clean up stack directories + +3. **Enable Watchtower** + - Remove Watchtower disable labels + - Configure Watchtower for the stack + +## Troubleshooting + +### Update Conflicts + +If you see conflicts between the tools: +1. Check which tool is managing each stack +2. Ensure proper labels are set +3. Monitor logs for race conditions +4. Consider separating responsibilities more clearly + +### Service Failures + +If services fail after updates: +1. Check which tool performed the update +2. Verify the update was applied correctly +3. Check for configuration conflicts +4. Review logs for error messages + +### Performance Issues + +If updates are slow or failing: +1. Check resource usage during updates +2. Verify network connectivity +3. Consider staggering update intervals +4. Monitor for resource contention + +## Advanced Configuration + +### Selective Updates + +You can configure both tools to update different aspects: + +```yaml +# ComposeSync handles compose file updates +# Watchtower handles image updates for specific services +services: + app: + labels: + - "com.centurylinklabs.watchtower.enable=true" + - "com.centurylinklabs.watchtower.scope=app" + + database: + labels: + - "com.centurylinklabs.watchtower.enable=false" +``` + +### Update Scheduling + +Coordinate update schedules to avoid conflicts: + +```env +# ComposeSync: Check every 6 hours +UPDATE_INTERVAL_SECONDS=21600 + +# Watchtower: Check every 12 hours (different schedule) +# Configure in Watchtower settings +``` + +### Notification Filtering + +Use different webhook endpoints for different tools: +```env +# ComposeSync notifications +NOTIFICATION_WEBHOOK_URL=https://webhook.com/composesync + +# Watchtower notifications (separate endpoint) +# Configure in Watchtower settings +``` + +This allows you to handle notifications differently based on the source. \ No newline at end of file diff --git a/Docs/webhooks.md b/Docs/webhooks.md new file mode 100644 index 0000000..fc92c69 --- /dev/null +++ b/Docs/webhooks.md @@ -0,0 +1,222 @@ +# Webhook Notifications + +This guide covers how to set up and configure webhook notifications in ComposeSync. + +## Overview + +ComposeSync can send webhook notifications when updates are applied or when errors occur. This is useful for: +- Monitoring update status remotely +- Integrating with notification systems (Discord, Slack, etc.) +- Alerting on failed updates +- Keeping track of when stacks are updated + +## Configuration + +To enable webhook notifications, add this to your `.env` file: + +```env +NOTIFICATION_WEBHOOK_URL=https://your-webhook-url.com/endpoint +``` + +## Webhook Payload + +The webhook will be called with a JSON payload containing: + +- `event`: The type of event (`update_success`, `update_failure`, `error`) +- `stack_name`: The name of the stack being updated +- `timestamp`: When the event occurred +- `message`: A human-readable description of what happened +- `version_id`: The version identifier for the update (if applicable) +- `diff`: A unified diff (truncated to 50 lines) showing the changes applied to the main compose file (for update_success and update_failure events; null for errors) + +## Event Types + +### Update Success + +Sent when a stack is successfully updated. Includes a diff of the changes: + +```json +{ + "event": "update_success", + "stack_name": "immich", + "timestamp": "2024-01-15T10:30:00Z", + "message": "Successfully updated stack immich to version a1b2c3d", + "version_id": "a1b2c3d", + "diff": "--- compose-a1b2c3d.yml.bak\n+++ docker-compose.yml\n@@ -1,6 +1,6 @@\n version: '3.8'\n services:\n immich-server:\n- image: ghcr.io/immich-app/immich-server:release\n+ image: ghcr.io/immich-app/immich-server:release-1.91.0\n... (diff truncated, showing first 50 lines)" +} +``` + +### Update Failure + +Sent when a stack update fails and rollback occurs. Includes the diff that was attempted: + +```json +{ + "event": "update_failure", + "stack_name": "immich", + "timestamp": "2024-01-15T10:30:00Z", + "message": "Failed to update stack immich, rolled back to previous version", + "version_id": "a1b2c3d", + "diff": "--- compose-a1b2c3d.yml.bak\n+++ docker-compose.yml\n@@ -1,6 +1,6 @@\n ... (diff truncated, showing first 50 lines)" +} +``` + +### Error + +Sent when a general error occurs. The diff field is null: + +```json +{ + "event": "error", + "stack_name": "immich", + "timestamp": "2024-01-15T10:30:00Z", + "message": "Failed to download compose file for stack immich", + "version_id": null, + "diff": null +} +``` + +## Integration Examples + +### Discord + +1. Create a Discord webhook in your server settings +2. Configure ComposeSync with the webhook URL: + +```env +NOTIFICATION_WEBHOOK_URL=https://discord.com/api/webhooks/YOUR_WEBHOOK_ID/YOUR_WEBHOOK_TOKEN +``` + +Discord will automatically format the JSON payload into a readable message. + +### Slack + +1. Create a Slack webhook in your workspace settings +2. Configure ComposeSync with the webhook URL: + +```env +NOTIFICATION_WEBHOOK_URL=https://hooks.slack.com/services/YOUR_WEBHOOK_URL +``` + +Slack will display the notification in your configured channel. + +### Custom Webhook Server + +You can create your own webhook server to handle notifications: + +```python +from flask import Flask, request +import json + +app = Flask(__name__) + +@app.route('/webhook', methods=['POST']) +def webhook(): + data = request.json + + if data['event'] == 'update_success': + print(f"✅ {data['stack_name']} updated successfully") + elif data['event'] == 'update_failure': + print(f"❌ {data['stack_name']} update failed") + elif data['event'] == 'error': + print(f"⚠️ Error with {data['stack_name']}: {data['message']}") + + return 'OK', 200 + +if __name__ == '__main__': + app.run(host='0.0.0.0', port=5000) +``` + +## Dry-Run Mode + +**Important:** Webhook notifications are sent regardless of whether you're in dry-run mode. This allows you to test your webhook configuration safely without applying actual changes. + +## Testing Webhooks + +To test your webhook configuration: + +1. Enable dry-run mode: + ```env + DRY_RUN=true + ``` + +2. Restart the service: + ```bash + sudo systemctl restart composesync + ``` + +3. Check your webhook endpoint for test notifications + +## Troubleshooting + +### Webhook Not Sent + +If webhooks aren't being sent: + +1. Check the webhook URL is correct +2. Verify the service can reach the webhook endpoint +3. Check the service logs for webhook errors: + ```bash + sudo journalctl -u composesync -f + ``` + +### Webhook Failures + +If webhook calls are failing: + +1. Check the webhook endpoint is accessible +2. Verify the endpoint accepts POST requests +3. Check for authentication requirements +4. Test the webhook URL manually: + ```bash + curl -X POST -H "Content-Type: application/json" \ + -d '{"test": "message"}' \ + https://your-webhook-url.com/endpoint + ``` + +### Rate Limiting + +Some webhook services have rate limits. If you're hitting limits: + +1. Increase the update interval +2. Use a different webhook service +3. Implement your own webhook server with rate limiting + +## Best Practices + +### 1. Use HTTPS + +Always use HTTPS URLs for webhooks to ensure security: +```env +NOTIFICATION_WEBHOOK_URL=https://your-webhook-url.com/endpoint +``` + +### 2. Test Your Webhook + +Always test your webhook configuration with dry-run mode before going live. + +### 3. Monitor Webhook Failures + +Set up monitoring for webhook failures to ensure you don't miss important notifications. + +### 4. Use Descriptive Messages + +The webhook messages are designed to be human-readable and informative. + +### 5. Handle Different Event Types + +Configure your webhook endpoint to handle all event types appropriately. + +## Advanced Configuration + +### Custom Webhook Headers + +If your webhook service requires custom headers, you may need to modify the webhook sending code in the update script. + +### Multiple Webhooks + +To send to multiple webhook endpoints, you can modify the webhook sending function to iterate through multiple URLs. + +### Webhook Authentication + +For webhooks requiring authentication, you can include credentials in the URL or modify the webhook sending code to include headers. \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a5cfb6f --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 ComposeSync Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..a73e805 --- /dev/null +++ b/README.md @@ -0,0 +1,90 @@ +# ComposeSync + +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) + +Automated Docker Compose update agent that downloads and applies updates to `docker-compose.yml` files from remote sources while preserving your custom overrides. + +**ComposeSync is designed to run directly on the host system as a systemd service, not in a container.** This provides better performance, reliability, and direct access to the Docker daemon. + +## TLDR + +ComposeSync automatically keeps your Docker Compose stacks up to date by: +1. Downloading the latest `docker-compose.yml` from a URL or Git repo +2. Preserving your `docker-compose.override.yml` customizations +3. Applying updates safely with automatic rollback on failure + +**Quick Example:** +```bash +# Install +sudo ./install.sh + +# Configure your stack +sudo nano /opt/composesync/.env +``` + +```env +STACKS=1 +STACK_1_NAME=immich +STACK_1_URL=https://github.com/immich-app/immich/releases/latest/download/docker-compose.yml +STACK_1_PATH=/opt/composesync/stacks/immich +STACK_1_TOOL=wget +``` + +**Or for Git repositories:** +```env +STACKS=1 +STACK_1_NAME=immich +STACK_1_URL=https://github.com/immich-app/immich.git +STACK_1_PATH=/opt/composesync/stacks/immich +STACK_1_TOOL=git +STACK_1_GIT_SUBPATH=deployments/docker +STACK_1_GIT_REF=main +``` + +```bash +# Create your override file +sudo mkdir -p /opt/composesync/stacks/immich +sudo nano /opt/composesync/stacks/immich/docker-compose.override.yml + +# Start the service +sudo systemctl start composesync +``` + +That's it! ComposeSync will now automatically update your Immich stack every hour while preserving your customizations. + +## Features + +- **Multi-Stack Support** - Manage multiple Docker Compose stacks +- **Multiple Sources** - HTTP/HTTPS URLs and Git repositories +- **Override Preservation** - Keeps your `docker-compose.override.yml` safe +- **Automatic Rollback** - Reverts failed updates automatically +- **Versioned History** - Easy rollback to previous versions +- **Dry-Run Mode** - Test updates without applying them +- **Webhook Notifications** - Get notified of updates and errors +- **Lock File Protection** - Prevents concurrent updates +- **Multiple Compose Files** - Handle complex stacks with multiple YAML files + +## Documentation + +- **[Installation Guide](Docs/installation.md)** - Detailed setup instructions +- **[Configuration Reference](Docs/configuration.md)** - All configuration options +- **[Multi-Stack Setup](Docs/multi-stack.md)** - Managing multiple stacks +- **[Git Repository Setup](Docs/git-repositories.md)** - Using Git sources +- **[Safety Features](Docs/safety-features.md)** - Rollback, locks, and error handling +- **[Webhook Notifications](Docs/webhooks.md)** - Setting up notifications +- **[Dry-Run Mode](Docs/dry-run.md)** - Testing configurations safely +- **[Watchtower Integration](Docs/watchtower.md)** - Working with Watchtower +- **[Troubleshooting](Docs/troubleshooting.md)** - Common issues and solutions + +## Quick Links + +- **[View Logs](Docs/monitoring.md#viewing-logs)**: `sudo journalctl -u composesync -f` +- **[Service Control](Docs/monitoring.md#service-control)**: Start, stop, restart +- **[Manual Rollback](Docs/safety-features.md#manual-rollback)**: Revert to previous version +- **[Configuration File](Docs/configuration.md#environment-file)**: `/opt/composesync/.env` + +## Contributing + +Feel free to submit issues and enhancement requests! + +This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details. \ No newline at end of file diff --git a/composesync.service b/composesync.service new file mode 100644 index 0000000..ce12963 --- /dev/null +++ b/composesync.service @@ -0,0 +1,39 @@ +[Unit] +Description=ComposeSync - Docker Compose File Updater +After=docker.service +Requires=docker.service + +[Service] +Type=simple +# User to run the service as. Must be in the 'docker' group. +User=YOUR_USERNAME +Group=docker +WorkingDirectory=/opt/composesync +EnvironmentFile=/opt/composesync/.env +# Allow the service to access Docker socket +Environment=DOCKER_HOST=unix:///var/run/docker.sock +# Set the base directory for stacks +Environment=COMPOSESYNC_BASE_DIR=/etc/composesync +# Set the number of versions to keep (default: 10) +Environment=KEEP_VERSIONS=10 +# Set update interval in seconds (default: 3600) +Environment=UPDATE_INTERVAL_SECONDS=3600 +# Set update mode (notify_only or notify_and_apply) +Environment=UPDATE_MODE=notify_and_apply +# Optional: Set webhook URL for notifications +# Environment=NOTIFICATION_WEBHOOK_URL= + +# The script will be installed in /usr/local/bin +ExecStart=/opt/composesync/update-agent.sh +Restart=always +RestartSec=5s +# Optional: Adjust CPU priority +# Nice=10 + +# Ensure clean shutdown +TimeoutStopSec=30 +KillMode=mixed +KillSignal=SIGTERM + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/install.sh b/install.sh new file mode 100644 index 0000000..a39fcc6 --- /dev/null +++ b/install.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Exit on error +set -e + +# Check if running as root +if [ "$EUID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +# Get username +read -p "Enter the username to run ComposeSync (must be in docker group): " USERNAME + +# Verify user exists and is in docker group +if ! id "$USERNAME" &>/dev/null; then + echo "Error: User $USERNAME does not exist" + exit 1 +fi + +if ! groups "$USERNAME" | grep -q docker; then + echo "Error: User $USERNAME is not in the docker group" + echo "Add user to docker group with: sudo usermod -aG docker $USERNAME" + exit 1 +fi + +# Create necessary directories +mkdir -p /opt/composesync +chown $USERNAME:docker /opt/composesync + +# Copy files to installation directory +cp update-agent.sh /opt/composesync/ +chmod +x /opt/composesync/update-agent.sh +chown $USERNAME:docker /opt/composesync/update-agent.sh + +# Create default .env file if it doesn't exist +if [ ! -f /opt/composesync/.env ]; then + cat > /opt/composesync/.env << EOF +# Base directory for stacks +COMPOSESYNC_BASE_DIR=/opt/composesync/stacks + +# Number of versions to keep (default: 10) +KEEP_VERSIONS=10 + +# Update interval in seconds (default: 3600) +UPDATE_INTERVAL_SECONDS=3600 + +# Update mode (notify_only or notify_and_apply) +UPDATE_MODE=notify_and_apply + +# Optional: Webhook URL for notifications +# NOTIFICATION_WEBHOOK_URL= +EOF + chown $USERNAME:docker /opt/composesync/.env +fi + +# Update service file with username +sed "s/YOUR_USERNAME/$USERNAME/" composesync.service > /etc/systemd/system/composesync.service + +# Reload systemd +systemctl daemon-reload + +# Enable and start the service +systemctl enable composesync +systemctl start composesync + +echo "ComposeSync has been installed and started!" +echo "You can check the status with: systemctl status composesync" +echo "View logs with: journalctl -u composesync -f" +echo "" +echo "To manage your stacks, create directories in /opt/composesync/stacks/" +echo "Example: sudo mkdir -p /opt/composesync/stacks/immich" +echo " sudo chown $USERNAME:docker /opt/composesync/stacks/immich" \ No newline at end of file diff --git a/update-agent.sh b/update-agent.sh new file mode 100644 index 0000000..9bf0236 --- /dev/null +++ b/update-agent.sh @@ -0,0 +1,434 @@ +#!/bin/bash + +# Exit on error +set -e + +# Load environment variables +if [ -f .env ]; then + source .env +fi + +# Default values +BASE_DIR=${COMPOSESYNC_BASE_DIR:-"/opt/composesync/stacks"} +KEEP_VERSIONS=${KEEP_VERSIONS:-10} +UPDATE_INTERVAL=${UPDATE_INTERVAL_SECONDS:-3600} +UPDATE_MODE=${UPDATE_MODE:-"notify_and_apply"} +DRY_RUN=${DRY_RUN:-"false"} +STACKS=${STACKS:-1} + +# Function to log messages +log() { + local prefix="" + if [ "$DRY_RUN" = "true" ]; then + prefix="[DRY-RUN] " + fi + echo "[$(date '+%Y-%m-%d %H:%M:%S')] ${prefix}$1" +} + +# Function to acquire lock +acquire_lock() { + local lock_file="$1" + local lock_timeout=300 # 5 minutes timeout + + # Try to create the lock file + if mkdir "$lock_file" 2>/dev/null; then + # Successfully acquired lock + return 0 + else + # Check if lock is stale + if [ -d "$lock_file" ]; then + local lock_age=$(($(date +%s) - $(stat -c %Y "$lock_file"))) + if [ $lock_age -gt $lock_timeout ]; then + log "Found stale lock, removing..." + rm -rf "$lock_file" + if mkdir "$lock_file" 2>/dev/null; then + return 0 + fi + fi + fi + return 1 + fi +} + +# Function to release lock +release_lock() { + local lock_file="$1" + rm -rf "$lock_file" +} + +# Function to download a file +download_file() { + local url=$1 + local output=$2 + local tool=$3 + local subpath=$4 + local git_ref=$5 + + log "Downloading $url to $output" + + case $tool in + "wget") + if ! wget -q -O "$output" "$url"; then + log "ERROR: Failed to download $url" + return 1 + fi + # Verify file is not empty + if [ ! -s "$output" ]; then + log "ERROR: Downloaded file $output is empty" + return 1 + fi + ;; + "git") + if [ ! -d "${output}.git" ]; then + if ! git clone --quiet "$url" "${output}.git"; then + log "ERROR: Failed to clone $url" + return 1 + fi + else + if ! (cd "${output}.git" && git fetch --quiet); then + log "ERROR: Failed to fetch from $url" + return 1 + fi + fi + + # Checkout specific branch/tag if specified + if [ -n "$git_ref" ]; then + if ! (cd "${output}.git" && git checkout --quiet "$git_ref"); then + log "ERROR: Failed to checkout $git_ref" + return 1 + fi + fi + + if [ -n "$subpath" ]; then + if [ ! -f "${output}.git/$subpath" ]; then + log "ERROR: Subpath $subpath not found in repository" + return 1 + fi + cp "${output}.git/$subpath" "$output" + else + if [ ! -f "${output}.git/docker-compose.yml" ]; then + log "ERROR: docker-compose.yml not found in repository root" + return 1 + fi + cp "${output}.git/docker-compose.yml" "$output" + fi + ;; + *) + log "ERROR: Unsupported tool $tool" + return 1 + ;; + esac +} + +# Function to get version identifier +get_version_id() { + local path=$1 + local tool=$2 + + case $tool in + "git") + if [ -d "${path}.git" ]; then + git -C "${path}.git" rev-parse --short HEAD + else + date +%Y%m%d%H%M%S + fi + ;; + *) + date +%Y%m%d%H%M%S + ;; + esac +} + +# Function to clean up old backup directories +cleanup_old_backups() { + local path=$1 + local keep_versions=$2 + + log "Cleaning up old backup directories (keeping $keep_versions)" + find "$path/backups" -maxdepth 1 -type d -name "backup-*" | sort -r | tail -n +$((keep_versions + 1)) | xargs -r rm -rf +} + +# Function to send webhook notification +send_webhook_notification() { + local event="$1" + local stack_name="$2" + local message="$3" + local version_id="$4" + local diff="$5" + + if [ -z "$NOTIFICATION_WEBHOOK_URL" ]; then + return + fi + + # Escape newlines and quotes in diff for JSON + local diff_json + if [ -n "$diff" ]; then + diff_json=$(echo "$diff" | head -50 | sed 's/\\/\\\\/g; s/\"/\\\"/g; s/$/\\n/' | tr -d '\r' | tr -d '\000') + else + diff_json="" + fi + + local payload="{\n" + payload+=" \"event\": \"$event\",\n" + payload+=" \"stack_name\": \"$stack_name\",\n" + payload+=" \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\",\n" + payload+=" \"message\": \"$message\",\n" + payload+=" \"version_id\": \"$version_id\",\n" + payload+=" \"diff\": \"$diff_json\"\n" + payload+="}" + + curl -s -X POST -H "Content-Type: application/json" -d "$payload" "$NOTIFICATION_WEBHOOK_URL" >/dev/null 2>&1 || true +} + +# Function to process a stack +process_stack() { + local stack_num=$1 + local name_var="STACK_${stack_num}_NAME" + local url_var="STACK_${stack_num}_URL" + local path_var="STACK_${stack_num}_PATH" + local tool_var="STACK_${stack_num}_TOOL" + local interval_var="STACK_${stack_num}_INTERVAL" + local keep_versions_var="STACK_${stack_num}_KEEP_VERSIONS" + local git_subpath_var="STACK_${stack_num}_GIT_SUBPATH" + local git_ref_var="STACK_${stack_num}_GIT_REF" + + local name=${!name_var} + local url=${!url_var} + local path=${!path_var} + local tool=${!tool_var} + local interval=${!interval_var:-$UPDATE_INTERVAL} + local keep_versions=${!keep_versions_var:-$KEEP_VERSIONS} + local git_subpath=${!git_subpath_var} + local git_ref=${!git_ref_var} + + if [ -z "$name" ] || [ -z "$url" ] || [ -z "$path" ] || [ -z "$tool" ]; then + log "Error: Missing required configuration for stack $stack_num" + return 1 + fi + + # Create lock file path + local lock_file="$path/.lock" + + # Try to acquire lock + if ! acquire_lock "$lock_file"; then + log "Warning: Could not acquire lock for stack $name, skipping..." + return 1 + fi + + # Ensure lock is released on exit + trap 'release_lock "$lock_file"' EXIT + + log "Processing stack $name" + + # Create stack directory and backups directory if they don't exist + mkdir -p "$path" + mkdir -p "$path/backups" + + # Download main compose file + local compose_file="$path/docker-compose.yml" + if ! download_file "$url" "$compose_file" "$tool" "$git_subpath" "$git_ref"; then + log "ERROR: Failed to download main compose file for stack $name, skipping..." + return 1 + fi + + # Get version identifier + local version_id=$(get_version_id "$path" "$tool") + log "Version ID: $version_id" + + # Create backup directory for this update + local backup_dir="$path/backups/backup-$(date +%Y%m%d%H%M%S)" + if [ "$DRY_RUN" != "true" ]; then + mkdir -p "$backup_dir" + fi + + # Download extra files if configured + # Support custom ordering via STACK_N_EXTRA_FILES_ORDER (comma-separated list of numbers) + local extra_file_order_var="STACK_${stack_num}_EXTRA_FILES_ORDER" + local extra_file_order=${!extra_file_order_var} + local extra_files=() + if [ -n "$extra_file_order" ]; then + # Custom order specified + IFS=',' read -ra order_arr <<< "$extra_file_order" + for idx in "${order_arr[@]}"; do + idx=$(echo "$idx" | xargs) # trim whitespace + local extra_file_var="STACK_${stack_num}_EXTRA_FILES_${idx}" + local extra_file_url=${!extra_file_var} + if [ -n "$extra_file_url" ]; then + local extra_filename=$(basename "$extra_file_url") + local extra_file_path="$path/$extra_filename" + log "Downloading extra file $idx (custom order): $extra_filename" + if ! download_file "$extra_file_url" "$extra_file_path" "wget"; then + log "ERROR: Failed to download extra file $extra_filename, skipping..." + continue + fi + extra_files+=("$extra_file_path") + # Version the extra file + local versioned_extra_file="$path/compose-${version_id}-${extra_filename}.bak" + if [ "$DRY_RUN" != "true" ]; then + cp "$extra_file_path" "$versioned_extra_file" + cp "$extra_file_path" "$backup_dir/$extra_filename" + else + log "Would create versioned file: $versioned_extra_file" + log "Would backup file to: $backup_dir/$extra_filename" + fi + fi + done + else + # Default: process in numeric order starting from 1 + local extra_file_num=1 + while true; do + local extra_file_var="STACK_${stack_num}_EXTRA_FILES_${extra_file_num}" + local extra_file_url=${!extra_file_var} + if [ -z "$extra_file_url" ]; then + break + fi + local extra_filename=$(basename "$extra_file_url") + local extra_file_path="$path/$extra_filename" + log "Downloading extra file $extra_file_num: $extra_filename" + if ! download_file "$extra_file_url" "$extra_file_path" "wget"; then + log "ERROR: Failed to download extra file $extra_filename, skipping..." + ((extra_file_num++)) + continue + fi + extra_files+=("$extra_file_path") + # Version the extra file + local versioned_extra_file="$path/compose-${version_id}-${extra_filename}.bak" + if [ "$DRY_RUN" != "true" ]; then + cp "$extra_file_path" "$versioned_extra_file" + cp "$extra_file_path" "$backup_dir/$extra_filename" + else + log "Would create versioned file: $versioned_extra_file" + log "Would backup file to: $backup_dir/$extra_filename" + fi + ((extra_file_num++)) + done + fi + + # Create versioned copy of main compose file + local versioned_file="$path/compose-${version_id}.yml.bak" + if [ "$DRY_RUN" != "true" ]; then + cp "$compose_file" "$versioned_file" + # Backup the main compose file + cp "$compose_file" "$backup_dir/docker-compose.yml" + # Backup the override file if it exists + if [ -f "$path/docker-compose.override.yml" ]; then + cp "$path/docker-compose.override.yml" "$backup_dir/docker-compose.override.yml" + fi + else + log "Would create versioned file: $versioned_file" + log "Would backup main compose file to: $backup_dir/docker-compose.yml" + if [ -f "$path/docker-compose.override.yml" ]; then + log "Would backup override file to: $backup_dir/docker-compose.override.yml" + fi + fi + + # Clean up old versions and backups + if [ "$DRY_RUN" != "true" ]; then + log "Cleaning up old versions (keeping $keep_versions)" + find "$path" -name "compose-*.yml.bak" -type f | sort -r | tail -n +$((keep_versions + 1)) | xargs -r rm + cleanup_old_backups "$path" "$keep_versions" + else + log "Would clean up old versions (keeping $keep_versions)" + log "Would clean up old backup directories (keeping $keep_versions)" + fi + + # Check for changes + if [ -f "$compose_file" ] && [ -f "$versioned_file" ]; then + if ! diff -q "$compose_file" "$versioned_file" >/dev/null; then + log "Changes detected in $name" + # Generate diff (truncated to 50 lines) + local diff_output + diff_output=$(diff -u "$versioned_file" "$compose_file" | head -50) + if [ $(diff -u "$versioned_file" "$compose_file" | wc -l) -gt 50 ]; then + diff_output="$diff_output\n(diff truncated, showing first 50 lines)" + fi + if [ "$UPDATE_MODE" = "notify_and_apply" ]; then + if [ "$DRY_RUN" = "true" ]; then + log "DRY-RUN: Would apply changes to $name" + + # Build docker compose command with all files (safer array-based approach) + local compose_cmd_array=("docker" "compose" "-f" "$compose_file") + for extra_file in "${extra_files[@]}"; do + compose_cmd_array+=("-f" "$extra_file") + done + if [ -f "$path/docker-compose.override.yml" ]; then + compose_cmd_array+=("-f" "$path/docker-compose.override.yml") + fi + compose_cmd_array+=("up" "-d") + + log "DRY-RUN: Would run: ${compose_cmd_array[*]}" + log "DRY-RUN: Changes that would be applied:" + echo "$diff_output" + send_webhook_notification "update_success" "$name" "[DRY-RUN] Would apply changes to $name" "$version_id" "$diff_output" + else + log "Applying changes to $name" + + # Build docker compose command with all files (safer array-based approach) + local compose_cmd_array=("docker" "compose" "-f" "$compose_file") + for extra_file in "${extra_files[@]}"; do + compose_cmd_array+=("-f" "$extra_file") + done + if [ -f "$path/docker-compose.override.yml" ]; then + compose_cmd_array+=("-f" "$path/docker-compose.override.yml") + fi + compose_cmd_array+=("up" "-d") + + log "Running: ${compose_cmd_array[*]}" + if "${compose_cmd_array[@]}"; then + log "Successfully updated stack $name" + send_webhook_notification "update_success" "$name" "Successfully updated stack $name" "$version_id" "$diff_output" + else + log "ERROR: Failed to update stack $name, attempting rollback..." + send_webhook_notification "update_failure" "$name" "Failed to update stack $name, rolled back to previous version" "$version_id" "$diff_output" + + # Rollback: restore from backup + if [ -f "$backup_dir/docker-compose.yml" ]; then + cp "$backup_dir/docker-compose.yml" "$compose_file" + log "Restored main compose file from backup" + fi + + # Restore extra files + for extra_file in "${extra_files[@]}"; do + local extra_filename=$(basename "$extra_file") + if [ -f "$backup_dir/$extra_filename" ]; then + cp "$backup_dir/$extra_filename" "$extra_file" + log "Restored $extra_filename from backup" + fi + done + + # Restore override file if it existed + if [ -f "$backup_dir/docker-compose.override.yml" ]; then + cp "$backup_dir/docker-compose.override.yml" "$path/docker-compose.override.yml" + log "Restored override file from backup" + fi + + # Try to restart with rolled back configuration + log "Attempting to restart stack with rolled back configuration..." + if "${compose_cmd_array[@]}"; then + log "Successfully rolled back stack $name" + else + log "CRITICAL: Failed to rollback stack $name - manual intervention required" + fi + fi + fi + else + log "Changes detected but not applied (notify_only mode)" + send_webhook_notification "update_success" "$name" "Changes detected in $name (notify_only mode)" "$version_id" "$diff_output" + fi + else + log "No changes detected in $name" + fi + fi +} + +# Main loop +while true; do + log "Starting update check" + + for ((i=1; i<=STACKS; i++)); do + process_stack $i + done + + log "Update check complete. Sleeping for $UPDATE_INTERVAL seconds" + sleep $UPDATE_INTERVAL +done \ No newline at end of file