ci: Add basic shellcheck, whitespace & format checks

.gitlab-ci.yml

@@ -10,6 +10,17 @@ variables:
   image: quay.io/buildah/stable:v1.40.1
   needs: []
 
+basic-checks:
+  stage: build
+  image: quay.io/fedora/fedora:latest
+  needs: []
+  script: |
+    set -xeuo pipefail
+    dnf install -y file jq python3-yaml ShellCheck
+    ./ci/find-whitespace
+    ./ci/shellcheck
+    ./ci/validate
+
 build-minimal:
   extends: .build-image
   script: |

ci/find-whitespace

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+set -euo pipefail
+
+main() {
+
+    local files_with_whitespace=""
+    local files_with_missing_empty_line_at_eof=""
+
+    while IFS= read -r -d '' f; do
+        echo "[+] Checking ${f}"
+
+        # Looking for whitespace at end of line
+        if grep -Eq " +$" "${f}"; then
+            # List of files to ignore
+            if \
+                [[ "${f}" == "./live/isolinux/boot.msg" ]] \
+                ; then
+                echo "[+] Checking ${f}: Ignoring whitespace at end of line"
+            else
+                echo "[+] Checking ${f}: Found whitespace at end of line"
+                files_with_whitespace+=" ${f}"
+            fi
+        fi
+
+        # Looking for missing empty line at end of file
+        if [[ -n $(tail -c 1 "${f}") ]]; then
+            # List of files to ignore
+            if \
+                [[ "${f}" == "./tests/kola/ignition/resource/authenticated-gs/data/expected/"* ]] ||\
+                [[ "${f}" == "./tests/kola/ignition/resource/authenticated-s3/data/expected/"* ]] ||\
+                [[ "${f}" == "./tests/kola/ignition/resource/remote/data/expected/"* ]] \
+                ; then
+                echo "[+] Checking ${f}: Ignoring missing empty line at end of file"
+            else
+                echo "[+] Checking ${f}: Missing empty line at end of file"
+                files_with_missing_empty_line_at_eof+=" ${f}"
+            fi
+        fi
+    done< <(find . -path "./.git" -prune -o -type f -print0)
+
+    echo ""
+    if [[ -n "${files_with_whitespace}" ]]; then
+        echo "[+] Found files with whitespace at the end of line"
+        echo "${files_with_whitespace}" | tr ' ' '\n'
+    else
+        echo "[+] No files with whitespace at the end of line"
+    fi
+
+    echo ""
+    if [[ -n "${files_with_missing_empty_line_at_eof}" ]]; then
+        echo "[+] Found files with missing empty line at end of file"
+        echo "${files_with_missing_empty_line_at_eof}" | tr ' ' '\n'
+    else
+        echo "[+] No files with missing empty line at end of file"
+    fi
+
+    if [[ -n "${files_with_whitespace}" ]] || [[ -n "${files_with_missing_empty_line_at_eof}" ]]; then
+        exit 1
+    fi
+
+    exit 0
+}
+
+main "${@}"

ci/shellcheck

@@ -0,0 +1,35 @@
+#!/bin/bash
+# Template generated by https://github.com/coreos/repo-templates; do not edit downstream
+
+set -euo pipefail
+
+main() {
+    local found_errors="false"
+    # Let's start with error, then we can do warning, info, style
+    local -r severity="error"
+
+    while IFS= read -r -d '' f; do
+        # Skip non-text files that are very unlikely to be shell scripts
+        if [[ "$(file -b --mime-type "${f}" | sed 's|/.*||')" != "text" ]]; then
+            continue
+        fi
+        shebang="$(head -1 "${f}")"
+        if [[ "${f}" == *.sh ]] || \
+            [[ ${shebang} =~ ^#!/.*/bash.* ]] || \
+            [[ ${shebang} =~ ^#!/.*/env\ bash ]]; then
+            echo "[+] Checking ${f}"
+            shellcheck --external-sources --shell bash --severity="${severity}" "${f}" || found_errors="true"
+            bash -n "${f}" || found_errors="true"
+        fi
+    done< <(find . -path "./.git" -prune -o -path "./vendor" -prune -o -type f -print0)
+
+    if [[ "${found_errors}" != "false" ]]; then
+        echo "[+] Found errors with ShellCheck"
+        exit 1
+    fi
+
+    echo "[+] No error found with ShellCheck"
+    exit 0
+}
+
+main "${@}"

ci/validate

@@ -0,0 +1,71 @@
+#!/usr/bin/python3
+# Validate basic syntax of shell script and yaml.
+
+import os
+import re
+import stat
+import subprocess
+import yaml
+
+INITRD_SERVICES_WITHOUT_BEFORE = {
+    # Depended on by other services
+    'coreos-livepxe-rootfs.service',
+}
+
+validated=0
+
+def openat(dirfd, name, mode='r'):
+    def opener(path, flags):
+        return os.open(path, flags, dir_fd=dirfd)
+    return open(name, mode, opener=opener)
+
+
+def validate_initrd_service(rootfd, name):
+    with openat(rootfd, name) as fh:
+        if ([l for l in fh.readlines() if l.startswith('Before=')] or
+                name in INITRD_SERVICES_WITHOUT_BEFORE):
+            global validated
+            validated += 1
+        else:
+            raise Exception(
+                f'{name} has no Before= and may race with switch-root'
+            )
+
+
+BASH_UNBRACKETED_IF = re.compile(r'\sif\s+"?\$')
+def validate_shell(rootfd, name):
+    subprocess.check_call(['bash', '-n', name], preexec_fn=lambda: os.fchdir(rootfd))
+    with openat(rootfd, name) as fh:
+        if BASH_UNBRACKETED_IF.search(fh.read()):
+            raise Exception(f'Possible unbracketed conditional in {name}')
+    global validated
+    validated +=1
+
+
+for root, dirs, files, rootfd in os.fwalk('.'):
+    # Skip .git
+    if '.git' in dirs:
+        dirs.remove('.git')
+    for name in files:
+        print(f"[+] Looking at {name}")
+        if name.endswith(('.yaml', '.yml')):
+            with open(os.open(name, dir_fd=rootfd, flags=os.O_RDONLY)) as f:
+                yaml.safe_load(f)
+                validated +=1
+            continue
+        elif name.endswith('.sh'):
+            validate_shell(rootfd, name)
+            continue
+        elif 'dracut/modules.d' in root and name.endswith('.service'):
+            validate_initrd_service(rootfd, name)
+        stbuf = os.lstat(name, dir_fd=rootfd)
+        if not stat.S_ISREG(stbuf.st_mode):
+            continue
+        if not stbuf.st_mode & stat.S_IXUSR:
+            continue
+        mimetype = subprocess.check_output(['file', '-b', '--mime-type', name], encoding='UTF-8',
+                                        preexec_fn=lambda: os.fchdir(rootfd)).strip()
+        if mimetype == 'text/x-shellscript':
+            validate_shell(rootfd, name)
+
+print(f"Validated {validated} files")