Merge pull request #2350 from github/angelapwen/add-exclude-pr-check-param

PR check generator: add `excludeOsAndVersionCombination`

.github/workflows/__multi-language-autodetect.yml

@@ -39,10 +39,16 @@ jobs:
             version: stable-v2.16.6
           - os: macos-latest
             version: default
+          - os: ubuntu-latest
+            version: default
           - os: macos-latest
             version: linked
+          - os: ubuntu-latest
+            version: linked
           - os: macos-latest
             version: nightly-latest
+          - os: ubuntu-latest
+            version: nightly-latest
     name: Multi-language repository
     permissions:
       contents: read

.github/workflows/__scaling-reserved-ram.yml

@@ -39,10 +39,16 @@ jobs:
             version: stable-v2.16.6
           - os: macos-latest
             version: default
+          - os: ubuntu-latest
+            version: default
           - os: macos-latest
             version: linked
+          - os: ubuntu-latest
+            version: linked
           - os: macos-latest
             version: nightly-latest
+          - os: ubuntu-latest
+            version: nightly-latest
     name: Scaling reserved RAM
     permissions:
       contents: read

.github/workflows/__swift-custom-build.yml

@@ -29,10 +29,16 @@ jobs:
         include:
           - os: macos-latest
             version: linked
+          - os: ubuntu-latest
+            version: linked
           - os: macos-latest
             version: default
+          - os: ubuntu-latest
+            version: default
           - os: macos-latest
             version: nightly-latest
+          - os: ubuntu-latest
+            version: nightly-latest
     name: Swift analysis using a custom build command
     permissions:
       contents: read

.github/workflows/__unset-environment.yml

@@ -27,17 +27,11 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - os: macos-12
+          - os: ubuntu-latest
-            version: stable-v2.14.6
-          - os: macos-latest
-            version: stable-v2.15.5
-          - os: macos-latest
-            version: stable-v2.16.6
-          - os: macos-latest
-            version: linked
-          - os: macos-latest
             version: default
-          - os: macos-latest
+          - os: ubuntu-latest
+            version: linked
+          - os: ubuntu-latest
             version: nightly-latest
     name: Test unsetting environment variables
     permissions:

.github/workflows/debug-artifacts.yml

@@ -22,12 +22,10 @@ jobs:
       fail-fast: false
       matrix:
         version:
-        # TODO: Once CLI v2.17.4 is available and the platform is switched back to ubuntu,
+        - stable-20230403
-        #       stable-20230403, stable-v2.13.5, and stable-v2.14.6 can be added back to this matrix,
+        - stable-v2.13.5
-        #       and the VERSIONS variable in the bash script below. 
+        - stable-v2.14.6
-        #       Prior to CLI v2.15.1, ARM runners were not supported by the build tracer.
         - stable-v2.15.5
-        - stable-v2.16.6
         - default
         - linked
         - nightly-latest
@@ -35,7 +33,7 @@ jobs:
     env:
       CODEQL_ACTION_TEST_MODE: true
     timeout-minutes: 45
-    runs-on: macos-latest # TODO: Switch back to ubuntu for `nightly-latest` and `linked` once CLI v2.17.4 is available.
+    runs-on: ubuntu-latest
     steps:
       - name: Check out repository
         uses: actions/checkout@v4
@@ -73,7 +71,7 @@ jobs:
       - name: Check expected artifacts exist
         shell: bash
         run: |
-          VERSIONS="stable-v2.15.5 stable-v2.16.6 default linked nightly-latest"
+          VERSIONS="stable-20230403 stable-v2.13.5 stable-v2.14.6 stable-v2.15.5 default linked nightly-latest"
           LANGUAGES="cpp csharp go java javascript python"
           for version in $VERSIONS; do
             pushd "./my-debug-artifacts-${version//./}"

pr-checks/checks/multi-language-autodetect.yml

@@ -1,7 +1,14 @@
 name: "Multi-language repository"
 description: "An end-to-end integration test of a multi-language repository using automatic language detection"
-# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available.
+operatingSystems: ["macos", "ubuntu"]
-operatingSystems: ["macos"] 
+excludeOsAndVersionCombination: [ 
+  # Known failure for Swift on Linux before CLI v2.17.4.
+  [ "ubuntu", "stable-20230403" ], 
+  [ "ubuntu", "stable-v2.13.5" ], 
+  [ "ubuntu", "stable-v2.14.6" ], 
+  [ "ubuntu", "stable-v2.15.5" ],
+  [ "ubuntu", "stable-v2.16.6" ],
+]
 steps:
   - uses: actions/setup-go@v5
     with:

pr-checks/checks/scaling-reserved-ram.yml

@@ -1,7 +1,14 @@
 name: "Scaling reserved RAM"
 description: "An end-to-end integration test of a multi-language repository with the scaling_reserved_ram feature flag enabled"
-# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available.
+operatingSystems: ["macos", "ubuntu"]
-operatingSystems: ["macos"]
+excludeOsAndVersionCombination: [ 
+  # Known failure for Swift on Linux before CLI v2.17.4.
+  [ "ubuntu", "stable-20230403" ], 
+  [ "ubuntu", "stable-v2.13.5" ], 
+  [ "ubuntu", "stable-v2.14.6" ], 
+  [ "ubuntu", "stable-v2.15.5" ],
+  [ "ubuntu", "stable-v2.16.6" ],
+]
 env:
   CODEQL_ACTION_SCALING_RESERVED_RAM: true
 steps:

pr-checks/checks/swift-custom-build.yml

@@ -1,8 +1,7 @@
 name: "Swift analysis using a custom build command"
 description: "Tests creation of a Swift database using custom build"
 versions: ["linked", "default", "nightly-latest"]
-# TODO: Add ubuntu back for `nightly-latest` and `latest` once CLI v2.17.4 is available.
+operatingSystems: ["macos", "ubuntu"]
-operatingSystems: ["macos"] 
 env:
   DOTNET_GENERATE_ASPNET_CERTIFICATE: "false"
 steps:

pr-checks/checks/unset-environment.yml

@@ -1,8 +1,15 @@
 name: "Test unsetting environment variables"
 description: "An end-to-end integration test that unsets some environment variables"
-# TODO: Switch back to all versions once CLI v2.17.4 is available and running on ubuntu again.
+operatingSystems: ["ubuntu"]
-versions: ["stable-v2.14.6", "stable-v2.15.5", "stable-v2.16.6", "linked", "default", "nightly-latest"] 
+excludeOsAndVersionCombination: [ 
-operatingSystems: ["macos"] # TODO: Switch back to ubuntu for `nightly-latest` and `latest` once CLI v2.17.4 is available.
+  # Known failure for Swift on Linux before CLI v2.17.4.
+  [ "ubuntu", "stable-20230403" ], 
+  [ "ubuntu", "stable-v2.13.5" ], 
+  [ "ubuntu", "stable-v2.14.6" ], 
+  [ "ubuntu", "stable-v2.15.5" ],
+  [ "ubuntu", "stable-v2.16.6" ],
+]
+
 steps:
   - uses: ./../action/init
     id: init

pr-checks/sync.py

@@ -27,6 +27,12 @@ defaultTestVersions = [
     "nightly-latest"
 ]
 
+def is_os_and_version_excluded(os, version, exclude_params):
+    for exclude_param in exclude_params:
+        if exclude_param[0] == os and exclude_param[1] == version:
+            return True
+    return False
+
 # When updating the ruamel.yaml version here, update the PR check in
 # `.github/workflows/pr-checks.yml` too.
 header = """# Warning: This file is generated automatically, and should not be modified.
@@ -56,27 +62,32 @@ allJobs = {}
 for file in (this_dir / 'checks').glob('*.yml'):
     with open(file, 'r') as checkStream:
         checkSpecification = yaml.load(checkStream)
-
     matrix = []
+    excludedOsesAndVersions = checkSpecification.get('excludeOsAndVersionCombination', [])
     for version in checkSpecification.get('versions', defaultTestVersions):
-        runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"]
+        runnerImages = ["ubuntu-latest", "macos-latest", "windows-latest"]            
-        if checkSpecification.get('operatingSystems', None):
+        operatingSystems = checkSpecification.get('operatingSystems', ["ubuntu", "macos", "windows"])
-            runnerImages = [image for image in runnerImages for operatingSystem in checkSpecification['operatingSystems']
-                            if image.startswith(operatingSystem)]
 
-        for runnerImage in runnerImages:
+        for operatingSystem in operatingSystems:
-            # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer.
+            runnerImagesForOs = [image for image in runnerImages if image.startswith(operatingSystem)]
-            # "macos-latest" is now an ARM runner, so we run tests on the old CLIs on Intel runners instead.
+
-            if version in ["stable-20230403", "stable-v2.13.4", "stable-v2.13.5", "stable-v2.14.6"] and runnerImage == "macos-latest":
+            for runnerImage in runnerImagesForOs:
-                matrix.append({
+                # Skip appending this combination to the matrix if it is explicitly excluded.
-                    'os': "macos-12",
+                if is_os_and_version_excluded(operatingSystem, version, excludedOsesAndVersions):
-                    'version': version
+                    continue
-                })
+
-            else:
+                # Prior to CLI v2.15.1, ARM runners were not supported by the build tracer.
-                matrix.append({
+                # "macos-latest" is now an ARM runner, so we run tests on the old CLIs on Intel runners instead.
-                    'os': runnerImage,
+                if version in ["stable-20230403", "stable-v2.13.4", "stable-v2.13.5", "stable-v2.14.6"] and runnerImage == "macos-latest":
-                    'version': version
+                    matrix.append({
-                })
+                        'os': "macos-12",
+                        'version': version
+                    })
+                else:
+                    matrix.append({
+                        'os': runnerImage,
+                        'version': version
+                    })
 
         useAllPlatformBundle = "false" # Default to false
         if checkSpecification.get('useAllPlatformBundle'):