Merge pull request #26 from github/integration-tests

Testing overhaul

.github/codeql/codeql-config.yml

@@ -1,4 +1,6 @@
-me: "CodeQL config"
+name: "CodeQL config"
 queries: 
   - name: Run custom queries
     uses: ./queries
+paths-ignore:
+  - tests

.github/workflows/codeql.yml

@@ -13,5 +13,6 @@ jobs:
     - uses: actions/checkout@v1
     - uses: ./init
       with:
-       config-file: ./.github/codeql/codeql-config.yml
+        languages: javascript
+        config-file: ./.github/codeql/codeql-config.yml
     - uses: ./analyze

.github/workflows/integration-testing.yml

@@ -1,22 +1,117 @@
 name: "Integration Testing"
 
-on: [push]
+on: [push, pull_request]
 
 jobs:
-  dispatch-events:
-    if: github.event.repository.full_name == 'github/codeql-action'
-    runs-on: ubuntu-latest
-    steps:
-    - name: Send repository dispatch events
-      run: |
-        curl -X POST \
-          -H "Authorization: Bearer ${{ secrets.CODEQL_TESTING_TOKEN }}" \
-          -H "Accept: application/vnd.github.everest-preview+json" \
-          https://api.github.com/repos/Anthophila/amazon-cognito-js-copy/dispatches \
-          -d '{"event_type":"codeql-integration","client_payload": {"sha": "${{ github.sha }}"}}'
+  multi-language-repo_test-autodetect-languages:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
 
-        curl -X POST \
-          -H "Authorization: Bearer ${{ secrets.CODEQL_TESTING_TOKEN }}" \
-          -H "Accept: application/vnd.github.everest-preview+json" \
-          https://api.github.com/repos/Anthophila/electron-test-action/dispatches \
-          -d '{"event_type":"codeql-integration","client_payload": {"sha": "${{ github.sha }}"}}'
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        shopt -s dotglob
+        mv * ../action/
+        mv ../action/tests/multi-language-repo/* .
+    - uses: ./../action/init
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+
+  multi-language-repo_test-custom-queries:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        shopt -s dotglob
+        mv * ../action/
+        mv ../action/tests/multi-language-repo/* .
+    - uses: ./../action/init
+      with:
+        languages: cpp,csharp,java,javascript,python
+        config-file: ./.github/codeql/custom-queries.yml
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+
+  # Currently is not possible to analyze Go in conjunction with other languages in macos
+  multi-language-repo_test-go-custom-queries:
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+
+    steps:
+    - uses: actions/setup-go@v2
+      if: ${{ matrix.os ==  'macos-latest' }}
+      with:
+        go-version: '^1.13.1'
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        shopt -s dotglob
+        mv * ../action/
+        mv ../action/tests/multi-language-repo/* .
+    - uses: ./../action/init
+      with:
+        languages: go
+        config-file: ./.github/codeql/custom-queries.yml
+    - name: Build code
+      shell: bash
+      run: ./build.sh
+    - uses: ./../action/analyze
+      env:
+        TEST_MODE: true
+
+
+  multi-language-repo_rubocop:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Move codeql-action
+      shell: bash
+      run: |
+        mkdir ../action
+        shopt -s dotglob
+        mv * ../action/
+        mv ../action/tests/multi-language-repo/* .
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: 2.6
+    - name: Install Code Scanning integration
+      run: bundle add code-scanning-rubocop --version 0.2.0 --skip-install
+    - name: Install dependencies
+      run: bundle install
+    - name: Rubocop run
+      run: |
+        bash -c "
+          bundle exec rubocop --require code_scanning --format CodeScanning::SarifFormatter -o rubocop.sarif
+          [[ $? -ne 2 ]]
+        "
+    - uses: ./../action/upload-sarif
+      with:
+        sarif_file: rubocop.sarif
+      env:
+        TEST_MODE: true

.github/workflows/lint-build-test.yml

@@ -1,8 +1,16 @@
-name: "Check generated JavaScript"
+name: "Lint, Build & Test"
 
-on: [pull_request]
+on: [push, pull_request]
 
 jobs:
+  tslint:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: tslint
+      run: npm run-script lint
+
   check-js:
     runs-on: ubuntu-latest
 
@@ -25,3 +33,11 @@ jobs:
           exit 1
         fi
         echo "Success: JavaScript files are up to date"
+
+  npm-test:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: npm run-script test
+      run: npm run-script test

.github/workflows/npm-test.yml

@@ -1,12 +0,0 @@
-name: "npm run-script test"
-
-on: [push]
-
-jobs:
-  npm-test:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: npm run-script test
-      run: npm run-script test

.github/workflows/ts-lint.yml

@@ -1,12 +0,0 @@
-name: "TSLint"
-
-on: [push]
-
-jobs:
-  tslint:
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v1
-    - name: tslint
-      run: npm run-script lint

lib/upload-lib.js

@@ -58,6 +58,11 @@ exports.combineSarifFiles = combineSarifFiles;
 // If the request fails then this will retry a small number of times.
 async function uploadPayload(payload) {
     core.info('Uploading results');
+    // If in test mode we don't want to upload the results
+    const testMode = process.env['TEST_MODE'] === 'true' || false;
+    if (testMode) {
+        return true;
+    }
     const githubToken = core.getInput('token');
     const ph = new auth.BearerCredentialHandler(githubToken);
     const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);

src/upload-lib.ts

@@ -52,6 +52,12 @@ export function combineSarifFiles(sarifFiles: string[]): string {
 async function uploadPayload(payload): Promise<boolean> {
     core.info('Uploading results');
 
+    // If in test mode we don't want to upload the results
+    const testMode = process.env['TEST_MODE'] === 'true' || false;
+    if (testMode) {
+        return true;
+    }
+
     const githubToken = core.getInput('token');
     const ph: auth.BearerCredentialHandler = new auth.BearerCredentialHandler(githubToken);
     const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);

tests/multi-language-repo/.github/codeql/custom-queries.yml

@@ -0,0 +1,29 @@
+name: Use custom queries
+
+disable-default-queries: true
+
+queries: 
+# Query suites
+  - name: Select a query suite 
+    uses: ./codeql-qlpacks/complex-python-qlpack/rootAndBar.qls
+# QL pack subset
+  - name: Select a ql file 
+    uses: ./codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql
+  - name: Select a subfolder
+    uses: ./codeql-qlpacks/complex-javascript-qlpack/foo
+  - name: Select a folder with two subfolders 
+    uses: ./codeql-qlpacks/complex-javascript-qlpack/foo2
+# Inrepo QL pack
+  - name: Select an inrepo ql pack
+    uses: ./codeql-qlpacks/csharp-qlpack
+  - name: Java queries 
+    uses: ./codeql-qlpacks/java-qlpack
+# External QL packs
+  - name: Go queries 
+    uses: Anthophila/go-querypack@master
+  - name: Cpp queries 
+    uses: Anthophila/cpp-querypack@second-branch
+  - name: Javascript queries 
+    uses: Anthophila/javascript-querypack/show_ifs2.ql@master
+  - name: Python queries 
+    uses: Anthophila/python-querypack/show_ifs2.ql@second-branch 

tests/multi-language-repo/Gemfile

@@ -0,0 +1,2 @@
+source "https://rubygems.org" do
+end

tests/multi-language-repo/Main.java

@@ -0,0 +1,8 @@
+class Main {
+    public static void main(String args[]) {
+        if (true) {
+            System.out.println("Hello, World!");
+        }
+    }
+}
+

tests/multi-language-repo/build.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+gcc -o main main.c
+
+dotnet build
+
+javac Main.java
+

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/foo/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Foo Show Ifs
+ * @description Foo Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/foo-show-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "foo if"

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/foo2/bar/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Bar Show Ifs
+ * @description Bar Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/bar-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "bar if"

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/foo2/barfoo/barfoobar/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Barfoobar Show Ifs
+ * @description Barfoobar Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/barfoobar-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "barfoobar if"

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/foo2/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Foo2 Show Ifs
+ * @description Foo2 Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/foo2-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "foo2 if"

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: complex-javascript-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-javascript

tests/multi-language-repo/codeql-qlpacks/complex-javascript-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Root Show Ifs
+ * @description Root Show Ifs
+ * @kind problem
+ * @id complex-javascript-querypack/root-show-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "root if"

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/foo/bar/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Foo/Bar Show Ifs
+ * @description Foo/Bar Show Ifs
+ * @kind problem
+ * @id complex-python-querypack/foo/bar/show-ifs
+ */
+
+import python
+
+from If i
+select i, "foo/bar if"

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/foo/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Foo Show Ifs
+ * @description Foo Show Ifs
+ * @kind problem
+ * @id complex-python-querypack/foo/show-ifs
+ */
+
+import python
+
+from If i
+select i, "foo if"

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: inrepo-python-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-python

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/rootAndBar.qls

@@ -0,0 +1,2 @@
+- query: show_ifs.ql
+- query: foo/bar/show_ifs.ql

tests/multi-language-repo/codeql-qlpacks/complex-python-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Show Ifs
+ * @description Show Ifs
+ * @kind problem
+ * @id complex-python-querypack/show-ifs
+ */
+
+import python
+
+from If i
+select i, "hello if"

tests/multi-language-repo/codeql-qlpacks/cpp-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: inrepo-cpp-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-cpp

tests/multi-language-repo/codeql-qlpacks/cpp-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Show Cpp Ifs
+ * @description Show Cpp Ifs
+ * @kind problem
+ * @id inrepo-cpp-querypack/show-ifs
+ */
+
+import cpp
+
+from IfStmt i
+select i, "hello if"

tests/multi-language-repo/codeql-qlpacks/csharp-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: inrepo-csharp-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-csharp

tests/multi-language-repo/codeql-qlpacks/csharp-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Show Csharp Ifs
+ * @description Show Csharp Ifs
+ * @kind problem
+ * @id inrepo-csharp-querypack/show-ifs
+ */
+
+import csharp
+
+from IfStmt i
+select i, "hello if"

tests/multi-language-repo/codeql-qlpacks/go-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: inrepo-go-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-go

tests/multi-language-repo/codeql-qlpacks/go-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Show Go Ifs
+ * @description Show Go Ifs
+ * @kind problem
+ * @id inrepo-go-querypack/show-ifs
+ */
+
+import go
+
+from IfStmt i
+select i, "hello if"

tests/multi-language-repo/codeql-qlpacks/java-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: inrepo-java-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-java

tests/multi-language-repo/codeql-qlpacks/java-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Show Java Ifs
+ * @description Show Java Ifs
+ * @kind problem
+ * @id inrepo-java-querypack/show-ifs
+ */
+
+import java
+
+from IfStmt i
+select i, "hello if"

tests/multi-language-repo/codeql-qlpacks/javascript-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: inrepo-javascript-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-javascript

tests/multi-language-repo/codeql-qlpacks/javascript-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Show Javascript Ifs
+ * @description Show Javascript Ifs
+ * @kind problem
+ * @id inrepo-javascript-querypack/show-ifs
+ */
+
+import javascript
+
+from IfStmt i
+select i, "hello if"

tests/multi-language-repo/codeql-qlpacks/python-qlpack/qlpack.yml

@@ -0,0 +1,3 @@
+name: inrepo-python-querypack
+version: 0.0.1
+libraryPathDependencies: codeql-python

tests/multi-language-repo/codeql-qlpacks/python-qlpack/show_ifs.ql

@@ -0,0 +1,11 @@
+/**
+ * @name Show Python Ifs
+ * @description Show Python Ifs
+ * @kind problem
+ * @id inrepo-python-querypack/show-ifs
+ */
+
+import python
+
+from If i
+select i, "hello if"

tests/multi-language-repo/csharp.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <RootNamespace>multi_language_test</RootNamespace>
+  </PropertyGroup>
+
+</Project>

tests/multi-language-repo/main.c

@@ -0,0 +1,8 @@
+#include "stdio.h"
+
+int main(int argc, char **argv) {
+    if (1) {
+        printf("Hello, World!\n");
+    }
+}
+

tests/multi-language-repo/main.cs

@@ -0,0 +1,12 @@
+using System;
+
+namespace HelloWorldApp {
+    class Geeks {
+        static void Main(string[] args) {
+            if (true) {
+                Console.WriteLine("Hello World!");
+            }
+        }
+    }
+}
+

tests/multi-language-repo/main.go

@@ -0,0 +1,9 @@
+package main
+
+import "fmt"
+
+func main() {
+	if true {
+		fmt.Println("hello world")
+	}
+}

tests/multi-language-repo/main.js

@@ -0,0 +1,3 @@
+if (true) {
+    console.log("Hello, World!");
+}

tests/multi-language-repo/main.py

@@ -0,0 +1,9 @@
+#!/usr/bin/python3
+
+def main():
+    if True:
+        print("Hello, World!")
+
+if __name__ == '__main__':
+    main()
+

tests/multi-language-repo/main.rb

@@ -0,0 +1,11 @@
+#!/usr/bin/ruby
+# frozen_string_literal: true
+
+def main
+  v = ARGV[0]
+
+  puts 'with arg?' unless v.nil?
+  puts 'hello there'
+end
+
+main