Merge pull request #13 from github/upload_retry

Add retrying to the upload

lib/upload-lib.js

@@ -54,6 +54,52 @@ function combineSarifFiles(sarifFiles) {
     return JSON.stringify(combinedSarif);
 }
 exports.combineSarifFiles = combineSarifFiles;
+// Upload the given payload.
+// If the request fails then this will retry a small number of times.
+async function uploadPayload(payload) {
+    core.info('Uploading results');
+    const githubToken = core.getInput('token');
+    const ph = new auth.BearerCredentialHandler(githubToken);
+    const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);
+    const url = 'https://api.github.com/repos/' + process.env['GITHUB_REPOSITORY'] + '/code-scanning/analysis';
+    // Make up to 4 attempts to upload, and sleep for these
+    // number of seconds between each attempt.
+    // We don't want to backoff too much to avoid wasting action
+    // minutes, but just waiting a little bit could maybe help.
+    const backoffPeriods = [1, 5, 15];
+    for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
+        const res = await client.put(url, payload);
+        core.debug('response status: ' + res.message.statusCode);
+        const statusCode = res.message.statusCode;
+        if (statusCode === 202) {
+            core.info("Successfully uploaded results");
+            return;
+        }
+        const requestID = res.message.headers["x-github-request-id"];
+        // On any other status code that's not 5xx mark the upload as failed
+        if (!statusCode || statusCode < 500 || statusCode >= 600) {
+            core.setFailed('Upload failed (' + requestID + '): (' + statusCode + ') ' + await res.readBody());
+            return;
+        }
+        // On a 5xx status code we may retry the request
+        if (attempt < backoffPeriods.length) {
+            // Log the failure as a warning but don't mark the action as failed yet
+            core.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
+                ') failed (' + requestID + '). Retrying in ' + backoffPeriods[attempt] +
+                ' seconds: (' + statusCode + ') ' + await res.readBody());
+            // Sleep for the backoff period
+            await new Promise(r => setTimeout(r, backoffPeriods[attempt] * 1000));
+            continue;
+        }
+        else {
+            // If the upload fails with 5xx then we assume it is a temporary problem
+            // and not an error that the user has caused or can fix.
+            // We avoid marking the job as failed to avoid breaking CI workflows.
+            core.error('Upload failed (' + requestID + '): (' + statusCode + ') ' + await res.readBody());
+            return;
+        }
+    }
+}
 // Uploads a single sarif file or a directory of sarif files
 // depending on what the path happens to refer to.
 async function upload(input) {
@@ -112,26 +158,8 @@ async function uploadFiles(sarifFiles) {
             "started_at": startedAt,
             "tool_names": toolNames,
         });
-        core.info('Uploading results');
+        // Make the upload
-        const githubToken = core.getInput('token');
+        await uploadPayload(payload);
-        const ph = new auth.BearerCredentialHandler(githubToken);
-        const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);
-        const url = 'https://api.github.com/repos/' + process.env['GITHUB_REPOSITORY'] + '/code-scanning/analysis';
-        const res = await client.put(url, payload);
-        const requestID = res.message.headers["x-github-request-id"];
-        core.debug('response status: ' + res.message.statusCode);
-        if (res.message.statusCode === 500) {
-            // If the upload fails with 500 then we assume it is a temporary problem
-            // with turbo-scan and not an error that the user has caused or can fix.
-            // We avoid marking the job as failed to avoid breaking CI workflows.
-            core.error('Upload failed (' + requestID + '): ' + await res.readBody());
-        }
-        else if (res.message.statusCode !== 202) {
-            core.setFailed('Upload failed (' + requestID + '): ' + await res.readBody());
-        }
-        else {
-            core.info("Successfully uploaded results");
-        }
         // Mark that we have made an upload
         fs.writeFileSync(sentinelFile, '');
     }

src/upload-lib.ts

@@ -47,6 +47,61 @@ export function combineSarifFiles(sarifFiles: string[]): string {
     return JSON.stringify(combinedSarif);
 }
 
+// Upload the given payload.
+// If the request fails then this will retry a small number of times.
+async function uploadPayload(payload) {
+    core.info('Uploading results');
+
+    const githubToken = core.getInput('token');
+    const ph: auth.BearerCredentialHandler = new auth.BearerCredentialHandler(githubToken);
+    const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);
+    const url = 'https://api.github.com/repos/' + process.env['GITHUB_REPOSITORY'] + '/code-scanning/analysis';
+
+    // Make up to 4 attempts to upload, and sleep for these
+    // number of seconds between each attempt.
+    // We don't want to backoff too much to avoid wasting action
+    // minutes, but just waiting a little bit could maybe help.
+    const backoffPeriods = [1, 5, 15];
+
+    for (let attempt = 0; attempt <= backoffPeriods.length; attempt++) {
+
+        const res: http.HttpClientResponse = await client.put(url, payload);
+        core.debug('response status: ' + res.message.statusCode);
+
+        const statusCode = res.message.statusCode;
+        if (statusCode === 202) {
+            core.info("Successfully uploaded results");
+            return;
+        }
+
+        const requestID = res.message.headers["x-github-request-id"];
+
+        // On any other status code that's not 5xx mark the upload as failed
+        if (!statusCode || statusCode < 500 || statusCode >= 600) {
+            core.setFailed('Upload failed (' + requestID + '): (' + statusCode + ') ' + await res.readBody());
+            return;
+        }
+
+        // On a 5xx status code we may retry the request
+        if (attempt < backoffPeriods.length) {
+            // Log the failure as a warning but don't mark the action as failed yet
+            core.warning('Upload attempt (' + (attempt + 1) + ' of ' + (backoffPeriods.length + 1) +
+              ') failed (' + requestID + '). Retrying in ' + backoffPeriods[attempt] +
+              ' seconds: (' + statusCode + ') ' + await res.readBody());
+            // Sleep for the backoff period
+            await new Promise(r => setTimeout(r, backoffPeriods[attempt] * 1000));
+            continue;
+
+        } else {
+            // If the upload fails with 5xx then we assume it is a temporary problem
+            // and not an error that the user has caused or can fix.
+            // We avoid marking the job as failed to avoid breaking CI workflows.
+            core.error('Upload failed (' + requestID + '): (' + statusCode + ') ' + await res.readBody());
+            return;
+        }
+    }
+}
+
 // Uploads a single sarif file or a directory of sarif files
 // depending on what the path happens to refer to.
 export async function upload(input: string) {
@@ -112,25 +167,8 @@ async function uploadFiles(sarifFiles: string[]) {
             "tool_names": toolNames,
         });
 
-        core.info('Uploading results');
+        // Make the upload
-        const githubToken = core.getInput('token');
+        await uploadPayload(payload);
-        const ph: auth.BearerCredentialHandler = new auth.BearerCredentialHandler(githubToken);
-        const client = new http.HttpClient('Code Scanning : Upload SARIF', [ph]);
-        const url = 'https://api.github.com/repos/' + process.env['GITHUB_REPOSITORY'] + '/code-scanning/analysis';
-        const res: http.HttpClientResponse = await client.put(url, payload);
-        const requestID = res.message.headers["x-github-request-id"];
-
-        core.debug('response status: ' + res.message.statusCode);
-        if (res.message.statusCode === 500) {
-            // If the upload fails with 500 then we assume it is a temporary problem
-            // with turbo-scan and not an error that the user has caused or can fix.
-            // We avoid marking the job as failed to avoid breaking CI workflows.
-            core.error('Upload failed (' + requestID + '): ' + await res.readBody());
-        } else if (res.message.statusCode !== 202) {
-            core.setFailed('Upload failed (' + requestID + '): ' + await res.readBody());
-        } else {
-            core.info("Successfully uploaded results");
-        }
 
         // Mark that we have made an upload
         fs.writeFileSync(sentinelFile, '');