Merge pull request #1853 from jeetchoudhary/main

Fix API calls failing because of secondary api limits

src/main/java/org/kohsuke/github/GitHubAbuseLimitHandler.java

@@ -28,9 +28,47 @@ public abstract class GitHubAbuseLimitHandler extends GitHubConnectorResponseErr
      *             Signals that an I/O exception has occurred.
      */
     @Override
-    boolean isError(@Nonnull GitHubConnectorResponse connectorResponse) throws IOException {
-        return connectorResponse.statusCode() == HttpURLConnection.HTTP_FORBIDDEN
-                && connectorResponse.header("Retry-After") != null;
+    boolean isError(@Nonnull GitHubConnectorResponse connectorResponse) {
+        return isForbidden(connectorResponse) && hasRetryOrLimitHeader(connectorResponse);
+    }
+
+    /**
+     * Checks if the response status code is HTTP_FORBIDDEN (403).
+     *
+     * @param connectorResponse
+     *            the response from the GitHub connector
+     * @return true if the status code is HTTP_FORBIDDEN
+     */
+    private boolean isForbidden(GitHubConnectorResponse connectorResponse) {
+        return connectorResponse.statusCode() == HttpURLConnection.HTTP_FORBIDDEN;
+    }
+
+    /**
+     * Checks if the response contains either "Retry-After" or "gh-limited-by" headers. GitHub does not guarantee the
+     * presence of the Retry-After header. However, the gh-limited-by header is included in the response when the error
+     * is due to rate limiting
+     *
+     * @param connectorResponse
+     *            the response from the GitHub connector
+     * @return true if either "Retry-After" or "gh-limited-by" headers are present
+     * @see <a href=
+     *      "https://docs.github.com/en/rest/using-the-rest-api/best-practices-for-using-the-rest-api?apiVersion=2022-11-28#handle-rate-limit-errors-appropriately</a>
+     */
+    private boolean hasRetryOrLimitHeader(GitHubConnectorResponse connectorResponse) {
+        return hasHeader(connectorResponse, "Retry-After") || hasHeader(connectorResponse, "gh-limited-by");
+    }
+
+    /**
+     * Checks if the response contains a specific header.
+     *
+     * @param connectorResponse
+     *            the response from the GitHub connector
+     * @param headerName
+     *            the name of the header to check for
+     * @return true if the specified header is present
+     */
+    private boolean hasHeader(GitHubConnectorResponse connectorResponse, String headerName) {
+        return connectorResponse.header(headerName) != null;
     }
 
     /**

src/test/java/org/kohsuke/github/AbuseLimitHandlerTest.java

@@ -307,4 +307,72 @@ public void onError(IOException e, HttpURLConnection uc) throws IOException {
         assertThat(mockGitHub.getRequestCount(), equalTo(4));
     }
 
+    /**
+     * Tests the behavior of the GitHub API client when the abuse limit handler is set to WAIT then the handler waits
+     * appropriately when secondary rate limits are encountered.
+     *
+     * @throws Exception
+     *             if any error occurs during the test execution.
+     */
+    @Test
+    public void testHandler_Wait_Secondary_Limits() throws Exception {
+        // Customized response that templates the date to keep things working
+        snapshotNotAllowed();
+        final HttpURLConnection[] savedConnection = new HttpURLConnection[1];
+        gitHub = getGitHubBuilder().withEndpoint(mockGitHub.apiServer().baseUrl())
+                .withAbuseLimitHandler(new AbuseLimitHandler() {
+                    /**
+                     * Overriding method because the actual method will wait for one minute causing slowness in unit
+                     * tests
+                     */
+                    @Override
+                    public void onError(IOException e, HttpURLConnection uc) throws IOException {
+                        savedConnection[0] = uc;
+                        // Verify
+                        assertThat(uc.getDate(), Matchers.greaterThanOrEqualTo(new Date().getTime() - 10000));
+                        assertThat(uc.getExpiration(), equalTo(0L));
+                        assertThat(uc.getIfModifiedSince(), equalTo(0L));
+                        assertThat(uc.getLastModified(), equalTo(1581014017000L));
+                        assertThat(uc.getRequestMethod(), equalTo("GET"));
+                        assertThat(uc.getResponseCode(), equalTo(403));
+                        assertThat(uc.getResponseMessage(), containsString("Forbidden"));
+                        assertThat(uc.getURL().toString(),
+                                endsWith("/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits"));
+                        assertThat(uc.getHeaderFieldInt("X-RateLimit-Limit", 10), equalTo(5000));
+                        assertThat(uc.getHeaderFieldInt("X-RateLimit-Remaining", 10), equalTo(4000));
+                        assertThat(uc.getHeaderFieldInt("X-Foo", 20), equalTo(20));
+                        assertThat(uc.getHeaderFieldLong("X-RateLimit-Limit", 15L), equalTo(5000L));
+                        assertThat(uc.getHeaderFieldLong("X-RateLimit-Remaining", 15L), equalTo(4000L));
+                        assertThat(uc.getHeaderFieldLong("X-Foo", 20L), equalTo(20L));
+                        assertThat(uc.getHeaderField("gh-limited-by"), equalTo("search-elapsed-time-shared-grouped"));
+                        assertThat(uc.getContentEncoding(), nullValue());
+                        assertThat(uc.getContentType(), equalTo("application/json; charset=utf-8"));
+                        assertThat(uc.getContentLength(), equalTo(-1));
+                        assertThat(uc.getHeaderFields(), instanceOf(Map.class));
+                        assertThat(uc.getHeaderFields().size(), Matchers.greaterThan(25));
+                        assertThat(uc.getHeaderField("Status"), equalTo("403 Forbidden"));
+
+                        try (InputStream errorStream = uc.getErrorStream()) {
+                            assertThat(errorStream, notNullValue());
+                            String errorString = IOUtils.toString(errorStream, StandardCharsets.UTF_8);
+                            assertThat(errorString,
+                                    containsString(
+                                            "You have exceeded a secondary rate limit. Please wait a few minutes before you try again"));
+                        }
+                        AbuseLimitHandler.FAIL.onError(e, uc);
+                    }
+                })
+                .build();
+
+        gitHub.getMyself();
+        assertThat(mockGitHub.getRequestCount(), equalTo(1));
+        try {
+            getTempRepository();
+            fail();
+        } catch (Exception e) {
+            assertThat(e, instanceOf(HttpException.class));
+            assertThat(e.getMessage(), equalTo("Abuse limit reached"));
+        }
+        assertThat(mockGitHub.getRequestCount(), equalTo(2));
+    }
 }

src/test/resources/org/kohsuke/github/AbuseLimitHandlerTest/wiremock/testHandler_Wait_Secondary_Limits/__files/1-user.json

@@ -0,0 +1,45 @@
+{
+  "login": "bitwiseman",
+  "id": 1958953,
+  "node_id": "MDQ6VXNlcjE5NTg5NTM=",
+  "avatar_url": "https://avatars3.githubusercontent.com/u/1958953?v=4",
+  "gravatar_id": "",
+  "url": "https://api.github.com/users/bitwiseman",
+  "html_url": "https://github.com/bitwiseman",
+  "followers_url": "https://api.github.com/users/bitwiseman/followers",
+  "following_url": "https://api.github.com/users/bitwiseman/following{/other_user}",
+  "gists_url": "https://api.github.com/users/bitwiseman/gists{/gist_id}",
+  "starred_url": "https://api.github.com/users/bitwiseman/starred{/owner}{/repo}",
+  "subscriptions_url": "https://api.github.com/users/bitwiseman/subscriptions",
+  "organizations_url": "https://api.github.com/users/bitwiseman/orgs",
+  "repos_url": "https://api.github.com/users/bitwiseman/repos",
+  "events_url": "https://api.github.com/users/bitwiseman/events{/privacy}",
+  "received_events_url": "https://api.github.com/users/bitwiseman/received_events",
+  "type": "User",
+  "site_admin": false,
+  "name": "Liam Newman",
+  "company": "Cloudbees, Inc.",
+  "blog": "",
+  "location": "Seattle, WA, USA",
+  "email": "[email protected]",
+  "hireable": null,
+  "bio": "https://twitter.com/bitwiseman",
+  "public_repos": 181,
+  "public_gists": 7,
+  "followers": 146,
+  "following": 9,
+  "created_at": "2012-07-11T20:38:33Z",
+  "updated_at": "2020-02-06T17:29:39Z",
+  "private_gists": 8,
+  "total_private_repos": 10,
+  "owned_private_repos": 0,
+  "disk_usage": 33697,
+  "collaborators": 0,
+  "two_factor_authentication": true,
+  "plan": {
+    "name": "free",
+    "space": 976562499,
+    "collaborators": 0,
+    "private_repos": 10000
+  }
+}

src/test/resources/org/kohsuke/github/AbuseLimitHandlerTest/wiremock/testHandler_Wait_Secondary_Limits/__files/3-r_h_t_fail.json

@@ -0,0 +1,126 @@
+{
+  "id": 238757196,
+  "node_id": "MDEwOlJlcG9zaXRvcnkyMzg3NTcxOTY=",
+  "name": "temp-testHandler_Wait_Secondary_Limits",
+  "full_name": "hub4j-test-org/temp-testHandler_Wait_Secondary_Limits",
+  "private": false,
+  "owner": {
+    "login": "hub4j-test-org",
+    "id": 7544739,
+    "node_id": "MDEyOk9yZ2FuaXphdGlvbjc1NDQ3Mzk=",
+    "avatar_url": "https://avatars3.githubusercontent.com/u/7544739?v=4",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/hub4j-test-org",
+    "html_url": "https://github.com/hub4j-test-org",
+    "followers_url": "https://api.github.com/users/hub4j-test-org/followers",
+    "following_url": "https://api.github.com/users/hub4j-test-org/following{/other_user}",
+    "gists_url": "https://api.github.com/users/hub4j-test-org/gists{/gist_id}",
+    "starred_url": "https://api.github.com/users/hub4j-test-org/starred{/owner}{/repo}",
+    "subscriptions_url": "https://api.github.com/users/hub4j-test-org/subscriptions",
+    "organizations_url": "https://api.github.com/users/hub4j-test-org/orgs",
+    "repos_url": "https://api.github.com/users/hub4j-test-org/repos",
+    "events_url": "https://api.github.com/users/hub4j-test-org/events{/privacy}",
+    "received_events_url": "https://api.github.com/users/hub4j-test-org/received_events",
+    "type": "Organization",
+    "site_admin": false
+  },
+  "html_url": "https://github.com/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits",
+  "description": "A test repository for testing the github-api project: temp-testHandler_Wait_Secondary_Limits",
+  "fork": false,
+  "url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits",
+  "forks_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/forks",
+  "keys_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/keys{/key_id}",
+  "collaborators_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/collaborators{/collaborator}",
+  "teams_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/teams",
+  "hooks_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/hooks",
+  "issue_events_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/issues/events{/number}",
+  "events_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/events",
+  "assignees_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/assignees{/user}",
+  "branches_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/branches{/branch}",
+  "tags_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/tags",
+  "blobs_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/git/blobs{/sha}",
+  "git_tags_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/git/tags{/sha}",
+  "git_refs_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/git/refs{/sha}",
+  "trees_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/git/trees{/sha}",
+  "statuses_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/statuses/{sha}",
+  "languages_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/languages",
+  "stargazers_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/stargazers",
+  "contributors_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/contributors",
+  "subscribers_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/subscribers",
+  "subscription_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/subscription",
+  "commits_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/commits{/sha}",
+  "git_commits_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/git/commits{/sha}",
+  "comments_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/comments{/number}",
+  "issue_comment_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/issues/comments{/number}",
+  "contents_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/contents/{+path}",
+  "compare_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/compare/{base}...{head}",
+  "merges_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/merges",
+  "archive_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/{archive_format}{/ref}",
+  "downloads_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/downloads",
+  "issues_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/issues{/number}",
+  "pulls_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/pulls{/number}",
+  "milestones_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/milestones{/number}",
+  "notifications_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/notifications{?since,all,participating}",
+  "labels_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/labels{/name}",
+  "releases_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/releases{/id}",
+  "deployments_url": "https://api.github.com/repos/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits/deployments",
+  "created_at": "2020-02-06T18:33:39Z",
+  "updated_at": "2020-02-06T18:33:43Z",
+  "pushed_at": "2020-02-06T18:33:41Z",
+  "git_url": "git://github.com/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits.git",
+  "ssh_url": "[email protected]:hub4j-test-org/temp-testHandler_Wait_Secondary_Limits.git",
+  "clone_url": "https://github.com/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits.git",
+  "svn_url": "https://github.com/hub4j-test-org/temp-testHandler_Wait_Secondary_Limits",
+  "homepage": "http://github-api.kohsuke.org/",
+  "size": 0,
+  "stargazers_count": 0,
+  "watchers_count": 0,
+  "language": null,
+  "has_issues": true,
+  "has_projects": true,
+  "has_downloads": true,
+  "has_wiki": true,
+  "has_pages": false,
+  "forks_count": 0,
+  "mirror_url": null,
+  "archived": false,
+  "disabled": false,
+  "open_issues_count": 0,
+  "license": null,
+  "forks": 0,
+  "open_issues": 0,
+  "watchers": 0,
+  "default_branch": "main",
+  "permissions": {
+    "admin": true,
+    "push": true,
+    "pull": true
+  },
+  "temp_clone_token": "",
+  "allow_squash_merge": true,
+  "allow_merge_commit": true,
+  "allow_rebase_merge": true,
+  "delete_branch_on_merge": false,
+  "organization": {
+    "login": "hub4j-test-org",
+    "id": 7544739,
+    "node_id": "MDEyOk9yZ2FuaXphdGlvbjc1NDQ3Mzk=",
+    "avatar_url": "https://avatars3.githubusercontent.com/u/7544739?v=4",
+    "gravatar_id": "",
+    "url": "https://api.github.com/users/hub4j-test-org",
+    "html_url": "https://github.com/hub4j-test-org",
+    "followers_url": "https://api.github.com/users/hub4j-test-org/followers",
+    "following_url": "https://api.github.com/users/hub4j-test-org/following{/other_user}",
+    "gists_url": "https://api.github.com/users/hub4j-test-org/gists{/gist_id}",
+    "starred_url": "https://api.github.com/users/hub4j-test-org/starred{/owner}{/repo}",
+    "subscriptions_url": "https://api.github.com/users/hub4j-test-org/subscriptions",
+    "organizations_url": "https://api.github.com/users/hub4j-test-org/orgs",
+    "repos_url": "https://api.github.com/users/hub4j-test-org/repos",
+    "events_url": "https://api.github.com/users/hub4j-test-org/events{/privacy}",
+    "received_events_url": "https://api.github.com/users/hub4j-test-org/received_events",
+    "type": "Organization",
+    "site_admin": false
+  },
+  "network_count": 0,
+  "subscribers_count": 6
+}

src/test/resources/org/kohsuke/github/AbuseLimitHandlerTest/wiremock/testHandler_Wait_Secondary_Limits/mappings/1-user.json

@@ -0,0 +1,48 @@
+{
+  "id": "a60baf84-5b5c-4f86-af3d-cab0d609c7b2",
+  "name": "user",
+  "request": {
+    "url": "/user",
+    "method": "GET",
+    "headers": {
+      "Accept": {
+        "equalTo": "application/vnd.github.v3+json"
+      }
+    }
+  },
+  "response": {
+    "status": 200,
+    "bodyFileName": "1-user.json",
+    "headers": {
+      "Date": "{{now timezone='GMT' format='EEE, dd MMM yyyy HH:mm:ss z'}}",
+      "Content-Type": "application/json; charset=utf-8",
+      "Server": "GitHub.com",
+      "Status": "200 OK",
+      "X-RateLimit-Limit": "5000",
+      "X-RateLimit-Remaining": "4930",
+      "X-RateLimit-Reset": "{{now offset='3 seconds' format='unix'}}",
+      "Cache-Control": "private, max-age=60, s-maxage=60",
+      "Vary": [
+        "Accept, Authorization, Cookie, X-GitHub-OTP",
+        "Accept-Encoding"
+      ],
+      "ETag": "W/\"1cb30f031c67c499473b3aad01c7f7a5\"",
+      "Last-Modified": "Thu, 06 Feb 2020 17:29:39 GMT",
+      "X-OAuth-Scopes": "admin:org, admin:org_hook, admin:public_key, admin:repo_hook, delete_repo, gist, notifications, repo, user, write:discussion",
+      "X-Accepted-OAuth-Scopes": "",
+      "X-GitHub-Media-Type": "unknown, github.v3",
+      "Access-Control-Expose-Headers": "ETag, Link, Location, Retry-After, X-GitHub-OTP, X-RateLimit-Limit, X-RateLimit-Remaining, X-RateLimit-Reset, X-OAuth-Scopes, X-Accepted-OAuth-Scopes, X-Poll-Interval, X-GitHub-Media-Type",
+      "Access-Control-Allow-Origin": "*",
+      "Strict-Transport-Security": "max-age=31536000; includeSubdomains; preload",
+      "X-Frame-Options": "deny",
+      "X-Content-Type-Options": "nosniff",
+      "X-XSS-Protection": "1; mode=block",
+      "Referrer-Policy": "origin-when-cross-origin, strict-origin-when-cross-origin",
+      "Content-Security-Policy": "default-src 'none'",
+      "X-GitHub-Request-Id": "CC37:2605:3F884:4E941:5E3C5BFC"
+    }
+  },
+  "uuid": "a60baf84-5b5c-4f86-af3d-cab0d609c7b2",
+  "persistent": true,
+  "insertionIndex": 1
+}