FOUND-1643: FEATURE: Adding rate limit to CSCGlobal (#2490)

providers/cscglobal/api.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 	"os"
 	"sort"
@@ -642,6 +643,12 @@ func (client *providerClient) geturl(url string) ([]byte, error) {
 	req.Header.Add("Authorization", "Bearer "+client.token)
 	req.Header.Add("Accept", "application/json")
 
+	// Default CSCGlobal rate limit is twenty requests per second
+	var backoff = time.Second
+
+	const maxBackoff = time.Second * 15
+
+retry:
 	resp, err := hclient.Do(req)
 	if err != nil {
 		return nil, err
@@ -654,6 +661,22 @@ func (client *providerClient) geturl(url string) ([]byte, error) {
 
 	if resp.StatusCode == 400 {
 		// 400, error message is in the body as plain text
+		// Apparently CSCGlobal uses status code 400 for rate limit, grump
+
+		if string(bodyString) == "Requests exceeded API Rate limit." {
+			// a simple exponential back-off with a 3-minute max.
+			log.Printf("Delaying %v due to ratelimit\n", backoff)
+			time.Sleep(backoff)
+			backoff = backoff + (backoff / 2)
+			if backoff > maxBackoff {
+				return nil, fmt.Errorf("CSC Global API timeout max backoff (geturl): %s URL: %s%s",
+					bodyString,
+					req.Host, req.URL.RequestURI())
+			}
+
+			goto retry
+		}
+
 		return nil, fmt.Errorf("CSC Global API error (geturl): %s URL: %s%s",
 			bodyString,
 			req.Host, req.URL.RequestURI())