HETZNER: better rate limit handling (#936)

* HETZNER: better rate limit handling

- Hetzner is using a Proxy service 'kong' which broadcasts it limits
- honor 'Retry-After' of 429 responses
- delay requests per-se: see the amended docs for details

Signed-off-by: Jakob Ackermann <das7pad@outlook.com>

* HETZNER: apply review feedback: store quotaName as lower case

Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com>
Signed-off-by: Jakob Ackermann <das7pad@outlook.com>

Co-authored-by: Tom Limoncelli <tlimoncelli@stackoverflow.com>

providers/hetzner/api.go

@@ -7,6 +7,8 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"strconv"
+	"strings"
 	"time"
 )
 
@@ -29,6 +31,43 @@ func checkIsLockedSystemRecord(record record) error {
 	return nil
 }
 
+func getHomogenousDelay(headers http.Header, quotaName string) (time.Duration, error) {
+	quota, err := parseHeaderAsInt(headers, "X-Ratelimit-Limit-"+strings.Title(quotaName))
+	if err != nil {
+		return 0, err
+	}
+
+	var unit time.Duration
+	switch quotaName {
+	case "hour":
+		unit = time.Hour
+	case "minute":
+		unit = time.Minute
+	case "second":
+		unit = time.Second
+	}
+
+	delay := time.Duration(int64(unit) / quota)
+	return delay, nil
+}
+
+func getRetryAfterDelay(header http.Header) (time.Duration, error) {
+	retryAfter, err := parseHeaderAsInt(header, "Retry-After")
+	if err != nil {
+		return 0, err
+	}
+	delay := time.Duration(retryAfter * int64(time.Second))
+	return delay, nil
+}
+
+func parseHeaderAsInt(headers http.Header, headerName string) (int64, error) {
+	value, ok := headers[headerName]
+	if !ok {
+		return 0, fmt.Errorf("header %q is missing", headerName)
+	}
+	return strconv.ParseInt(value[0], 10, 0)
+}
+
 func (api *hetznerProvider) bulkCreateRecords(records []record) error {
 	for _, record := range records {
 		if err := checkIsLockedSystemRecord(record); err != nil {
@@ -185,6 +224,8 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte
 			}
 		}
 
+		api.requestRateLimiter.handleResponse(*resp)
+		// retry the request when rate-limited
 		if resp.StatusCode == 429 {
 			api.requestRateLimiter.handleRateLimitedRequest()
 			cleanupResponseBody()
@@ -206,9 +247,12 @@ func (api *hetznerProvider) request(endpoint string, method string, request inte
 }
 
 func (api *hetznerProvider) startRateLimited() {
-	// Simulate a request that is getting a 429 response.
-	api.requestRateLimiter.afterRequest()
-	api.requestRateLimiter.bumpDelay()
+	// _Now_ is the best reference we can get for the last request.
+	// Head-On-Head invocations of DNSControl benefit from fewer initial
+	//  rate-limited requests.
+	api.requestRateLimiter.lastRequest = time.Now()
+	// use the default delay until we have had a chance to parse limits.
+	api.requestRateLimiter.setDefaultDelay()
 }
 
 func (api *hetznerProvider) updateRecord(record record) error {
@@ -221,8 +265,9 @@ func (api *hetznerProvider) updateRecord(record record) error {
 }
 
 type requestRateLimiter struct {
-	delay       time.Duration
-	lastRequest time.Time
+	delay                     time.Duration
+	lastRequest               time.Time
+	optimizeForRateLimitQuota string
 }
 
 func (requestRateLimiter *requestRateLimiter) afterRequest() {
@@ -236,23 +281,50 @@ func (requestRateLimiter *requestRateLimiter) beforeRequest() {
 	time.Sleep(time.Until(requestRateLimiter.lastRequest.Add(requestRateLimiter.delay)))
 }
 
-func (requestRateLimiter *requestRateLimiter) bumpDelay() string {
-	var backoffType string
-	if requestRateLimiter.delay == 0 {
-		// At the time this provider was implemented (2020-10-18),
-		//  one request per second could go though when rate-limited.
-		requestRateLimiter.delay = time.Second
-		backoffType = "constant"
-	} else {
-		// The initial assumption of 1 req/s may no hold true forever.
-		// Future proof this provider, use exponential back-off.
-		requestRateLimiter.delay = requestRateLimiter.delay * 2
-		backoffType = "exponential"
+func (requestRateLimiter *requestRateLimiter) setDefaultDelay() {
+	// default to a rate-limit of 1 req/s -- the next response should update it.
+	requestRateLimiter.delay = time.Second
+}
+
+func (requestRateLimiter *requestRateLimiter) setOptimizeForRateLimitQuota(quota string) error {
+	quotaNormalized := strings.ToLower(quota)
+	switch quotaNormalized {
+	case "hour", "minute", "second":
+		requestRateLimiter.optimizeForRateLimitQuota = quotaNormalized
+	case "":
+		requestRateLimiter.optimizeForRateLimitQuota = "second"
+	default:
+		return fmt.Errorf("%q is not a valid quota, expected 'Hour', 'Minute', 'Second' or unset", quota)
 	}
-	return backoffType
+	return nil
 }
 
 func (requestRateLimiter *requestRateLimiter) handleRateLimitedRequest() {
-	backoffType := requestRateLimiter.bumpDelay()
-	fmt.Println(fmt.Sprintf("WARNING: request rate-limited, %s back-off is now at %s.", backoffType, requestRateLimiter.delay))
+	message := "Rate-Limited, consider bumping the setting 'optimize_for_rate_limit_quota': %q -> %q"
+	switch requestRateLimiter.optimizeForRateLimitQuota {
+	case "hour":
+		message = "Rate-Limited, you are already using the slowest request rate. Consider contacting the Hetzner Support for raising your quota."
+	case "minute":
+		message = fmt.Sprintf(message, "Minute", "Hour")
+	case "second":
+		message = fmt.Sprintf(message, "Second", "Minute")
+	}
+	fmt.Println(message)
+}
+
+func (requestRateLimiter *requestRateLimiter) handleResponse(resp http.Response) {
+	homogenousDelay, err := getHomogenousDelay(resp.Header, requestRateLimiter.optimizeForRateLimitQuota)
+	if err != nil {
+		requestRateLimiter.setDefaultDelay()
+		return
+	}
+
+	delay := homogenousDelay
+	if resp.StatusCode == 429 {
+		retryAfterDelay, err := getRetryAfterDelay(resp.Header)
+		if err == nil {
+			delay = retryAfterDelay
+		}
+	}
+	requestRateLimiter.delay = delay
 }

providers/hetzner/hetznerProvider.go

@@ -40,9 +40,19 @@ func New(settings map[string]string, _ json.RawMessage) (providers.DNSServicePro
 	api.apiKey = settings["api_key"]
 
 	if settings["rate_limited"] == "true" {
+		// backwards compatibility
+		settings["start_with_default_rate_limit"] = "true"
+	}
+	if settings["start_with_default_rate_limit"] == "true" {
 		api.startRateLimited()
 	}
 
+	quota := settings["optimize_for_rate_limit_quota"]
+	err := api.requestRateLimiter.setOptimizeForRateLimitQuota(quota)
+	if err != nil {
+		return nil, fmt.Errorf("unexpected value for optimize_for_rate_limit_quota: %w", err)
+	}
+
 	return api, nil
 }