From 39bad1e6e121cf9b63b9d2125e38a3f01e441c92 Mon Sep 17 00:00:00 2001
From: Billy Keyes <bkeyes@palantir.com>
Date: Fri, 21 Jan 2022 17:58:05 -0800
Subject: [PATCH] Add delay between pull request updates

In projects with many open PRs using the automatic updates feature,
Bulldozer can trigger GitHub's secondary rate limits by updating (and
possibly querying) pull requests too quickly. To avoid this, add some
delay between operations.

For queries, use a fixed 250ms delay for now. This shouldn't slow things
down too much, but does prevent Bulldozer from blasting through PRs as
fast as possible.

For updates, use an exponential delay that starts at 1s and maxes out at
60s. I believe the secondary limits (at least for GitHub Enterprise) are
computed over a minute, so this will limit us to one update per minute
for large batches. I don't think this will be prohibitively slow, but we
could explore making this configurable in the future.

As part of this, refactor the logic so that we don't block queries on
previous updates.
---
 server/handler/push.go | 64 +++++++++++++++++++++++++++++++++++++++---
 1 file changed, 60 insertions(+), 4 deletions(-)

diff --git a/server/handler/push.go b/server/handler/push.go
index 65a96d6d5..7924ee19d 100644
--- a/server/handler/push.go
+++ b/server/handler/push.go
@@ -17,13 +17,23 @@ package handler
 import (
 	"context"
 	"encoding/json"
+	"time"
 
 	"github.com/google/go-github/v40/github"
+	"github.com/palantir/bulldozer/bulldozer"
 	"github.com/palantir/bulldozer/pull"
 	"github.com/palantir/go-githubapp/githubapp"
 	"github.com/pkg/errors"
 )
 
+const (
+	PullQueryDelay = 250 * time.Millisecond
+
+	PullUpdateBaseDelay = 1 * time.Second
+	PullUpdateMaxDelay  = 60 * time.Second
+	PullUpdateDelayMult = 1.5
+)
+
 type Push struct {
 	Base
 }
@@ -75,18 +85,64 @@ func (h *Push) Handle(ctx context.Context, eventType, deliveryID string, payload
 		return err
 	}
 
-	for _, pr := range prs {
+	if config == nil {
+		logger.Debug().Msg("Skipping pull request updates to missing configuration")
+		return nil
+	}
+
+	var toUpdate []updateCtx
+	for i, pr := range prs {
 		logger := logger.With().Int(githubapp.LogKeyPRNum, pr.GetNumber()).Logger()
-		logger.Debug().Msgf("Considering pull request for update")
+		logger.Debug().Msg("Checking if pull request should update")
 
+		ctx := logger.WithContext(ctx)
 		pullCtx := pull.NewGithubContext(client, pr)
-		if _, err := h.UpdatePullRequest(logger.WithContext(ctx), pullCtx, client, config, pr, baseRef); err != nil {
-			logger.Error().Err(errors.WithStack(err)).Msg("Error updating pull request")
+
+		shouldUpdate, err := bulldozer.ShouldUpdatePR(ctx, pullCtx, config.Update)
+		if err != nil {
+			logger.Error().Err(err).Msg("Error determining if pull request should update, skipping")
+			continue
+		}
+		if shouldUpdate {
+			toUpdate = append(toUpdate, updateCtx{
+				ctx:     ctx,
+				pullCtx: pullCtx,
+			})
+		}
+
+		if i < len(prs)-1 {
+			time.Sleep(delay(i, PullQueryDelay, 1, PullQueryDelay))
+		}
+	}
+
+	logger.Info().Msgf("Found %d pull requests that need updates", len(toUpdate))
+	for i, pr := range toUpdate {
+		bulldozer.UpdatePR(pr.ctx, pr.pullCtx, client, config.Update, baseRef)
+		if i < len(toUpdate)-1 {
+			d := delay(i, PullUpdateBaseDelay, PullUpdateDelayMult, PullUpdateMaxDelay)
+			logger.Debug().Msgf("Waiting %v until next update to avoid GitHub rate limits", d)
+			time.Sleep(d)
 		}
 	}
 
 	return nil
 }
 
+type updateCtx struct {
+	ctx     context.Context
+	pullCtx pull.Context
+}
+
+func delay(iter int, base time.Duration, mult float64, max time.Duration) time.Duration {
+	t := base
+	for ; iter > 0; iter-- {
+		t = time.Duration(mult * float64(t))
+		if t > max {
+			return max
+		}
+	}
+	return t
+}
+
 // type assertion
 var _ githubapp.EventHandler = &Push{}