diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index f2adcc6..8348c84 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -8,6 +8,7 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/term" "io/fs" + "time" "os" "strconv" "strings" @@ -40,6 +41,8 @@ func main() { how_many := flag.Int("n", 50, "") flag.IntVar(how_many, "number", 50, "") + delay := flag.String("delay", "0ms", "") + var default_log_level string if version_string == "" { default_log_level = "debug" @@ -137,6 +140,10 @@ func main() { log.Warnf("Unable to initialize guest session! Might be a network issue") } // Don't exit here, some operations don't require a connection } + api.Delay, err = time.ParseDuration(*delay) + if err != nil { + die(fmt.Sprintf("Invalid delay: %q", *delay), false, 1) + } switch operation { case "login": diff --git a/pkg/scraper/api_request_utils.go b/pkg/scraper/api_request_utils.go index 28b60fa..3e7ba9c 100644 --- a/pkg/scraper/api_request_utils.go +++ b/pkg/scraper/api_request_utils.go @@ -22,6 +22,7 @@ type API struct { GuestToken string Client http.Client CSRFToken string + Delay time.Duration } type api_outstruct struct { diff --git a/pkg/scraper/api_types_v2.go b/pkg/scraper/api_types_v2.go index d0dc6a1..a594b40 100644 --- a/pkg/scraper/api_types_v2.go +++ b/pkg/scraper/api_types_v2.go @@ -898,6 +898,10 @@ type PaginatedQuery interface { func (api *API) GetMore(pq PaginatedQuery, response *APIV2Response, count int) error { last_response := response for last_response.GetCursorBottom() != "" && len(response.GetMainInstruction().Entries) < count { + if api.Delay != 0 { + fmt.Printf("Pausing for %s...", api.Delay) + time.Sleep(api.Delay) // Slow down the requests, if applicable + } fresh_response, err := pq.NextPage(api, last_response.GetCursorBottom()) if err != nil { return fmt.Errorf("error getting next page for %#v: %w", pq, err) // e.g., rate limited