From 0da26f3e42341f0183ad1906235de33bc396d771 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Wed, 21 Oct 2020 01:24:30 +0000 Subject: [PATCH] twitterchiver/related_fetcher: move the fetch token counter to right before we do the fetch --- go/twitterchiver/related_fetcher/fetcher.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/go/twitterchiver/related_fetcher/fetcher.go b/go/twitterchiver/related_fetcher/fetcher.go index afaea0a0d2..a609ac6ab8 100644 --- a/go/twitterchiver/related_fetcher/fetcher.go +++ b/go/twitterchiver/related_fetcher/fetcher.go @@ -333,10 +333,6 @@ func relatedFetchTick(ctx context.Context, cfg WorkerConfig) error { httpClient := cfg.OAuthConfig.Client(ctx, oauth1.NewToken(accessToken, accessSecret)) for _, tid := range tweetIDsToFetch { - if _, ok := <-fetchLimited; !ok { - log.Printf("[%v:%d] Out of fetch tokens", cfg.Name, t.ID) - break - } // Check if we already have tid. log.Printf("[%v:%d] Fetching %d", cfg.Name, t.ID, tid) @@ -350,6 +346,11 @@ func relatedFetchTick(ctx context.Context, cfg WorkerConfig) error { continue } + if _, ok := <-fetchLimited; !ok { + log.Printf("[%v:%d] Out of fetch tokens", cfg.Name, t.ID) + break + } + // We don't have it already; let's fetch it from ze Twitterz. req, err := http.NewRequest("GET", fmt.Sprintf("https://api.twitter.com/1.1/statuses/show.json?id=%d&include_entities=true&include_ext_alt_text=true", tid), nil) if err != nil {