lukegb/depot
1
0
Fork 0
Code Issues 1 Pull requests Projects Packages Activity Actions

twitterchiver/related_fetcher: move the fetch token counter to right before we do the fetch

Browse source
This commit is contained in:
Luke Granger-Brown 2020-10-21 01:24:30 +00:00
parent d6cfb865ad
commit 0da26f3e42
1 changed files with 5 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
go/twitterchiver/related_fetcher/fetcher.go
View file

@ -333,10 +333,6 @@ func relatedFetchTick(ctx context.Context, cfg WorkerConfig) error {
httpClient := cfg.OAuthConfig.Client(ctx, oauth1.NewToken(accessToken, accessSecret)) httpClient := cfg.OAuthConfig.Client(ctx, oauth1.NewToken(accessToken, accessSecret))
for _, tid := range tweetIDsToFetch { for _, tid := range tweetIDsToFetch {
if _, ok := <-fetchLimited; !ok {
log.Printf("[%v:%d] Out of fetch tokens", cfg.Name, t.ID)
break
}
// Check if we already have tid. // Check if we already have tid.
log.Printf("[%v:%d] Fetching %d", cfg.Name, t.ID, tid) log.Printf("[%v:%d] Fetching %d", cfg.Name, t.ID, tid)
@ -350,6 +346,11 @@ func relatedFetchTick(ctx context.Context, cfg WorkerConfig) error {
continue continue
} }
if _, ok := <-fetchLimited; !ok {
log.Printf("[%v:%d] Out of fetch tokens", cfg.Name, t.ID)
break
}
// We don't have it already; let's fetch it from ze Twitterz. // We don't have it already; let's fetch it from ze Twitterz.
req, err := http.NewRequest("GET", fmt.Sprintf("https://api.twitter.com/1.1/statuses/show.json?id=%d&include_entities=true&include_ext_alt_text=true", tid), nil) req, err := http.NewRequest("GET", fmt.Sprintf("https://api.twitter.com/1.1/statuses/show.json?id=%d&include_entities=true&include_ext_alt_text=true", tid), nil)
if err != nil { if err != nil {