From a0d730cb47231c235ab90e2560dc104dc5668dd5 Mon Sep 17 00:00:00 2001 From: Luke Granger-Brown Date: Thu, 22 Oct 2020 01:28:39 +0000 Subject: [PATCH] twitterchiver/related_fetcher: use quoted_status_id instead of just quoted_status.id, sometimes only one is present --- go/twitterchiver/related_fetcher/fetcher.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/go/twitterchiver/related_fetcher/fetcher.go b/go/twitterchiver/related_fetcher/fetcher.go index a609ac6ab8..a906d1d4a7 100644 --- a/go/twitterchiver/related_fetcher/fetcher.go +++ b/go/twitterchiver/related_fetcher/fetcher.go @@ -101,6 +101,7 @@ type tweetData struct { ID int64 `json:"id"` Text string `json:"full_text"` QuotedStatus *tweetData `json:"quoted_status"` + QuotedStatusID *int64 `json:"quoted_status_id"` RetweetedStatus *tweetData `json:"retweeted_status"` InReplyToStatusID *int64 `json:"in_reply_to_status_id"` ExtendedEntities tweetExtendedEntities `json:"extended_entities"` @@ -297,15 +298,18 @@ func relatedFetchTick(ctx context.Context, cfg WorkerConfig) error { var wg sync.WaitGroup for _, t := range tweets { - var tweetIDsToFetch []int64 + tweetIDsToFetch := make(map[int64]bool) if t.QuotedStatus != nil { - tweetIDsToFetch = append(tweetIDsToFetch, t.QuotedStatus.ID) + tweetIDsToFetch[t.QuotedStatus.ID] = true + } + if t.QuotedStatusID != nil { + tweetIDsToFetch[*t.QuotedStatusID] = true } if t.RetweetedStatus != nil { - tweetIDsToFetch = append(tweetIDsToFetch, t.RetweetedStatus.ID) + tweetIDsToFetch[t.RetweetedStatus.ID] = true } if t.InReplyToStatusID != nil { - tweetIDsToFetch = append(tweetIDsToFetch, *t.InReplyToStatusID) + tweetIDsToFetch[*t.InReplyToStatusID] = true } log.Printf("[%v:%d] Got %d tweets to fetch (%v)", cfg.Name, t.ID, len(tweetIDsToFetch), tweetIDsToFetch) @@ -332,7 +336,7 @@ func relatedFetchTick(ctx context.Context, cfg WorkerConfig) error { } httpClient := cfg.OAuthConfig.Client(ctx, oauth1.NewToken(accessToken, accessSecret)) - for _, tid := range tweetIDsToFetch { + for tid := range tweetIDsToFetch { // Check if we already have tid. log.Printf("[%v:%d] Fetching %d", cfg.Name, t.ID, tid)