lukegb/depot
1
0
Fork 0
Code Issues 1 Pull requests Projects Packages Activity Actions

twitterchiver/related_fetcher: use quoted_status_id instead of just quoted_status.id, sometimes only one is present

Browse source
This commit is contained in:
Luke Granger-Brown 2020-10-22 01:28:39 +00:00
parent 27b1e70ac1
commit a0d730cb47
1 changed files with 9 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
go/twitterchiver/related_fetcher/fetcher.go
View file

@ -101,6 +101,7 @@ type tweetData struct {
ID int64 `json:"id"` ID int64 `json:"id"`
Text string `json:"full_text"` Text string `json:"full_text"`
QuotedStatus *tweetData `json:"quoted_status"` QuotedStatus *tweetData `json:"quoted_status"`
QuotedStatusID *int64 `json:"quoted_status_id"`
RetweetedStatus *tweetData `json:"retweeted_status"` RetweetedStatus *tweetData `json:"retweeted_status"`
InReplyToStatusID *int64 `json:"in_reply_to_status_id"` InReplyToStatusID *int64 `json:"in_reply_to_status_id"`
ExtendedEntities tweetExtendedEntities `json:"extended_entities"` ExtendedEntities tweetExtendedEntities `json:"extended_entities"`
@ -297,15 +298,18 @@ func relatedFetchTick(ctx context.Context, cfg WorkerConfig) error {
var wg sync.WaitGroup var wg sync.WaitGroup
for _, t := range tweets { for _, t := range tweets {
var tweetIDsToFetch []int64 tweetIDsToFetch := make(map[int64]bool)
if t.QuotedStatus != nil { if t.QuotedStatus != nil {
tweetIDsToFetch = append(tweetIDsToFetch, t.QuotedStatus.ID) tweetIDsToFetch[t.QuotedStatus.ID] = true
}
if t.QuotedStatusID != nil {
tweetIDsToFetch[*t.QuotedStatusID] = true
} }
if t.RetweetedStatus != nil { if t.RetweetedStatus != nil {
tweetIDsToFetch = append(tweetIDsToFetch, t.RetweetedStatus.ID) tweetIDsToFetch[t.RetweetedStatus.ID] = true
} }
if t.InReplyToStatusID != nil { if t.InReplyToStatusID != nil {
tweetIDsToFetch = append(tweetIDsToFetch, *t.InReplyToStatusID) tweetIDsToFetch[*t.InReplyToStatusID] = true
} }
log.Printf("[%v:%d] Got %d tweets to fetch (%v)", cfg.Name, t.ID, len(tweetIDsToFetch), tweetIDsToFetch) log.Printf("[%v:%d] Got %d tweets to fetch (%v)", cfg.Name, t.ID, len(tweetIDsToFetch), tweetIDsToFetch)
@ -332,7 +336,7 @@ func relatedFetchTick(ctx context.Context, cfg WorkerConfig) error {
} }
httpClient := cfg.OAuthConfig.Client(ctx, oauth1.NewToken(accessToken, accessSecret)) httpClient := cfg.OAuthConfig.Client(ctx, oauth1.NewToken(accessToken, accessSecret))
for _, tid := range tweetIDsToFetch { for tid := range tweetIDsToFetch {
// Check if we already have tid. // Check if we already have tid.
log.Printf("[%v:%d] Fetching %d", cfg.Name, t.ID, tid) log.Printf("[%v:%d] Fetching %d", cfg.Name, t.ID, tid)