me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add parsing of pinned tweets on scraped user feeds

Browse Source
This commit is contained in:
Alessio 2024-03-18 22:20:36 -07:00
parent 41e525d223
commit 898ad163ae
2 changed files with 27 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pkg/scraper/api_types_v2.go
View File

@ -565,6 +565,7 @@ func (e APIV2Entry) ToTweetTrove() TweetTrove {
type APIV2Instruction struct { type APIV2Instruction struct {
Type string `json:"type"` Type string `json:"type"`
Entries []APIV2Entry `json:"entries"` Entries []APIV2Entry `json:"entries"`
Entry APIV2Entry `json:"entry"`
} }
type APIV2Response struct { type APIV2Response struct {
@ -777,6 +778,20 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) {
return ret, nil // TODO: This doesn't need to return an error, it's always nil return ret, nil // TODO: This doesn't need to return an error, it's always nil
} }
func (r APIV2Response) GetPinnedTweetAsTweetTrove() TweetTrove {
for _, instr := range r.Data.User.Result.Timeline.Timeline.Instructions {
if instr.Type == "TimelinePinEntry" {
return instr.Entry.ToTweetTrove()
}
}
for _, instr := range r.Data.User.Result.TimelineV2.Timeline.Instructions {
if instr.Type == "TimelinePinEntry" {
return instr.Entry.ToTweetTrove()
}
}
return NewTweetTrove()
}
func (r APIV2Response) ToTweetTroveAsLikes() (TweetTrove, error) { func (r APIV2Response) ToTweetTroveAsLikes() (TweetTrove, error) {
ret, err := r.ToTweetTrove() ret, err := r.ToTweetTrove()
if err != nil { if err != nil {
@ -916,7 +931,10 @@ func (p PaginatedUserFeed) NextPage(api *API, cursor string) (APIV2Response, err
return api.GetGraphqlFeedFor(p.user_id, cursor) return api.GetGraphqlFeedFor(p.user_id, cursor)
} }
func (p PaginatedUserFeed) ToTweetTrove(r APIV2Response) (TweetTrove, error) { func (p PaginatedUserFeed) ToTweetTrove(r APIV2Response) (TweetTrove, error) {
return r.ToTweetTrove() ret, err := r.ToTweetTrove()
// Add the pinned tweet, if applicable
ret.MergeWith(r.GetPinnedTweetAsTweetTrove())
return ret, err
} }
func (api *API) GetTweetDetail(tweet_id TweetID, cursor string) (APIV2Response, error) { func (api *API) GetTweetDetail(tweet_id TweetID, cursor string) (APIV2Response, error) {

9
pkg/scraper/api_types_v2_test.go
View File

@ -489,13 +489,15 @@ func TestParseAPIV2UserFeed(t *testing.T) {
err = json.Unmarshal(data, &feed) err = json.Unmarshal(data, &feed)
require.NoError(err) require.NoError(err)
tweet_trove, err := feed.ToTweetTrove() _p := PaginatedUserFeed{}
tweet_trove, err := _p.ToTweetTrove(feed)
require.NoError(err) require.NoError(err)
// Check users // Check users
user := tweet_trove.Users[44067298] user := tweet_trove.Users[44067298]
assert.Equal(UserID(44067298), user.ID) assert.Equal(UserID(44067298), user.ID)
assert.Equal("Michael Malice", user.DisplayName) assert.Equal("Michael Malice", user.DisplayName)
assert.Equal(TweetID(1477347403023982596), user.PinnedTweetID)
retweeted_user := tweet_trove.Users[1326229737551912960] retweeted_user := tweet_trove.Users[1326229737551912960]
assert.Equal(UserID(1326229737551912960), retweeted_user.ID) assert.Equal(UserID(1326229737551912960), retweeted_user.ID)
@ -507,6 +509,11 @@ func TestParseAPIV2UserFeed(t *testing.T) {
// Check retweets // Check retweets
assert.Len(tweet_trove.Retweets, 2) assert.Len(tweet_trove.Retweets, 2)
// Check pinned tweet
pinned_tweet, is_ok := tweet_trove.Tweets[user.PinnedTweetID]
require.True(is_ok)
assert.Equal(856, pinned_tweet.NumLikes)
// Test cursor-bottom // Test cursor-bottom
bottom_cursor := feed.GetCursorBottom() bottom_cursor := feed.GetCursorBottom()
assert.Equal("HBaYgL2Fp/T7nCkAAA==", bottom_cursor) assert.Equal("HBaYgL2Fp/T7nCkAAA==", bottom_cursor)