me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix wrong 'Instruction' item being chosen following new API update :V

Browse Source
This commit is contained in:
Alessio 2022-02-12 15:51:55 -08:00
parent 13170c84b2
commit 4b2df1c628
2 changed files with 48 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
scraper/api_types_v2.go
View File

@ -276,16 +276,18 @@ type APIV2Entry struct {
} `json:"content"`
}
type APIV2Instruction struct {
Type string `json:"type"`
Entries []APIV2Entry`json:"entries"`
}
type APIV2Response struct {
Data struct {
User struct {
Result struct {
Timeline struct {
Timeline struct {
Instructions []struct {
Type string `json:"type"`
Entries []APIV2Entry`json:"entries"`
} `json:"instructions"`
Instructions []APIV2Instruction `json:"instructions"`
} `json:"timeline"`
} `json:"timeline"`
} `json:"result"`
@ -293,8 +295,18 @@ type APIV2Response struct {
} `json:"data"`
}
func (api_response APIV2Response) GetMainInstruction() *APIV2Instruction {
instructions := api_response.Data.User.Result.Timeline.Timeline.Instructions
for i := range instructions {
if instructions[i].Type == "TimelineAddEntries" {
return &instructions[i]
}
}
panic("No 'TimelineAddEntries' found")
}
func (api_response APIV2Response) GetCursorBottom() string {
entries := api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries
entries := api_response.GetMainInstruction().Entries
last_entry := entries[len(entries) - 1]
if last_entry.Content.CursorType != "Bottom" {
panic("No bottom cursor found")
@ -307,7 +319,7 @@ func (api_response APIV2Response) GetCursorBottom() string {
* Returns `true` if there's no non-cursor entries in this response, false otherwise
*/
func (api_response APIV2Response) IsEmpty() bool {
for _, e := range api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries {
for _, e := range api_response.GetMainInstruction().Entries {
if !strings.Contains(e.EntryID, "cursor") {
return false
}
@ -320,7 +332,7 @@ func (api_response APIV2Response) IsEmpty() bool {
*/
func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) {
ret := NewTweetTrove()
for _, entry := range api_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries { // TODO: the second Instruction is the pinned tweet
for _, entry := range api_response.GetMainInstruction().Entries { // TODO: the second Instruction is the pinned tweet
if !strings.HasPrefix(entry.EntryID, "tweet-") {
continue
}
@ -399,13 +411,13 @@ func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Respo
// TODO user-feed-infinite-fetch: what if you reach the end of the user's timeline? Might loop
// forever getting no new tweets
last_response := response
for last_response.GetCursorBottom() != "" && len(response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries) < min_tweets {
for last_response.GetCursorBottom() != "" && len(response.GetMainInstruction().Entries) < min_tweets {
fresh_response, err := api.GetGraphqlFeedFor(user_id, last_response.GetCursorBottom())
if err != nil {
return err
}
if fresh_response.GetCursorBottom() == last_response.GetCursorBottom() && len(fresh_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries) == 0 {
if fresh_response.GetCursorBottom() == last_response.GetCursorBottom() && len(fresh_response.GetMainInstruction().Entries) == 0 {
// Empty response, cursor same as previous: end of feed has been reached
return END_OF_FEED
}
@ -417,11 +429,11 @@ func (api API) GetMoreTweetsFromGraphqlFeed(user_id UserID, response *APIV2Respo
last_response = &fresh_response
// Copy over the entries
response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries = append(
response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries,
last_response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries...)
response.GetMainInstruction().Entries = append(
response.GetMainInstruction().Entries,
last_response.GetMainInstruction().Entries...)
fmt.Printf("Have %d entries so far\n", len(response.Data.User.Result.Timeline.Timeline.Instructions[0].Entries))
fmt.Printf("Have %d entries so far\n", len(response.GetMainInstruction().Entries))
}
return nil
}

23
scraper/api_types_v2_test.go
View File

@ -516,3 +516,26 @@ func TestAPIV2FeedIsEmpty(t *testing.T) {
assert.Len(trove.Users, 0)
assert.Len(trove.Retweets, 0)
}
/**
* Should get the right Instruction element
*/
func TestAPIV2GetMainInstructionFromFeed(t *testing.T) {
assert := assert.New(t)
data, err := ioutil.ReadFile("test_responses/api_v2/user_feed_apiv2.json")
if err != nil {
panic(err)
}
var feed APIV2Response
err = json.Unmarshal(data, &feed)
if err != nil {
t.Errorf(err.Error())
}
assert.Equal(len(feed.GetMainInstruction().Entries), 41)
// Test that this is a writable version
feed.GetMainInstruction().Entries = append(feed.GetMainInstruction().Entries, APIV2Entry{EntryID: "asdf"})
assert.Equal(len(feed.GetMainInstruction().Entries), 42)
assert.Equal(feed.GetMainInstruction().Entries[41].EntryID, "asdf")
}