diff --git a/cmd/tests.sh b/cmd/tests.sh index 2f0a909..737acc2 100755 --- a/cmd/tests.sh +++ b/cmd/tests.sh @@ -348,6 +348,13 @@ tw --session Offline_Twatter search "from:michaelmalice constitution" test $(sqlite3 twitter.db "select count(*) from tweets where user_id = 44067298 and text like '%constitution%'") -gt "30" # Not sure exactly how many +# Test fetching user Likes +tw fetch_user Offline_Twatter # TODO: why doesn't this work when authenticated? +tw --session Offline_Twatter get_user_likes Offline_Twatter +test $(sqlite3 twitter.db "select count(*) from likes") = "2" +test $(sqlite3 twitter.db "select count(*) from likes where tweet_id = 1671902735250124802") = "1" + + # Test liking and unliking tw --session Offline_Twatter like_tweet https://twitter.com/elonmusk/status/1589023388676554753 tw --session Offline_Twatter unlike_tweet https://twitter.com/elonmusk/status/1589023388676554753 diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index ff8d949..bc718a9 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -134,6 +134,8 @@ func main() { fetch_user_feed(target, *how_many) case "get_user_tweets_all": fetch_user_feed(target, 999999999) + case "get_user_likes": + get_user_likes(target, *how_many) case "download_tweet_content": download_tweet_content(target) case "search": @@ -271,6 +273,21 @@ func fetch_user_feed(handle string, how_many int) { happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) } +func get_user_likes(handle string, how_many int) { + user, err := profile.GetUserByHandle(scraper.UserHandle(handle)) + if err != nil { + die(fmt.Sprintf("Error getting user: %s\n %s", handle, err.Error()), false, -1) + } + + trove, err := scraper.GetUserLikes(user.ID, "") // TODO: how_many + if err != nil { + die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2) + } + profile.SaveTweetTrove(trove) + + happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) +} + func download_tweet_content(tweet_identifier string) { tweet_id, err := extract_id_from(tweet_identifier) if err != nil { diff --git a/persistence/tweet_trove_queries.go b/persistence/tweet_trove_queries.go index fae09bc..6b5f554 100644 --- a/persistence/tweet_trove_queries.go +++ b/persistence/tweet_trove_queries.go @@ -71,4 +71,11 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) { panic(fmt.Errorf("Error saving retweet with ID %d from user ID %d:\n %w", r.RetweetID, r.RetweetedByID, err)) } } + + for _, l := range trove.Likes { + err := p.SaveLike(l) + if err != nil { + panic(fmt.Errorf("Error saving Like: %#v\n %w", l, err)) + } + } } diff --git a/persistence/versions.go b/persistence/versions.go index 39b302f..ce94b69 100644 --- a/persistence/versions.go +++ b/persistence/versions.go @@ -108,6 +108,14 @@ var MIGRATIONS = []string{ drop table space_participants; alter table space_participants_uniq rename to space_participants; vacuum;`, + `create table likes(rowid integer primary key, + sort_order integer unique not null, + user_id integer not null, + tweet_id integer not null, + unique(user_id, tweet_id) + foreign key(user_id) references users(id) + foreign key(tweet_id) references tweets(id) + );`, } var ENGINE_DATABASE_VERSION = len(MIGRATIONS) diff --git a/scraper/api_graphql_utils.go b/scraper/api_graphql_utils.go index 39bead8..c2dada7 100644 --- a/scraper/api_graphql_utils.go +++ b/scraper/api_graphql_utils.go @@ -52,7 +52,7 @@ type GraphqlFeatures struct { ResponsiveWebUcGqlEnabled bool `json:"responsive_web_uc_gql_enabled,omitempty"` VibeApiEnabled bool `json:"vibe_api_enabled,omitempty"` InteractiveTextEnabled bool `json:"interactive_text_enabled,omitempty"` - ResponsiveWebTextConversationsEnabled bool `json:"responsive_web_text_conversations_enabled,omitempty"` + ResponsiveWebTextConversationsEnabled bool `json:"responsive_web_text_conversations_enabled"` } type GraphqlURL struct { diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index eaffe30..7d14bea 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -752,12 +752,6 @@ func (api *API) GetGraphqlFeedFor(user_id UserID, cursor string) (APIV2Response, return response, err } -func (api API) GetLikesFor(user_id UserID, cursor string) (APIV2Response, error) { - var response APIV2Response - err := api.do_http("https://twitter.com/i/api/graphql/2Z6LYO4UTM4BnWjaNCod6g/Likes?variables=%7B%22userId%22%3A%22"+fmt.Sprint(user_id)+"%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%7D&features=%7B%22responsive_web_twitter_blue_verified_badge_is_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22unified_cards_ad_metadata_container_dynamic_card_content_query_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_uc_gql_enabled%22%3Atrue%2C%22vibe_api_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Afalse%2C%22interactive_text_enabled%22%3Atrue%2C%22responsive_web_text_conversations_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Atrue%7D", cursor, &response) //nolint:lll // It's a URL, come on - return response, err -} - /** * Resend the request to get more tweets if necessary * @@ -870,3 +864,63 @@ func (api *API) GetMoreTweetReplies(tweet_id TweetID, response *APIV2Response, m } return nil } + +func (api API) GetUserLikes(user_id UserID, cursor string) (TweetTrove, error) { + url, err := url.Parse(GraphqlURL{ + BaseUrl: "https://twitter.com/i/api/graphql/2Z6LYO4UTM4BnWjaNCod6g/Likes", + Variables: GraphqlVariables{ + UserID: user_id, + Count: 20, + Cursor: cursor, + IncludePromotedContent: false, + WithSuperFollowsUserFields: true, + WithDownvotePerspective: false, + WithReactionsMetadata: false, + WithReactionsPerspective: false, + WithSuperFollowsTweetFields: true, + WithBirdwatchNotes: false, + WithVoice: true, + WithV2Timeline: false, + }, + Features: GraphqlFeatures{ + ResponsiveWebTwitterBlueVerifiedBadgeIsEnabled: true, + VerifiedPhoneLabelEnabled: false, + ResponsiveWebGraphqlTimelineNavigationEnabled: true, + UnifiedCardsAdMetadataContainerDynamicCardContentQueryEnabled: true, + TweetypieUnmentionOptimizationEnabled: true, + ResponsiveWebUcGqlEnabled: true, + VibeApiEnabled: true, + ResponsiveWebEditTweetApiEnabled: true, + GraphqlIsTranslatableRWebTweetIsTranslatableEnabled: true, + StandardizedNudgesMisinfo: true, + TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: false, + InteractiveTextEnabled: true, + ResponsiveWebTextConversationsEnabled: false, + ResponsiveWebEnhanceCardsEnabled: true, + }, + }.String()) + if err != nil { + panic(err) + } + + var response APIV2Response + err = api.do_http(url.String(), cursor, &response) + if err != nil { + panic(err) + } + trove, err := response.ToTweetTroveAsLikes() + if err != nil { + return TweetTrove{}, err + } + + // Fill out the liking UserID + for i := range trove.Likes { + l := trove.Likes[i] + l.UserID = user_id + trove.Likes[i] = l + } + return trove, nil +} +func GetUserLikes(user_id UserID, cursor string) (TweetTrove, error) { + return the_api.GetUserLikes(user_id, cursor) +}