From 4ea15f10afe15accfa4b5a8dce9c716bbb8356c3 Mon Sep 17 00:00:00 2001 From: Alessio Date: Sun, 15 Sep 2024 22:56:59 -0700 Subject: [PATCH] Add scraper function to get user by ID --- cmd/twitter/main.go | 26 ++++++++++++ internal/webserver/server.go | 2 + .../webserver/tpl/includes/nav_sidebar.tpl | 2 +- pkg/scraper/api_errors.go | 1 + pkg/scraper/api_graphql_utils.go | 2 + pkg/scraper/api_types_v2.go | 42 +++++++++++++++++++ 6 files changed, 74 insertions(+), 1 deletion(-) diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 30f8d82..1579042 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -163,6 +163,12 @@ func main() { login(target, password) case "fetch_user": fetch_user(scraper.UserHandle(target)) + case "fetch_user_by_id": + id, err := strconv.Atoi(target) + if err != nil { + panic(err) + } + fetch_user_by_id(scraper.UserID(id)) case "download_user_content": download_user_content(scraper.UserHandle(target)) case "fetch_tweet_only": @@ -310,6 +316,26 @@ func fetch_user(handle scraper.UserHandle) { happy_exit("Saved the user", nil) } +func fetch_user_by_id(id scraper.UserID) { + session, err := scraper.NewGuestSession() // This endpoint works better if you're not logged in + if err != nil { + panic(err) + } + user, err := session.GetUserByID(id) + if err != nil { + panic(err) + } + log.Debug(user) + + err = profile.SaveUser(&user) + if err != nil { + die(fmt.Sprintf("Error saving user: %s", err.Error()), false, 4) + } + + download_user_content(user.Handle) + happy_exit("Saved the user", nil) +} + /** * Scrape a single tweet and save it in the database. * diff --git a/internal/webserver/server.go b/internal/webserver/server.go index 18ef85a..2fb2a32 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -139,6 +139,8 @@ func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.StripPrefix("/messages", http.HandlerFunc(app.Messages)).ServeHTTP(w, r) case "nav-sidebar-poll-updates": app.NavSidebarPollUpdates(w, r) + case "communities": + panic("not implemented") default: app.UserFeed(w, r) } diff --git a/internal/webserver/tpl/includes/nav_sidebar.tpl b/internal/webserver/tpl/includes/nav_sidebar.tpl index 53dd247..526a1c6 100644 --- a/internal/webserver/tpl/includes/nav_sidebar.tpl +++ b/internal/webserver/tpl/includes/nav_sidebar.tpl @@ -51,7 +51,7 @@ - +
  • diff --git a/pkg/scraper/api_errors.go b/pkg/scraper/api_errors.go index 7283006..0c07e05 100644 --- a/pkg/scraper/api_errors.go +++ b/pkg/scraper/api_errors.go @@ -7,6 +7,7 @@ import ( var ( END_OF_FEED = errors.New("End of feed") ErrDoesntExist = errors.New("Doesn't exist") + ErrUserIsBanned = errors.New("user is banned") EXTERNAL_API_ERROR = errors.New("Unexpected result from external API") ErrorIsTombstone = errors.New("tweet is a tombstone") ErrRateLimited = errors.New("rate limited") diff --git a/pkg/scraper/api_graphql_utils.go b/pkg/scraper/api_graphql_utils.go index c22f421..9ea3fac 100644 --- a/pkg/scraper/api_graphql_utils.go +++ b/pkg/scraper/api_graphql_utils.go @@ -63,6 +63,7 @@ type GraphqlFeatures struct { ResponsiveWebTextConversationsEnabled bool `json:"responsive_web_text_conversations_enabled"` ResponsiveWebTwitterArticleTweetConsumptionEnabled bool `json:"responsive_web_twitter_article_tweet_consumption_enabled"` ResponsiveWebMediaDownloadVideoEnabled bool `json:"responsive_web_media_download_video_enabled"` + ResponsiveWebTwitterArticleNotesTabEnabled bool `json:"responsive_web_twitter_article_notes_tab_enabled"` SubscriptionsVerificationInfoVerifiedSinceEnabled bool `json:"subscriptions_verification_info_verified_since_enabled"` HiddenProfileLikesEnabled bool `json:"hidden_profile_likes_enabled"` HiddenProfileSubscriptionsEnabled bool `json:"hidden_profile_subscriptions_enabled"` @@ -81,6 +82,7 @@ type GraphqlFeatures struct { ArticlesPreviewEnabled bool `json:"articles_preview_enabled,omitempty"` GraphqlTimelineV2BookmarkTimeline bool `json:"graphql_timeline_v2_bookmark_timeline,omitempty"` CreatorSubscriptionsQuoteTweetPreviewEnabled bool `json:"creator_subscriptions_quote_tweet_preview_enabled"` + SubscriptionsFeatureCanGiftPremium bool `json:"subscriptions_feature_can_gift_premium,omitempty"` } type GraphqlURL struct { diff --git a/pkg/scraper/api_types_v2.go b/pkg/scraper/api_types_v2.go index e5662f4..4e73083 100644 --- a/pkg/scraper/api_types_v2.go +++ b/pkg/scraper/api_types_v2.go @@ -1384,6 +1384,48 @@ func (api API) GetUser(handle UserHandle) (User, error) { return ParseSingleUser(apiUser) } +func (api API) GetUserByID(u_id UserID) (User, error) { + url, err := url.Parse(GraphqlURL{ + BaseUrl: "https://x.com/i/api/graphql/Qw77dDjp9xCpUY-AXwt-yQ/UserByRestId", + Variables: GraphqlVariables{ + UserID: u_id, + }, + Features: GraphqlFeatures{ + RWebTipjarConsumptionEnabled: true, + ResponsiveWebGraphqlExcludeDirectiveEnabled: true, + VerifiedPhoneLabelEnabled: false, + ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled: false, + ResponsiveWebGraphqlTimelineNavigationEnabled: true, + SubscriptionsFeatureCanGiftPremium: true, + ResponsiveWebTwitterArticleNotesTabEnabled: true, + }, + }.String()) + if err != nil { + panic(err) + } + + var response UserResponse + err = api.do_http(url.String(), "", &response) + if err != nil { + return User{}, err + } + apiUser, err := response.ConvertToAPIUser() + if errors.Is(err, ErrDoesntExist) { + return User{}, err + } + if apiUser.ScreenName == "" { + if apiUser.IsBanned { + return User{}, ErrUserIsBanned + } else { + return User{}, ErrDoesntExist + } + } + if err != nil { + return User{}, fmt.Errorf("Error fetching user ID %d:\n %w", u_id, err) + } + return ParseSingleUser(apiUser) +} + // Paginated Search // ----------------