From dd68ee1fcee04f17a08bb62e0496fb62d92dd3e1 Mon Sep 17 00:00:00 2001 From: Alessio Date: Tue, 26 Dec 2023 19:54:08 -0600 Subject: [PATCH] Add scraper funcs for previous commit --- pkg/scraper/api_types_lists.go | 119 ++++++++++++++++++++++++++++ pkg/scraper/api_types_lists_test.go | 36 +++++++++ 2 files changed, 155 insertions(+) create mode 100644 pkg/scraper/api_types_lists.go create mode 100644 pkg/scraper/api_types_lists_test.go diff --git a/pkg/scraper/api_types_lists.go b/pkg/scraper/api_types_lists.go new file mode 100644 index 0000000..0f1e2af --- /dev/null +++ b/pkg/scraper/api_types_lists.go @@ -0,0 +1,119 @@ +package scraper + +import ( + "net/url" +) + +func (api *API) GetFollowees(user_id UserID, cursor string) (APIV2Response, error) { + url, err := url.Parse(GraphqlURL{ + BaseUrl: "https://twitter.com/i/api/graphql/0yD6Eiv23DKXRDU9VxlG2A/Following", + Variables: GraphqlVariables{ + UserID: user_id, + Cursor: cursor, + Count: 20, + IncludePromotedContent: false, + }, + Features: GraphqlFeatures{ + ResponsiveWebGraphqlExcludeDirectiveEnabled: true, + VerifiedPhoneLabelEnabled: false, + CreatorSubscriptionsTweetPreviewApiEnabled: true, + ResponsiveWebGraphqlTimelineNavigationEnabled: true, + ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled: false, + C9sTweetAnatomyModeratorBadgeEnabled: true, + TweetypieUnmentionOptimizationEnabled: true, + ResponsiveWebEditTweetApiEnabled: true, + GraphqlIsTranslatableRWebTweetIsTranslatableEnabled: true, + ViewCountsEverywhereApiEnabled: true, + LongformNotetweetsConsumptionEnabled: true, + ResponsiveWebTwitterArticleTweetConsumptionEnabled: false, + TweetAwardsWebTippingEnabled: false, + FreedomOfSpeechNotReachFetchEnabled: true, + StandardizedNudgesMisinfo: true, + TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: true, + RwebVideoTimestampsEnabled: true, + LongformNotetweetsRichTextReadEnabled: true, + LongformNotetweetsInlineMediaEnabled: true, + ResponsiveWebMediaDownloadVideoEnabled: false, + ResponsiveWebEnhanceCardsEnabled: false, + }, + }.String()) + if err != nil { + panic(err) + } + + var result APIV2Response + err = api.do_http(url.String(), "", &result) + return result, err +} + +type PaginatedFollowees struct { + user_id UserID +} + +func (p PaginatedFollowees) NextPage(api *API, cursor string) (APIV2Response, error) { + return api.GetFollowees(p.user_id, cursor) +} +func (p PaginatedFollowees) ToTweetTrove(r APIV2Response) (TweetTrove, error) { + return r.ToTweetTrove() +} + +func GetFollowees(user_id UserID, how_many int) (TweetTrove, error) { + return the_api.GetPaginatedQuery(PaginatedFollowees{user_id}, how_many) +} + +func (api *API) GetFollowers(user_id UserID, cursor string) (APIV2Response, error) { + url, err := url.Parse(GraphqlURL{ + BaseUrl: "https://twitter.com/i/api/graphql/3_7xfjmh897x8h_n6QBqTA/Followers", + Variables: GraphqlVariables{ + UserID: user_id, + Cursor: cursor, + Count: 20, + IncludePromotedContent: false, + }, + Features: GraphqlFeatures{ + ResponsiveWebGraphqlExcludeDirectiveEnabled: true, + VerifiedPhoneLabelEnabled: false, + CreatorSubscriptionsTweetPreviewApiEnabled: true, + ResponsiveWebGraphqlTimelineNavigationEnabled: true, + ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled: false, + C9sTweetAnatomyModeratorBadgeEnabled: true, + TweetypieUnmentionOptimizationEnabled: true, + ResponsiveWebEditTweetApiEnabled: true, + GraphqlIsTranslatableRWebTweetIsTranslatableEnabled: true, + ViewCountsEverywhereApiEnabled: true, + LongformNotetweetsConsumptionEnabled: true, + ResponsiveWebTwitterArticleTweetConsumptionEnabled: false, + TweetAwardsWebTippingEnabled: false, + FreedomOfSpeechNotReachFetchEnabled: true, + StandardizedNudgesMisinfo: true, + TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: true, + RwebVideoTimestampsEnabled: true, + LongformNotetweetsRichTextReadEnabled: true, + LongformNotetweetsInlineMediaEnabled: true, + ResponsiveWebMediaDownloadVideoEnabled: false, + ResponsiveWebEnhanceCardsEnabled: false, + }, + }.String()) + if err != nil { + panic(err) + } + + var result APIV2Response + err = api.do_http(url.String(), "", &result) + return result, err +} + +type PaginatedFollowers struct { + user_id UserID +} + +func (p PaginatedFollowers) NextPage(api *API, cursor string) (APIV2Response, error) { + return api.GetFollowers(p.user_id, cursor) +} +func (p PaginatedFollowers) ToTweetTrove(r APIV2Response) (TweetTrove, error) { + return r.ToTweetTrove() +} + +func GetFollowers(user_id UserID, how_many int) (TweetTrove, error) { + return the_api.GetPaginatedQuery(PaginatedFollowers{user_id}, how_many) +} diff --git a/pkg/scraper/api_types_lists_test.go b/pkg/scraper/api_types_lists_test.go new file mode 100644 index 0000000..f773961 --- /dev/null +++ b/pkg/scraper/api_types_lists_test.go @@ -0,0 +1,36 @@ +package scraper_test + +import ( + "testing" + + "encoding/json" + "os" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" +) + +func TestParseFolloweesList(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + data, err := os.ReadFile("test_responses/lists/followees.json") + require.NoError(err) + + var resp APIV2Response + err = json.Unmarshal(data, &resp) + require.NoError(err) + + tweet_trove, err := resp.ToTweetTrove() + require.NoError(err) + + // Check users + assert.Len(tweet_trove.Users, 4) + _, is_ok := tweet_trove.Users[1349149096909668363] + assert.True(is_ok) + + // Test cursor-bottom + bottom_cursor := resp.GetCursorBottom() + assert.Equal("0|1739810405452087290", bottom_cursor) +}