Add scraper funcs for previous commit

2023-12-26 19:54:08 -06:00 · 2023-12-26 19:54:08 -06:00 · dd68ee1fce
commit dd68ee1fce
parent df8093bbd9
2 changed files with 155 additions and 0 deletions
--- a/pkg/scraper/api_types_lists.go
+++ b/pkg/scraper/api_types_lists.go
@ -0,0 +1,119 @@
+package scraper
+
+import (
+	"net/url"
+)
+
+func (api *API) GetFollowees(user_id UserID, cursor string) (APIV2Response, error) {
+	url, err := url.Parse(GraphqlURL{
+		BaseUrl: "https://twitter.com/i/api/graphql/0yD6Eiv23DKXRDU9VxlG2A/Following",
+		Variables: GraphqlVariables{
+			UserID:                 user_id,
+			Cursor:                 cursor,
+			Count:                  20,
+			IncludePromotedContent: false,
+		},
+		Features: GraphqlFeatures{
+			ResponsiveWebGraphqlExcludeDirectiveEnabled:                    true,
+			VerifiedPhoneLabelEnabled:                                      false,
+			CreatorSubscriptionsTweetPreviewApiEnabled:                     true,
+			ResponsiveWebGraphqlTimelineNavigationEnabled:                  true,
+			ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled:      false,
+			C9sTweetAnatomyModeratorBadgeEnabled:                           true,
+			TweetypieUnmentionOptimizationEnabled:                          true,
+			ResponsiveWebEditTweetApiEnabled:                               true,
+			GraphqlIsTranslatableRWebTweetIsTranslatableEnabled:            true,
+			ViewCountsEverywhereApiEnabled:                                 true,
+			LongformNotetweetsConsumptionEnabled:                           true,
+			ResponsiveWebTwitterArticleTweetConsumptionEnabled:             false,
+			TweetAwardsWebTippingEnabled:                                   false,
+			FreedomOfSpeechNotReachFetchEnabled:                            true,
+			StandardizedNudgesMisinfo:                                      true,
+			TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: true,
+			RwebVideoTimestampsEnabled:                                     true,
+			LongformNotetweetsRichTextReadEnabled:                          true,
+			LongformNotetweetsInlineMediaEnabled:                           true,
+			ResponsiveWebMediaDownloadVideoEnabled:                         false,
+			ResponsiveWebEnhanceCardsEnabled:                               false,
+		},
+	}.String())
+	if err != nil {
+		panic(err)
+	}
+
+	var result APIV2Response
+	err = api.do_http(url.String(), "", &result)
+	return result, err
+}
+
+type PaginatedFollowees struct {
+	user_id UserID
+}
+
+func (p PaginatedFollowees) NextPage(api *API, cursor string) (APIV2Response, error) {
+	return api.GetFollowees(p.user_id, cursor)
+}
+func (p PaginatedFollowees) ToTweetTrove(r APIV2Response) (TweetTrove, error) {
+	return r.ToTweetTrove()
+}
+
+func GetFollowees(user_id UserID, how_many int) (TweetTrove, error) {
+	return the_api.GetPaginatedQuery(PaginatedFollowees{user_id}, how_many)
+}
+
+func (api *API) GetFollowers(user_id UserID, cursor string) (APIV2Response, error) {
+	url, err := url.Parse(GraphqlURL{
+		BaseUrl: "https://twitter.com/i/api/graphql/3_7xfjmh897x8h_n6QBqTA/Followers",
+		Variables: GraphqlVariables{
+			UserID:                 user_id,
+			Cursor:                 cursor,
+			Count:                  20,
+			IncludePromotedContent: false,
+		},
+		Features: GraphqlFeatures{
+			ResponsiveWebGraphqlExcludeDirectiveEnabled:                    true,
+			VerifiedPhoneLabelEnabled:                                      false,
+			CreatorSubscriptionsTweetPreviewApiEnabled:                     true,
+			ResponsiveWebGraphqlTimelineNavigationEnabled:                  true,
+			ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled:      false,
+			C9sTweetAnatomyModeratorBadgeEnabled:                           true,
+			TweetypieUnmentionOptimizationEnabled:                          true,
+			ResponsiveWebEditTweetApiEnabled:                               true,
+			GraphqlIsTranslatableRWebTweetIsTranslatableEnabled:            true,
+			ViewCountsEverywhereApiEnabled:                                 true,
+			LongformNotetweetsConsumptionEnabled:                           true,
+			ResponsiveWebTwitterArticleTweetConsumptionEnabled:             false,
+			TweetAwardsWebTippingEnabled:                                   false,
+			FreedomOfSpeechNotReachFetchEnabled:                            true,
+			StandardizedNudgesMisinfo:                                      true,
+			TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: true,
+			RwebVideoTimestampsEnabled:                                     true,
+			LongformNotetweetsRichTextReadEnabled:                          true,
+			LongformNotetweetsInlineMediaEnabled:                           true,
+			ResponsiveWebMediaDownloadVideoEnabled:                         false,
+			ResponsiveWebEnhanceCardsEnabled:                               false,
+		},
+	}.String())
+	if err != nil {
+		panic(err)
+	}
+
+	var result APIV2Response
+	err = api.do_http(url.String(), "", &result)
+	return result, err
+}
+
+type PaginatedFollowers struct {
+	user_id UserID
+}
+
+func (p PaginatedFollowers) NextPage(api *API, cursor string) (APIV2Response, error) {
+	return api.GetFollowers(p.user_id, cursor)
+}
+func (p PaginatedFollowers) ToTweetTrove(r APIV2Response) (TweetTrove, error) {
+	return r.ToTweetTrove()
+}
+
+func GetFollowers(user_id UserID, how_many int) (TweetTrove, error) {
+	return the_api.GetPaginatedQuery(PaginatedFollowers{user_id}, how_many)
+}
--- a/pkg/scraper/api_types_lists_test.go
+++ b/pkg/scraper/api_types_lists_test.go
@ -0,0 +1,36 @@
+package scraper_test
+
+import (
+	"testing"
+
+	"encoding/json"
+	"os"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
+)
+
+func TestParseFolloweesList(t *testing.T) {
+	assert := assert.New(t)
+	require := require.New(t)
+	data, err := os.ReadFile("test_responses/lists/followees.json")
+	require.NoError(err)
+
+	var resp APIV2Response
+	err = json.Unmarshal(data, &resp)
+	require.NoError(err)
+
+	tweet_trove, err := resp.ToTweetTrove()
+	require.NoError(err)
+
+	// Check users
+	assert.Len(tweet_trove.Users, 4)
+	_, is_ok := tweet_trove.Users[1349149096909668363]
+	assert.True(is_ok)
+
+	// Test cursor-bottom
+	bottom_cursor := resp.GetCursorBottom()
+	assert.Equal("0|1739810405452087290", bottom_cursor)
+}