me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add scraper funcs for previous commit

Browse Source
This commit is contained in:
Alessio 2023-12-26 19:54:08 -06:00
parent df8093bbd9
commit dd68ee1fce
2 changed files with 155 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
pkg/scraper/api_types_lists.go Normal file
View File

@ -0,0 +1,119 @@
package scraper
import (
"net/url"
)
func (api *API) GetFollowees(user_id UserID, cursor string) (APIV2Response, error) {
url, err := url.Parse(GraphqlURL{
BaseUrl: "https://twitter.com/i/api/graphql/0yD6Eiv23DKXRDU9VxlG2A/Following",
Variables: GraphqlVariables{
UserID: user_id,
Cursor: cursor,
Count: 20,
IncludePromotedContent: false,
},
Features: GraphqlFeatures{
ResponsiveWebGraphqlExcludeDirectiveEnabled: true,
VerifiedPhoneLabelEnabled: false,
CreatorSubscriptionsTweetPreviewApiEnabled: true,
ResponsiveWebGraphqlTimelineNavigationEnabled: true,
ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled: false,
C9sTweetAnatomyModeratorBadgeEnabled: true,
TweetypieUnmentionOptimizationEnabled: true,
ResponsiveWebEditTweetApiEnabled: true,
GraphqlIsTranslatableRWebTweetIsTranslatableEnabled: true,
ViewCountsEverywhereApiEnabled: true,
LongformNotetweetsConsumptionEnabled: true,
ResponsiveWebTwitterArticleTweetConsumptionEnabled: false,
TweetAwardsWebTippingEnabled: false,
FreedomOfSpeechNotReachFetchEnabled: true,
StandardizedNudgesMisinfo: true,
TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: true,
RwebVideoTimestampsEnabled: true,
LongformNotetweetsRichTextReadEnabled: true,
LongformNotetweetsInlineMediaEnabled: true,
ResponsiveWebMediaDownloadVideoEnabled: false,
ResponsiveWebEnhanceCardsEnabled: false,
},
}.String())
if err != nil {
panic(err)
}
var result APIV2Response
err = api.do_http(url.String(), "", &result)
return result, err
}
type PaginatedFollowees struct {
user_id UserID
}
func (p PaginatedFollowees) NextPage(api *API, cursor string) (APIV2Response, error) {
return api.GetFollowees(p.user_id, cursor)
}
func (p PaginatedFollowees) ToTweetTrove(r APIV2Response) (TweetTrove, error) {
return r.ToTweetTrove()
}
func GetFollowees(user_id UserID, how_many int) (TweetTrove, error) {
return the_api.GetPaginatedQuery(PaginatedFollowees{user_id}, how_many)
}
func (api *API) GetFollowers(user_id UserID, cursor string) (APIV2Response, error) {
url, err := url.Parse(GraphqlURL{
BaseUrl: "https://twitter.com/i/api/graphql/3_7xfjmh897x8h_n6QBqTA/Followers",
Variables: GraphqlVariables{
UserID: user_id,
Cursor: cursor,
Count: 20,
IncludePromotedContent: false,
},
Features: GraphqlFeatures{
ResponsiveWebGraphqlExcludeDirectiveEnabled: true,
VerifiedPhoneLabelEnabled: false,
CreatorSubscriptionsTweetPreviewApiEnabled: true,
ResponsiveWebGraphqlTimelineNavigationEnabled: true,
ResponsiveWebGraphqlSkipUserProfileImageExtensionsEnabled: false,
C9sTweetAnatomyModeratorBadgeEnabled: true,
TweetypieUnmentionOptimizationEnabled: true,
ResponsiveWebEditTweetApiEnabled: true,
GraphqlIsTranslatableRWebTweetIsTranslatableEnabled: true,
ViewCountsEverywhereApiEnabled: true,
LongformNotetweetsConsumptionEnabled: true,
ResponsiveWebTwitterArticleTweetConsumptionEnabled: false,
TweetAwardsWebTippingEnabled: false,
FreedomOfSpeechNotReachFetchEnabled: true,
StandardizedNudgesMisinfo: true,
TweetWithVisibilityResultsPreferGqlLimitedActionsPolicyEnabled: true,
RwebVideoTimestampsEnabled: true,
LongformNotetweetsRichTextReadEnabled: true,
LongformNotetweetsInlineMediaEnabled: true,
ResponsiveWebMediaDownloadVideoEnabled: false,
ResponsiveWebEnhanceCardsEnabled: false,
},
}.String())
if err != nil {
panic(err)
}
var result APIV2Response
err = api.do_http(url.String(), "", &result)
return result, err
}
type PaginatedFollowers struct {
user_id UserID
}
func (p PaginatedFollowers) NextPage(api *API, cursor string) (APIV2Response, error) {
return api.GetFollowers(p.user_id, cursor)
}
func (p PaginatedFollowers) ToTweetTrove(r APIV2Response) (TweetTrove, error) {
return r.ToTweetTrove()
}
func GetFollowers(user_id UserID, how_many int) (TweetTrove, error) {
return the_api.GetPaginatedQuery(PaginatedFollowers{user_id}, how_many)
}

36
pkg/scraper/api_types_lists_test.go Normal file
View File

@ -0,0 +1,36 @@
package scraper_test
import (
"testing"
"encoding/json"
"os"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
)
func TestParseFolloweesList(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
data, err := os.ReadFile("test_responses/lists/followees.json")
require.NoError(err)
var resp APIV2Response
err = json.Unmarshal(data, &resp)
require.NoError(err)
tweet_trove, err := resp.ToTweetTrove()
require.NoError(err)
// Check users
assert.Len(tweet_trove.Users, 4)
_, is_ok := tweet_trove.Users[1349149096909668363]
assert.True(is_ok)
// Test cursor-bottom
bottom_cursor := resp.GetCursorBottom()
assert.Equal("0|1739810405452087290", bottom_cursor)
}