me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update some docstrings, rename a few methods

Browse Source
This commit is contained in:
Alessio 2021-08-16 20:35:15 -07:00
parent 2b23fcbc53
commit 4a3c12b04a
4 changed files with 51 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/fetch_user_feed_to_stdout/main.go
View File

@ -19,7 +19,7 @@ func main() {
log.Fatal("Error getting user profile: " + err.Error())
}
tweets, retweets, users, err := scraper.GetFeedFull(user.ID, 1)
tweets, retweets, users, err := scraper.GetUserFeedFor(user.ID, 1)
if err != nil {
log.Fatal("Error getting user feed: " + err.Error())
}

15
scraper/api_request_utils.go
View File

@ -52,10 +52,19 @@ func (api API) GetFeedFor(user_id UserID, cursor string) (TweetResponse, error)
return response, err
}
// Resend the request to get more tweets if necessary
func (api API) GetMoreTweets(user_id UserID, response *TweetResponse, max_tweets int) error {
/**
* Resend the request to get more tweets if necessary
*
* args:
* - user_id: the user's UserID
* - response: an "out" parameter; the TweetResponse that tweets, RTs and users will be appended to
* - min_tweets: the desired minimum amount of tweets to get
*/
func (api API) GetMoreTweetsFromFeed(user_id UserID, response *TweetResponse, min_tweets int) error {
// TODO user-feed-infinite-fetch: what if you reach the end of the user's timeline? Might loop
// forever getting no new tweets
last_response := response
for last_response.GetCursor() != "" && len(response.GlobalObjects.Tweets) < max_tweets {
for last_response.GetCursor() != "" && len(response.GlobalObjects.Tweets) < min_tweets {
fresh_response, err := api.GetFeedFor(user_id, last_response.GetCursor())
if err != nil {
return err

28
scraper/tweet.go
View File

@ -125,7 +125,14 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
}
// Return a single tweet, nothing else
/**
* Get a single tweet with no replies from the API.
*
* args:
* - id: the ID of the tweet to get
*
* returns: the single Tweet
*/
func GetTweet(id TweetID) (Tweet, error) {
api := API{}
tweet_response, err := api.GetTweet(id, "")
@ -143,8 +150,15 @@ func GetTweet(id TweetID) (Tweet, error) {
}
// Return a list of tweets, including the original and the rest of its thread,
// along with a list of associated users
/**
* Return a list of tweets, including the original and the rest of its thread,
* along with a list of associated users.
*
* args:
* - id: the ID of the tweet to get
*
* returns: the tweet, list of its replies and context, and users associated with those replies
*/
func GetTweetFull(id TweetID) (tweets []Tweet, retweets []Retweet, users []User, err error) {
api := API{}
tweet_response, err := api.GetTweet(id, "")
@ -162,6 +176,14 @@ func GetTweetFull(id TweetID) (tweets []Tweet, retweets []Retweet, users []User,
return ParseTweetResponse(tweet_response)
}
/**
* Parse an API response object into a list of tweets, retweets and users
*
* args:
* - resp: the response from the API
*
* returns: a list of tweets, retweets and users in that response object
*/
func ParseTweetResponse(resp TweetResponse) (tweets []Tweet, retweets []Retweet, users []User, err error) {
var new_tweet Tweet
var new_retweet Retweet

18
scraper/user_feed.go
View File

@ -1,18 +1,26 @@
package scraper
// Return a list of tweets, including the original and the rest of its thread,
// along with a list of associated users
func GetFeedFull(user_id UserID, max_tweets int) (tweets []Tweet, retweets []Retweet, users []User, err error) {
/**
* Get a list of tweets that appear on the given user's page, along with a list of associated
* users for any retweets.
*
* args:
* - user_id: the ID of the user whomst feed to fetch
* - min_tweets: get at least this many tweets, if there are any
*
* returns: a slice of Tweets, Retweets, and Users
*/
func GetUserFeedFor(user_id UserID, min_tweets int) (tweets []Tweet, retweets []Retweet, users []User, err error) {
api := API{}
tweet_response, err := api.GetFeedFor(user_id, "")
if err != nil {
return
}
if len(tweet_response.GlobalObjects.Tweets) < max_tweets &&
if len(tweet_response.GlobalObjects.Tweets) < min_tweets &&
tweet_response.GetCursor() != "" {
err = api.GetMoreTweets(user_id, &tweet_response, max_tweets)
err = api.GetMoreTweetsFromFeed(user_id, &tweet_response, min_tweets)
if err != nil {
return
}