me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

REFACTOR: change 'fetch_tweet_only' to use the regular fetch-tweet scraper function, and extract the main tweet in the 'main' package instead of having a scraper function for it

Browse Source
This commit is contained in:
Alessio 2024-12-21 09:21:23 -08:00
parent 850662c3cb
commit e27cd12cdc
2 changed files with 12 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
cmd/twitter/main.go
View File

@ -372,17 +372,26 @@ func fetch_tweet_only(tweet_identifier string) {
die(err.Error(), false, -1)
}
tweet, err := api.GetTweet(tweet_id)
if is_scrape_failure(err) || errors.Is(err, scraper.ErrRateLimited) {
trove, err := api.GetTweetFullAPIV2(tweet_id, 1)
if err != nil {
die(fmt.Sprintf("Error fetching tweet: %s", err.Error()), false, -1)
}
// Find the main tweet and update its "is_conversation_downloaded" and "last_scraped_at"
tweet, ok := trove.Tweets[tweet_id]
if !ok {
panic("Trove didn't contain its own tweet!")
}
tweet.LastScrapedAt = scraper.Timestamp{time.Now()}
tweet.IsConversationScraped = true
log.Debug(tweet)
err2 := profile.SaveTweet(tweet)
if err2 != nil {
die(fmt.Sprintf("Error saving tweet: %s", err2.Error()), false, 4)
}
happy_exit("Saved the tweet", err)
happy_exit("Saved the tweet", nil)
}
/**

26
pkg/scraper/tweet.go
View File

@ -3,7 +3,6 @@ package scraper
import (
"database/sql/driver"
"errors"
"fmt"
"strings"
)
@ -74,28 +73,3 @@ type Tweet struct {
IsConversationScraped bool `db:"is_conversation_scraped"`
LastScrapedAt Timestamp `db:"last_scraped_at"`
}
// Get a single tweet with no replies from the API.
//
// args:
// - id: the ID of the tweet to get
//
// returns: the single Tweet
func (api *API) GetTweet(id TweetID) (Tweet, error) {
resp, err := api.GetTweetDetail(id, "")
if err != nil {
return Tweet{}, fmt.Errorf("Error getting tweet detail: %d\n %w", id, err)
}
trove, err := resp.ToTweetTrove()
if err != nil {
return Tweet{}, err
}
// Find the main tweet and update its "is_conversation_downloaded" and "last_scraped_at"
tweet, ok := trove.Tweets[id]
if !ok {
panic("Trove didn't contain its own tweet!")
}
tweet.LastScrapedAt = Timestamp{time.Now()}
tweet.IsConversationScraped = true
return tweet, nil
}