me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix some types to be more clear

Browse Source
This commit is contained in:
Alessio 2021-07-22 14:16:40 -07:00
parent 8ef1156d76
commit b5c33c2ad0
4 changed files with 17 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/fetch_tweet_to_stdout/main.go
View File

@ -12,7 +12,7 @@ import (
const INCLUDE_REPLIES = true; const INCLUDE_REPLIES = true;
// input: e.g., "https://twitter.com/michaelmalice/status/1395882872729477131" // input: e.g., "https://twitter.com/michaelmalice/status/1395882872729477131"
func parse_tweet(url string) (string, error) { func parse_tweet(url string) (scraper.TweetID, error) {
parts := strings.Split(url, "/") parts := strings.Split(url, "/")
if len(parts) != 6 { if len(parts) != 6 {
return "", fmt.Errorf("Tweet format isn't right (%d)", len(parts)) return "", fmt.Errorf("Tweet format isn't right (%d)", len(parts))
@ -20,7 +20,7 @@ func parse_tweet(url string) (string, error) {
if parts[0] != "https:" || parts[1] != "" || parts[2] != "twitter.com" || parts[4] != "status" { if parts[0] != "https:" || parts[1] != "" || parts[2] != "twitter.com" || parts[4] != "status" {
return "", fmt.Errorf("Tweet format isn't right") return "", fmt.Errorf("Tweet format isn't right")
} }
return parts[5], nil return scraper.TweetID(parts[5]), nil
} }
func main() { func main() {

8
scraper/api_request_utils.go
View File

@ -75,9 +75,9 @@ func (api API) GetMoreTweets(user_id UserID, response *TweetResponse, max_tweets
} }
func (api API) GetTweet(id string, cursor string) (TweetResponse, error) { func (api API) GetTweet(id TweetID, cursor string) (TweetResponse, error) {
client := &http.Client{Timeout: 10 * time.Second} client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("GET", API_CONVERSATION_BASE_PATH + id + ".json", nil) req, err := http.NewRequest("GET", API_CONVERSATION_BASE_PATH + string(id) + ".json", nil)
if err != nil { if err != nil {
return TweetResponse{}, err return TweetResponse{}, err
} }
@ -100,7 +100,7 @@ func (api API) GetTweet(id string, cursor string) (TweetResponse, error) {
if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusForbidden) { if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusForbidden) {
content, _ := ioutil.ReadAll(resp.Body) content, _ := ioutil.ReadAll(resp.Body)
return TweetResponse{}, fmt.Errorf("HTTP %d %s: %s", resp.StatusCode, resp.Status, content) return TweetResponse{}, fmt.Errorf("Error getting %q. HTTP %s: %s", req.URL, resp.Status, content)
} }
body, err := ioutil.ReadAll(resp.Body) body, err := ioutil.ReadAll(resp.Body)
@ -114,7 +114,7 @@ func (api API) GetTweet(id string, cursor string) (TweetResponse, error) {
} }
// Resend the request to get more replies if necessary // Resend the request to get more replies if necessary
func (api API) GetMoreReplies(tweet_id string, response *TweetResponse, max_replies int) error { func (api API) GetMoreReplies(tweet_id TweetID, response *TweetResponse, max_replies int) error {
last_response := response last_response := response
for last_response.GetCursor() != "" && len(response.GlobalObjects.Tweets) < max_replies { for last_response.GetCursor() != "" && len(response.GlobalObjects.Tweets) < max_replies {
fresh_response, err := api.GetTweet(tweet_id, last_response.GetCursor()) fresh_response, err := api.GetTweet(tweet_id, last_response.GetCursor())

12
scraper/tweet.go
View File

@ -23,7 +23,7 @@ type Tweet struct {
Urls []string Urls []string
Images []string Images []string
Mentions []string Mentions []UserHandle
Hashtags []string Hashtags []string
QuotedTweet TweetID QuotedTweet TweetID
} }
@ -67,7 +67,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
ret.Hashtags = append(ret.Hashtags, hashtag.Text) ret.Hashtags = append(ret.Hashtags, hashtag.Text)
} }
for _, mention := range apiTweet.Entities.Mentions { for _, mention := range apiTweet.Entities.Mentions {
ret.Mentions = append(ret.Mentions, mention.UserName) ret.Mentions = append(ret.Mentions, UserHandle(mention.UserName))
} }
ret.QuotedTweet = TweetID(apiTweet.QuotedStatusIDStr) ret.QuotedTweet = TweetID(apiTweet.QuotedStatusIDStr)
@ -77,14 +77,14 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
// Return a single tweet, nothing else // Return a single tweet, nothing else
func GetTweet(id string) (Tweet, error) { func GetTweet(id TweetID) (Tweet, error) {
api := API{} api := API{}
tweet_response, err := api.GetTweet(id, "") tweet_response, err := api.GetTweet(id, "")
if err != nil { if err != nil {
return Tweet{}, err return Tweet{}, fmt.Errorf("Error in API call: %s", err)
} }
single_tweet, ok := tweet_response.GlobalObjects.Tweets[id] single_tweet, ok := tweet_response.GlobalObjects.Tweets[string(id)]
if !ok { if !ok {
return Tweet{}, fmt.Errorf("Didn't get the tweet!\n%v", tweet_response) return Tweet{}, fmt.Errorf("Didn't get the tweet!\n%v", tweet_response)
@ -96,7 +96,7 @@ func GetTweet(id string) (Tweet, error) {
// Return a list of tweets, including the original and the rest of its thread, // Return a list of tweets, including the original and the rest of its thread,
// along with a list of associated users // along with a list of associated users
func GetTweetFull(id string) (tweets []Tweet, retweets []Retweet, users []User, err error) { func GetTweetFull(id TweetID) (tweets []Tweet, retweets []Retweet, users []User, err error) {
api := API{} api := API{}
tweet_response, err := api.GetTweet(id, "") tweet_response, err := api.GetTweet(id, "")
if err != nil { if err != nil {

9
scraper/user.go
View File

@ -3,17 +3,18 @@ package scraper
import ( import (
"time" "time"
"fmt" "fmt"
"strings"
) )
type UserID string type UserID string
type UserHandle string type UserHandle string
func UIDArrayToStrArray(uids []UserID) []string { func JoinArrayOfHandles(handles []UserHandle) string {
ret := []string{} ret := []string{}
for _, uid := range uids { for _, h := range handles {
ret = append(ret, string(uid)) ret = append(ret, string(h))
} }
return ret return strings.Join(ret, ",")
} }
type User struct { type User struct {