Clean up the tests some more

scraper/test_responses/single_tweets/tweet_with_quoted_tweet_as_link2.json

@@ -0,0 +1 @@
+{"created_at":"Sat May 22 20:03:04 +0000 2021","id_str":"1396194922009661441","full_text":"sometimes they're too dimwitted to even get the wrong title right\nhttps://t.co/EHCAHKpez0","display_text_range":[0,89],"entities":{"urls":[{"url":"https://t.co/EHCAHKpez0","expanded_url":"https://twitter.com/RvaTeddy/status/1396194494710788100","display_url":"twitter.com/RvaTeddy/statu…","indices":[66,89]}]},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","in_reply_to_status_id_str":"1395882872729477131","in_reply_to_user_id_str":"44067298","in_reply_to_screen_name":"michaelmalice","user_id_str":"44067298","is_quote_status":true,"quoted_status_id_str":"1396194494710788100","quoted_status_permalink":{"url":"https://t.co/EHCAHKpez0","expanded":"https://twitter.com/RvaTeddy/status/1396194494710788100","display":"twitter.com/RvaTeddy/statu…"},"retweet_count":1,"favorite_count":41,"reply_count":6,"quote_count":0,"conversation_id_str":"1395882872729477131","possibly_sensitive_editable":true,"lang":"en","self_thread":{"id_str":"1395882872729477131"}}

scraper/tweet.go

@@ -192,6 +192,7 @@ func GetTweetFull(id TweetID) (tweets []Tweet, retweets []Retweet, users []User,
  * returns: a list of tweets, retweets and users in that response object
  */
 func ParseTweetResponse(resp TweetResponse) (tweets []Tweet, retweets []Retweet, users []User, err error) {
+	// TODO: TweetCollection maybe should be its own type
 	var new_tweet Tweet
 	var new_retweet Retweet
 	for _, single_tweet := range resp.GlobalObjects.Tweets {

scraper/tweet_test.go

@@ -1,7 +1,6 @@
 package scraper_test
 
 import (
-	"fmt"
 	"encoding/json"
 	"io/ioutil"
 	"testing"
@@ -9,34 +8,18 @@ import (
 	"offline_twitter/scraper"
 )
 
+
 func TestParseSingleTweet(t *testing.T) {
-	data, err := ioutil.ReadFile("test_responses/dave_smith_anarchist_handbook.json")
+	data, err := ioutil.ReadFile("test_responses/single_tweets/tweet_with_unicode_chars.json")
 	if err != nil {
 		panic(err)
 	}
-	var tweet_resp scraper.TweetResponse
+	var apitweet scraper.APITweet
-	err = json.Unmarshal(data, &tweet_resp)
+	err = json.Unmarshal(data, &apitweet)
 	if err != nil {
 		t.Errorf(err.Error())
 	}
-
+	tweet, err := scraper.ParseSingleTweet(apitweet)
-	tweets := tweet_resp.GlobalObjects.Tweets
-	users := tweet_resp.GlobalObjects.Users
-
-	if len(tweets) != 11 {
-		t.Errorf("Expected %d tweets, got %d instead", 11, len(tweets))
-	}
-
-	if len(users) != 11 {
-		t.Errorf("Expected %d users, got %d instead", 11, len(users))
-	}
-
-	dave_smith_tweet, ok := tweets["1395881699142160387"]
-	if !ok {
-		t.Errorf("Didn't find the Dave Smith tweet.")
-	}
-
-	tweet, err := scraper.ParseSingleTweet(dave_smith_tweet)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
@@ -59,88 +42,69 @@ func TestParseSingleTweet(t *testing.T) {
 	if tweet.PostedAt.Unix() != 1621639105 {
 		t.Errorf("Expected %d, got %d", 1621639105, tweet.PostedAt.Unix())
 	}
+
+	if tweet.QuotedTweet != 0 {
+		t.Errorf("Incorrectly believes it quote-tweets tweet with ID %d", tweet.QuotedTweet)
+	}
 }
 
-func TestParseSingleTweet2(t *testing.T) {
+func TestParseTweetWithImage(t *testing.T) {
-	data, err := ioutil.ReadFile("test_responses/midriffs_anarchist_cookbook.json")
+	data, err := ioutil.ReadFile("test_responses/single_tweets/tweet_with_image.json")
 	if err != nil {
 		panic(err)
 	}
-	var tweet_resp scraper.TweetResponse
+	var apitweet scraper.APITweet
-	err = json.Unmarshal(data, &tweet_resp)
+	err = json.Unmarshal(data, &apitweet)
 	if err != nil {
 		t.Errorf(err.Error())
 	}
-
+	tweet, err := scraper.ParseSingleTweet(apitweet)
-	tweets := tweet_resp.GlobalObjects.Tweets
-	users := tweet_resp.GlobalObjects.Users
-
-	if len(tweets) != 12 {
-		t.Errorf("Expected %d tweets, got %d instead", 11, len(tweets))
-	}
-
-	if len(users) != 11 {
-		t.Errorf("Expected %d users, got %d instead", 11, len(users))
-	}
-
-	t1, ok := tweets["1395882872729477131"]
-	if !ok {
-		t.Fatalf("Didn't find first tweet")
-	}
-	t2, ok := tweets["1396194922009661441"]
-	if !ok {
-		t.Fatalf("Didn't find second tweet")
-	}
-
-	tweet1, err := scraper.ParseSingleTweet(t1)
-	if err != nil {
-		t.Fatalf(err.Error())
-	}
-	tweet2, err := scraper.ParseSingleTweet(t2)
 	if err != nil {
 		t.Fatalf(err.Error())
 	}
 
 	expected_text := "this saddens me every time"
-	if tweet1.Text != expected_text {
+	if tweet.Text != expected_text {
-		t.Errorf("Expected: %q, got: %q", expected_text, tweet1.Text)
+		t.Errorf("Expected: %q, got: %q", expected_text, tweet.Text)
+	}
+	if len(tweet.Images) != 1 {
+		t.Errorf("Expected 1 images but got %d", len(tweet.Images))
 	}
-	expected_text = "sometimes they're too dimwitted to even get the wrong title right"
-	if tweet2.Text != expected_text {
-		t.Errorf("Expected: %q, got: %q", expected_text, tweet2.Text)
 }
 
-	if len(tweet1.Images) != 1 {
+func TestParseTweetWithQuotedTweetAsLink(t *testing.T) {
-		t.Errorf("Expected 1 images but got %d", len(tweet1.Images))
+	data, err := ioutil.ReadFile("test_responses/single_tweets/tweet_with_quoted_tweet_as_link2.json")
+	if err != nil {
+		panic(err)
 	}
-
+	var apitweet scraper.APITweet
-	if tweet2.InReplyTo != tweet1.ID {
+	err = json.Unmarshal(data, &apitweet)
-		t.Errorf("Expected %q, got %q", tweet1.ID, tweet2.InReplyTo)
+	if err != nil {
+		t.Errorf(err.Error())
 	}
-	if tweet1.QuotedTweet != 0 {
+	tweet, err := scraper.ParseSingleTweet(apitweet)
-		t.Errorf("Incorrectly believes it quote-tweets %q", tweet1.QuotedTweet)
-	}
-
-	if tweet2.QuotedTweet == 0 {
-		t.Errorf("Should be a quoted tweet")
-	}
-
-	quoted_tweet_, ok := tweets[fmt.Sprint(tweet2.QuotedTweet)]
-	if !ok {
-		t.Errorf("Couldn't find the quoted tweet")
-	}
-
-	quoted_tweet, err := scraper.ParseSingleTweet(quoted_tweet_)
 	if err != nil {
 		t.Errorf(err.Error())
 	}
 
-	expected_text = "I always liked \"The Anarchist's Cookbook.\""
+	expected_text := "sometimes they're too dimwitted to even get the wrong title right"
-	if quoted_tweet.Text != expected_text {
+	if tweet.Text != expected_text {
-		t.Errorf("Expected %q, got %q", expected_text, quoted_tweet.Text)
+		t.Errorf("Expected: %q, got: %q", expected_text, tweet.Text)
-	}
 	}
 
+	expected_replied_id := scraper.TweetID(1395882872729477131)
+	if tweet.InReplyTo != expected_replied_id {
+		t.Errorf("Expected %q, got %q", expected_replied_id, tweet.InReplyTo)
+	}
+	if len(tweet.ReplyMentions) != 1 || tweet.ReplyMentions[0] != "michaelmalice" {
+		t.Errorf("Wanted %v, got %v", []string{"michaelmalice"}, tweet.ReplyMentions)
+	}
+
+	expected_quoted_id := scraper.TweetID(1396194494710788100)
+	if tweet.QuotedTweet != expected_quoted_id {
+		t.Errorf("Should be a quoted tweet with ID %d, but got %d instead", expected_quoted_id, tweet.QuotedTweet)
+	}
+}
 
 func TestParseTweetWithVideo(t *testing.T) {
 	data, err := ioutil.ReadFile("test_responses/single_tweets/tweet_with_video.json")