Add downloading for link previews

cmd/tests.sh

@@ -127,6 +127,16 @@ tw fetch_tweet https://twitter.com/CovfefeAnon/status/1428904664645394433
 urls_count_after_2x=$(sqlite3 twitter.db "select count(*) from urls")
 test $urls_count_after_2x = $urls_count_after
 
+# Download the link's preview image
+test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1428904664645394433") = "0"
+test $(sqlite3 twitter.db "select is_content_downloaded from urls where tweet_id = 1428904664645394433") = "0"
+test $(find link_preview_images/* | wc -l) = "0"
+tw download_tweet_content 1428904664645394433
+test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1428904664645394433") = "1"
+test $(sqlite3 twitter.db "select is_content_downloaded from urls where tweet_id = 1428904664645394433") = "1"
+test $(find link_preview_images/* | wc -l) = "1"
+test -f link_preview_images/WX1Rv2AJ_800x320_1.jpg
+
 
 # TODO: Maybe this file should be broken up into multiple test scripts
 

persistence/media_download.go

@@ -72,6 +72,21 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload
     return p.SaveVideo(*v)
 }
 
+/**
+ * Downloads an URL thumbnail image, and if successful, marks it as downloaded in the DB
+ */
+func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownloader) error {
+    if url.HasCard {
+        outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath)
+        err := downloader.Curl(url.ThumbnailRemoteUrl, outfile)
+        if err != nil {
+            return err
+        }
+    }
+    url.IsContentDownloaded = true
+    return p.SaveUrl(*url)
+}
+
 /**
  * Download a tweet's video and picture content.
  *
@@ -99,6 +114,13 @@ func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader M
             return err
         }
     }
+
+    for i := range t.Urls {
+        err := p.download_link_thumbnail(&t.Urls[i], downloader)
+        if err != nil {
+            return err
+        }
+    }
     t.IsContentDownloaded = true
     return p.SaveTweet(*t)
 }

persistence/profile.go

@@ -54,6 +54,7 @@ func NewProfile(target_dir string) (Profile, error) {
 	settings_file := path.Join(target_dir, "settings.yaml")
 	sqlite_file := path.Join(target_dir, "twitter.db")
 	profile_images_dir := path.Join(target_dir, "profile_images")
+	link_thumbnails_dir := path.Join(target_dir, "link_preview_images")
 	images_dir := path.Join(target_dir, "images")
 	videos_dir := path.Join(target_dir, "videos")
 
@@ -101,6 +102,13 @@ func NewProfile(target_dir string) (Profile, error) {
 		return Profile{}, err
 	}
 
+	// Create `link_thumbnail_images`
+	fmt.Printf("Creating............. %s/\n", link_thumbnails_dir)
+	err = os.Mkdir(link_thumbnails_dir, os.FileMode(0755))
+	if err != nil {
+		return Profile{}, err
+	}
+
 	// Create `images`
 	fmt.Printf("Creating............. %s/\n", images_dir)
 	err = os.Mkdir(images_dir, os.FileMode(0755))

persistence/profile_test.go

@@ -83,8 +83,8 @@ func TestNewProfile(t *testing.T) {
 	if err != nil {
 		panic(err)
 	}
-	if len(contents) != 6 {
-		t.Fatalf("Expected 6 contents, got %d instead", len(contents))
+	if len(contents) != 7 {
+		t.Fatalf("Expected 7 contents, got %d instead", len(contents))
 	}
 
 	expected_files := []struct {
@@ -92,6 +92,7 @@ func TestNewProfile(t *testing.T) {
 		isDir bool
 	} {
 		{"images", true},
+		{"link_preview_images", true},
 		{"profile_images", true},
 		{"settings.yaml", false},
 		{"twitter.db", false},