diff --git a/cmd/tests.sh b/cmd/tests.sh index 8a1b768..1ba26a4 100755 --- a/cmd/tests.sh +++ b/cmd/tests.sh @@ -127,6 +127,16 @@ tw fetch_tweet https://twitter.com/CovfefeAnon/status/1428904664645394433 urls_count_after_2x=$(sqlite3 twitter.db "select count(*) from urls") test $urls_count_after_2x = $urls_count_after +# Download the link's preview image +test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1428904664645394433") = "0" +test $(sqlite3 twitter.db "select is_content_downloaded from urls where tweet_id = 1428904664645394433") = "0" +test $(find link_preview_images/* | wc -l) = "0" +tw download_tweet_content 1428904664645394433 +test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1428904664645394433") = "1" +test $(sqlite3 twitter.db "select is_content_downloaded from urls where tweet_id = 1428904664645394433") = "1" +test $(find link_preview_images/* | wc -l) = "1" +test -f link_preview_images/WX1Rv2AJ_800x320_1.jpg + # TODO: Maybe this file should be broken up into multiple test scripts diff --git a/persistence/media_download.go b/persistence/media_download.go index db2325d..fbd7380 100644 --- a/persistence/media_download.go +++ b/persistence/media_download.go @@ -72,6 +72,21 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload return p.SaveVideo(*v) } +/** + * Downloads an URL thumbnail image, and if successful, marks it as downloaded in the DB + */ +func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownloader) error { + if url.HasCard { + outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath) + err := downloader.Curl(url.ThumbnailRemoteUrl, outfile) + if err != nil { + return err + } + } + url.IsContentDownloaded = true + return p.SaveUrl(*url) +} + /** * Download a tweet's video and picture content. * @@ -99,6 +114,13 @@ func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader M return err } } + + for i := range t.Urls { + err := p.download_link_thumbnail(&t.Urls[i], downloader) + if err != nil { + return err + } + } t.IsContentDownloaded = true return p.SaveTweet(*t) } diff --git a/persistence/profile.go b/persistence/profile.go index 1ef75ee..713ee88 100644 --- a/persistence/profile.go +++ b/persistence/profile.go @@ -54,6 +54,7 @@ func NewProfile(target_dir string) (Profile, error) { settings_file := path.Join(target_dir, "settings.yaml") sqlite_file := path.Join(target_dir, "twitter.db") profile_images_dir := path.Join(target_dir, "profile_images") + link_thumbnails_dir := path.Join(target_dir, "link_preview_images") images_dir := path.Join(target_dir, "images") videos_dir := path.Join(target_dir, "videos") @@ -101,6 +102,13 @@ func NewProfile(target_dir string) (Profile, error) { return Profile{}, err } + // Create `link_thumbnail_images` + fmt.Printf("Creating............. %s/\n", link_thumbnails_dir) + err = os.Mkdir(link_thumbnails_dir, os.FileMode(0755)) + if err != nil { + return Profile{}, err + } + // Create `images` fmt.Printf("Creating............. %s/\n", images_dir) err = os.Mkdir(images_dir, os.FileMode(0755)) diff --git a/persistence/profile_test.go b/persistence/profile_test.go index a9b7efa..14232e9 100644 --- a/persistence/profile_test.go +++ b/persistence/profile_test.go @@ -83,8 +83,8 @@ func TestNewProfile(t *testing.T) { if err != nil { panic(err) } - if len(contents) != 6 { - t.Fatalf("Expected 6 contents, got %d instead", len(contents)) + if len(contents) != 7 { + t.Fatalf("Expected 7 contents, got %d instead", len(contents)) } expected_files := []struct { @@ -92,6 +92,7 @@ func TestNewProfile(t *testing.T) { isDir bool } { {"images", true}, + {"link_preview_images", true}, {"profile_images", true}, {"settings.yaml", false}, {"twitter.db", false},