Move content downloading to the background when scraping

cmd/twitter/main.go

@@ -270,7 +270,7 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) {
 	if err != nil {
 		die(err.Error(), false, -1)
 	}
-	profile.SaveTweetTrove(trove)
+	profile.SaveTweetTrove(trove, true)
 
 	happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)))
 }
@@ -291,7 +291,7 @@ func fetch_user_feed(handle string, how_many int) {
 	if err != nil {
 		die(fmt.Sprintf("Error scraping feed: %s\n  %s", handle, err.Error()), false, -2)
 	}
-	profile.SaveTweetTrove(trove)
+	profile.SaveTweetTrove(trove, true)
 
 	happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
 }
@@ -306,7 +306,7 @@ func get_user_likes(handle string, how_many int) {
 	if err != nil {
 		die(fmt.Sprintf("Error scraping feed: %s\n  %s", handle, err.Error()), false, -2)
 	}
-	profile.SaveTweetTrove(trove)
+	profile.SaveTweetTrove(trove, true)
 
 	happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
 }
@@ -316,7 +316,7 @@ func fetch_timeline(is_for_you bool) {
 	if err != nil {
 		die(fmt.Sprintf("Error fetching timeline:\n  %s", err.Error()), false, -2)
 	}
-	profile.SaveTweetTrove(trove)
+	profile.SaveTweetTrove(trove, true)
 
 	happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
 }
@@ -353,7 +353,7 @@ func search(query string, how_many int) {
 	if err != nil {
 		die(fmt.Sprintf("Error scraping search results: %s", err.Error()), false, -100)
 	}
-	profile.SaveTweetTrove(trove)
+	profile.SaveTweetTrove(trove, true)
 
 	happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)))
 }
@@ -413,7 +413,7 @@ func start_webserver(addr string) {
 
 func fetch_inbox(how_many int) {
 	trove, _ := scraper.GetInbox(how_many)
-	profile.SaveDMTrove(trove)
+	profile.SaveDMTrove(trove, true)
 	happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)))
 }
 
@@ -424,7 +424,7 @@ func fetch_dm(id string, how_many int) {
 	}
 	max_id := scraper.DMMessageID(^uint(0) >> 1)
 	trove := scraper.GetConversation(room.ID, max_id, how_many)
-	profile.SaveDMTrove(trove)
+	profile.SaveDMTrove(trove, true)
 	happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)))
 }
 
@@ -435,6 +435,6 @@ func send_dm(room_id string, text string, in_reply_to_id int) {
 	}
 
 	trove := scraper.SendDMMessage(room.ID, text, scraper.DMMessageID(in_reply_to_id))
-	profile.SaveDMTrove(trove)
+	profile.SaveDMTrove(trove, true)
 	happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)))
 }

internal/webserver/handler_tweet_detail.go

@@ -65,7 +65,8 @@ func (app *Application) ensure_tweet(id scraper.TweetID, is_forced bool, is_conv
 	if is_needing_scrape && !app.IsScrapingDisabled {
 		trove, err := scraper.GetTweetFullAPIV2(id, 50) // TODO: parameterizable
 		if err == nil {
-			app.Profile.SaveTweetTrove(trove)
+			app.Profile.SaveTweetTrove(trove, false)
+			go app.Profile.SaveTweetTrove(trove, true) // Download the content in the background
 			is_available = true
 		} else {
 			app.ErrorLog.Print(err)

internal/webserver/handler_user_feed.go

@@ -63,14 +63,16 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) {
 				app.ErrorLog.Print(err)
 				// TOOD: show error in UI
 			}
-			app.Profile.SaveTweetTrove(trove)
+			app.Profile.SaveTweetTrove(trove, false)
+			go app.Profile.SaveTweetTrove(trove, true)
 		} else if len(parts) == 2 && parts[1] == "likes" {
 			trove, err := scraper.GetUserLikes(user.ID, 50) // TODO: parameterizable
 			if err != nil {
 				app.ErrorLog.Print(err)
 				// TOOD: show error in UI
 			}
-			app.Profile.SaveTweetTrove(trove)
+			app.Profile.SaveTweetTrove(trove, false)
+			go app.Profile.SaveTweetTrove(trove, true)
 		}
 	}
 

internal/webserver/stopwatch.go

@@ -37,7 +37,8 @@ func (app *Application) background_scrape() {
 		return
 	}
 	fmt.Println("Saving scrape results...")
-	app.Profile.SaveTweetTrove(trove)
+	app.Profile.SaveTweetTrove(trove, false)
+	go app.Profile.SaveTweetTrove(trove, true)
 	fmt.Println("Scraping succeeded.")
 	is_for_you_only = false
 }
@@ -71,7 +72,8 @@ func (app *Application) background_user_likes_scrape() {
 		return
 	}
 	fmt.Println("Saving scrape results...")
-	app.Profile.SaveTweetTrove(trove)
+	app.Profile.SaveTweetTrove(trove, false)
+	go app.Profile.SaveTweetTrove(trove, true)
 	fmt.Println("Scraping succeeded.")
 }
 
@@ -107,7 +109,8 @@ func (app *Application) background_dm_polling_scrape() {
 		trove, inbox_cursor = scraper.PollInboxUpdates(inbox_cursor)
 	}
 	fmt.Println("Saving DM results...")
-	app.Profile.SaveDMTrove(trove)
+	app.Profile.SaveDMTrove(trove, false)
+	go app.Profile.SaveDMTrove(trove, true)
 	fmt.Println("Scraping DMs succeeded.")
 }
 

internal/webserver/tpl/includes/author_info.tpl

@@ -3,7 +3,11 @@
   <a class="unstyled-link" href="/{{.Handle}}">
     <img
       class="profile-image"
-      src="/content/{{.GetProfileImageLocalPath}}"
+      {{if .IsContentDownloaded}}
+        src="/content/{{.GetProfileImageLocalPath}}"
+      {{else}}
+        src="{{.ProfileImageUrl}}"
+      {{end}}
     />
   </a>
   <span class="name-and-handle">

internal/webserver/tpl/tweet_page_includes/chat_view.tpl

@@ -8,7 +8,11 @@
         <div class="dm-message-container">
           <div class="sender-profile-image-container">
             <a class="unstyled-link" href="/{{$user.Handle}}">
-              <img class="profile-image" src="/content/{{$user.GetProfileImageLocalPath}}" />
+              {{if $user.IsContentDownloaded}}
+                <img class="profile-image" src="/content/{{$user.GetProfileImageLocalPath}}" />
+              {{else}}
+                <img class="profile-image" src="{{$user.ProfileImageUrl}}" />
+              {{end}}
             </a>
           </div>
           <div class="dm-message-content-container">

internal/webserver/tpl/tweet_page_includes/single_tweet.tpl

@@ -57,7 +57,12 @@
         {{end}}
         {{template "text-with-entities" $main_tweet.Text}}
         {{range $main_tweet.Images}}
-          <img class="tweet-image" src="/content/images/{{.LocalFilename}}"
+          <img class="tweet-image"
+            {{if .IsDownloaded}}
+              src="/content/images/{{.LocalFilename}}"
+            {{else}}
+              src="{{.RemoteURL}}"
+            {{end}}
             width="{{.Width}}" height="{{.Height}}"
             {{if (gt (len $main_tweet.Images) 1)}}
               style="max-width: 45%"
@@ -65,8 +70,18 @@
           />
         {{end}}
         {{range $main_tweet.Videos}}
-          <video controls hx-trigger="click consume" width="{{.Width}}" height="{{.Height}}">
+          <video controls hx-trigger="click consume" width="{{.Width}}" height="{{.Height}}"
-            <source src="/content/videos/{{.LocalFilename}}">
+            {{if .IsDownloaded}}
+              poster="/content/video_thumbnails/{{.ThumbnailLocalPath}}"
+            {{else}}
+              poster="{{.ThumbnailRemoteUrl}}"
+            {{end}}
+          >
+            {{if .IsDownloaded}}
+              <source src="/content/videos/{{.LocalFilename}}">
+            {{else}}
+              <source src="{{.RemoteURL}}">
+            {{end}}
           </video>
         {{end}}
         {{range $main_tweet.Urls}}
@@ -77,7 +92,12 @@
               href="{{.Text}}"
               style="max-width: {{if (ne .ThumbnailWidth 0)}}{{.ThumbnailWidth}}px {{else}}fit-content {{end}}"
             >
-              <img src="/content/link_preview_images/{{.ThumbnailLocalPath}}"
+              <img
+                {{if .IsContentDownloaded}}
+                  src="/content/link_preview_images/{{.ThumbnailLocalPath}}"
+                {{else}}
+                  src="{{.ThumbnailRemoteUrl}}"
+                {{end}}
                 class="embedded-link-preview"
                 width="{{.ThumbnailWidth}}" height="{{.ThumbnailHeight}}"
               />

internal/webserver/tpl/user_feed.tpl

@@ -4,7 +4,11 @@
   {{$user := (user .UserID)}}
   <div class="user-feed-header">
     {{if $user.BannerImageLocalPath}}
-      <img class="profile-banner-image" src="/content/profile_images/{{$user.BannerImageLocalPath}}" />
+      {{if $user.IsContentDownloaded}}
+        <img class="profile-banner-image" src="/content/profile_images/{{$user.BannerImageLocalPath}}" />
+      {{else}}
+        <img class="profile-banner-image" src="{{$user.BannerImageUrl}}" />
+      {{end}}
     {{end}}
 
     <div class="user-feed-header-info-container">

pkg/persistence/dm_trove_queries.go

@@ -8,8 +8,8 @@ import (
 
 // Convenience function that saves all the objects in a TweetTrove.
 // Panics if anything goes wrong.
-func (p Profile) SaveDMTrove(trove DMTrove) {
+func (p Profile) SaveDMTrove(trove DMTrove, should_download bool) {
-	p.SaveTweetTrove(trove.TweetTrove)
+	p.SaveTweetTrove(trove.TweetTrove, should_download)
 
 	for _, r := range trove.Rooms {
 		err := p.SaveChatRoom(r)

pkg/persistence/tweet_trove_queries.go

@@ -8,7 +8,7 @@ import (
 
 // Convenience function that saves all the objects in a TweetTrove.
 // Panics if anything goes wrong.
-func (p Profile) SaveTweetTrove(trove TweetTrove) {
+func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool) {
 	for i, u := range trove.Users {
 		err := p.SaveUser(&u)
 		if err != nil {
@@ -37,10 +37,12 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) {
 		}
 		trove.Users[i] = u
 
-		// Download their tiny profile image
+		if should_download {
-		err = p.DownloadUserProfileImageTiny(&u)
+			// Download their tiny profile image
-		if err != nil {
+			err = p.DownloadUserProfileImageTiny(&u)
-			panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n  %w", u.ID, u.Handle, err))
+			if err != nil {
+				panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n  %w", u.ID, u.Handle, err))
+			}
 		}
 	}
 
@@ -57,9 +59,11 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) {
 			panic(fmt.Errorf("Error saving tweet ID %d:\n  %w", t.ID, err))
 		}
 
-		err = p.DownloadTweetContentFor(&t)
+		if should_download {
-		if err != nil {
+			err = p.DownloadTweetContentFor(&t)
-			panic(fmt.Errorf("Error downloading tweet content for tweet ID %d:\n  %w", t.ID, err))
+			if err != nil {
+				panic(fmt.Errorf("Error downloading tweet content for tweet ID %d:\n  %w", t.ID, err))
+			}
 		}
 	}