me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move content downloading to the background when scraping

Browse Source
This commit is contained in:
Alessio 2023-11-26 22:13:47 -08:00
parent 08783d2bf4
commit fee47205ec
10 changed files with 73 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/twitter/main.go
View File

@ -270,7 +270,7 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) {
if err != nil {
die(err.Error(), false, -1)
}
profile.SaveTweetTrove(trove)
profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)))
}
@ -291,7 +291,7 @@ func fetch_user_feed(handle string, how_many int) {
if err != nil {
die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2)
}
profile.SaveTweetTrove(trove)
profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
}
@ -306,7 +306,7 @@ func get_user_likes(handle string, how_many int) {
if err != nil {
die(fmt.Sprintf("Error scraping feed: %s\n %s", handle, err.Error()), false, -2)
}
profile.SaveTweetTrove(trove)
profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
}
@ -316,7 +316,7 @@ func fetch_timeline(is_for_you bool) {
if err != nil {
die(fmt.Sprintf("Error fetching timeline:\n %s", err.Error()), false, -2)
}
profile.SaveTweetTrove(trove)
profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
}
@ -353,7 +353,7 @@ func search(query string, how_many int) {
if err != nil {
die(fmt.Sprintf("Error scraping search results: %s", err.Error()), false, -100)
}
profile.SaveTweetTrove(trove)
profile.SaveTweetTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d tweets and %d users", len(trove.Tweets), len(trove.Users)))
}
@ -413,7 +413,7 @@ func start_webserver(addr string) {
func fetch_inbox(how_many int) {
trove, _ := scraper.GetInbox(how_many)
profile.SaveDMTrove(trove)
profile.SaveDMTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)))
}
@ -424,7 +424,7 @@ func fetch_dm(id string, how_many int) {
}
max_id := scraper.DMMessageID(^uint(0) >> 1)
trove := scraper.GetConversation(room.ID, max_id, how_many)
profile.SaveDMTrove(trove)
profile.SaveDMTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)))
}
@ -435,6 +435,6 @@ func send_dm(room_id string, text string, in_reply_to_id int) {
}
trove := scraper.SendDMMessage(room.ID, text, scraper.DMMessageID(in_reply_to_id))
profile.SaveDMTrove(trove)
profile.SaveDMTrove(trove, true)
happy_exit(fmt.Sprintf("Saved %d messages from %d chats", len(trove.Messages), len(trove.Rooms)))
}

3
internal/webserver/handler_tweet_detail.go
View File

@ -65,7 +65,8 @@ func (app *Application) ensure_tweet(id scraper.TweetID, is_forced bool, is_conv
if is_needing_scrape && !app.IsScrapingDisabled {
trove, err := scraper.GetTweetFullAPIV2(id, 50) // TODO: parameterizable
if err == nil {
app.Profile.SaveTweetTrove(trove)
app.Profile.SaveTweetTrove(trove, false)
go app.Profile.SaveTweetTrove(trove, true) // Download the content in the background
is_available = true
} else {
app.ErrorLog.Print(err)

6
internal/webserver/handler_user_feed.go
View File

@ -63,14 +63,16 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) {
app.ErrorLog.Print(err)
// TOOD: show error in UI
}
app.Profile.SaveTweetTrove(trove)
app.Profile.SaveTweetTrove(trove, false)
go app.Profile.SaveTweetTrove(trove, true)
} else if len(parts) == 2 && parts[1] == "likes" {
trove, err := scraper.GetUserLikes(user.ID, 50) // TODO: parameterizable
if err != nil {
app.ErrorLog.Print(err)
// TOOD: show error in UI
}
app.Profile.SaveTweetTrove(trove)
app.Profile.SaveTweetTrove(trove, false)
go app.Profile.SaveTweetTrove(trove, true)
}
}

9
internal/webserver/stopwatch.go
View File

@ -37,7 +37,8 @@ func (app *Application) background_scrape() {
return
}
fmt.Println("Saving scrape results...")
app.Profile.SaveTweetTrove(trove)
app.Profile.SaveTweetTrove(trove, false)
go app.Profile.SaveTweetTrove(trove, true)
fmt.Println("Scraping succeeded.")
is_for_you_only = false
}
@ -71,7 +72,8 @@ func (app *Application) background_user_likes_scrape() {
return
}
fmt.Println("Saving scrape results...")
app.Profile.SaveTweetTrove(trove)
app.Profile.SaveTweetTrove(trove, false)
go app.Profile.SaveTweetTrove(trove, true)
fmt.Println("Scraping succeeded.")
}
@ -107,7 +109,8 @@ func (app *Application) background_dm_polling_scrape() {
trove, inbox_cursor = scraper.PollInboxUpdates(inbox_cursor)
}
fmt.Println("Saving DM results...")
app.Profile.SaveDMTrove(trove)
app.Profile.SaveDMTrove(trove, false)
go app.Profile.SaveDMTrove(trove, true)
fmt.Println("Scraping DMs succeeded.")
}

6
internal/webserver/tpl/includes/author_info.tpl
View File

@ -3,7 +3,11 @@
<a class="unstyled-link" href="/{{.Handle}}">
<img
class="profile-image"
src="/content/{{.GetProfileImageLocalPath}}"
{{if .IsContentDownloaded}}
src="/content/{{.GetProfileImageLocalPath}}"
{{else}}
src="{{.ProfileImageUrl}}"
{{end}}
/>
</a>
<span class="name-and-handle">

6
internal/webserver/tpl/tweet_page_includes/chat_view.tpl
View File

@ -8,7 +8,11 @@
<div class="dm-message-container">
<div class="sender-profile-image-container">
<a class="unstyled-link" href="/{{$user.Handle}}">
<img class="profile-image" src="/content/{{$user.GetProfileImageLocalPath}}" />
{{if $user.IsContentDownloaded}}
<img class="profile-image" src="/content/{{$user.GetProfileImageLocalPath}}" />
{{else}}
<img class="profile-image" src="{{$user.ProfileImageUrl}}" />
{{end}}
</a>
</div>
<div class="dm-message-content-container">

28
internal/webserver/tpl/tweet_page_includes/single_tweet.tpl
View File

@ -57,7 +57,12 @@
{{end}}
{{template "text-with-entities" $main_tweet.Text}}
{{range $main_tweet.Images}}
<img class="tweet-image" src="/content/images/{{.LocalFilename}}"
<img class="tweet-image"
{{if .IsDownloaded}}
src="/content/images/{{.LocalFilename}}"
{{else}}
src="{{.RemoteURL}}"
{{end}}
width="{{.Width}}" height="{{.Height}}"
{{if (gt (len $main_tweet.Images) 1)}}
style="max-width: 45%"
@ -65,8 +70,18 @@
/>
{{end}}
{{range $main_tweet.Videos}}
<video controls hx-trigger="click consume" width="{{.Width}}" height="{{.Height}}">
<source src="/content/videos/{{.LocalFilename}}">
<video controls hx-trigger="click consume" width="{{.Width}}" height="{{.Height}}"
{{if .IsDownloaded}}
poster="/content/video_thumbnails/{{.ThumbnailLocalPath}}"
{{else}}
poster="{{.ThumbnailRemoteUrl}}"
{{end}}
>
{{if .IsDownloaded}}
<source src="/content/videos/{{.LocalFilename}}">
{{else}}
<source src="{{.RemoteURL}}">
{{end}}
</video>
{{end}}
{{range $main_tweet.Urls}}
@ -77,7 +92,12 @@
href="{{.Text}}"
style="max-width: {{if (ne .ThumbnailWidth 0)}}{{.ThumbnailWidth}}px {{else}}fit-content {{end}}"
>
<img src="/content/link_preview_images/{{.ThumbnailLocalPath}}"
<img
{{if .IsContentDownloaded}}
src="/content/link_preview_images/{{.ThumbnailLocalPath}}"
{{else}}
src="{{.ThumbnailRemoteUrl}}"
{{end}}
class="embedded-link-preview"
width="{{.ThumbnailWidth}}" height="{{.ThumbnailHeight}}"
/>

6
internal/webserver/tpl/user_feed.tpl
View File

@ -4,7 +4,11 @@
{{$user := (user .UserID)}}
<div class="user-feed-header">
{{if $user.BannerImageLocalPath}}
<img class="profile-banner-image" src="/content/profile_images/{{$user.BannerImageLocalPath}}" />
{{if $user.IsContentDownloaded}}
<img class="profile-banner-image" src="/content/profile_images/{{$user.BannerImageLocalPath}}" />
{{else}}
<img class="profile-banner-image" src="{{$user.BannerImageUrl}}" />
{{end}}
{{end}}
<div class="user-feed-header-info-container">

4
pkg/persistence/dm_trove_queries.go
View File

@ -8,8 +8,8 @@ import (
// Convenience function that saves all the objects in a TweetTrove.
// Panics if anything goes wrong.
func (p Profile) SaveDMTrove(trove DMTrove) {
p.SaveTweetTrove(trove.TweetTrove)
func (p Profile) SaveDMTrove(trove DMTrove, should_download bool) {
p.SaveTweetTrove(trove.TweetTrove, should_download)
for _, r := range trove.Rooms {
err := p.SaveChatRoom(r)

20
pkg/persistence/tweet_trove_queries.go
View File

@ -8,7 +8,7 @@ import (
// Convenience function that saves all the objects in a TweetTrove.
// Panics if anything goes wrong.
func (p Profile) SaveTweetTrove(trove TweetTrove) {
func (p Profile) SaveTweetTrove(trove TweetTrove, should_download bool) {
for i, u := range trove.Users {
err := p.SaveUser(&u)
if err != nil {
@ -37,10 +37,12 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) {
}
trove.Users[i] = u
// Download their tiny profile image
err = p.DownloadUserProfileImageTiny(&u)
if err != nil {
panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
if should_download {
// Download their tiny profile image
err = p.DownloadUserProfileImageTiny(&u)
if err != nil {
panic(fmt.Errorf("Error downloading user content for user with ID %d and handle %s:\n %w", u.ID, u.Handle, err))
}
}
}
@ -57,9 +59,11 @@ func (p Profile) SaveTweetTrove(trove TweetTrove) {
panic(fmt.Errorf("Error saving tweet ID %d:\n %w", t.ID, err))
}
err = p.DownloadTweetContentFor(&t)
if err != nil {
panic(fmt.Errorf("Error downloading tweet content for tweet ID %d:\n %w", t.ID, err))
if should_download {
err = p.DownloadTweetContentFor(&t)
if err != nil {
panic(fmt.Errorf("Error downloading tweet content for tweet ID %d:\n %w", t.ID, err))
}
}
}