111 lines
2.7 KiB
Go
111 lines
2.7 KiB
Go
package webserver
|
|
|
|
import (
|
|
"fmt"
|
|
"gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
|
|
"time"
|
|
)
|
|
|
|
var is_for_you_only = true // Do one initial scrape of the "for you" feed and then just regular feed after that
|
|
|
|
func (app *Application) background_scrape() {
|
|
// Avoid crashing the thread if a scrape fails
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
// TODO
|
|
fmt.Println("Background Home Timeline thread: panicked!")
|
|
if err, ok := r.(error); ok {
|
|
fmt.Println(err.Error())
|
|
} else {
|
|
fmt.Println(r)
|
|
}
|
|
}
|
|
}()
|
|
|
|
fmt.Println("Starting home timeline scrape...")
|
|
|
|
// Do nothing if scraping is currently disabled
|
|
if app.IsScrapingDisabled {
|
|
fmt.Println("Skipping home timeline scrape!")
|
|
return
|
|
}
|
|
|
|
fmt.Println("Scraping home timeline...")
|
|
trove, err := scraper.GetHomeTimeline("", is_for_you_only)
|
|
if err != nil {
|
|
app.ErrorLog.Printf("Background scrape failed: %s", err.Error())
|
|
return
|
|
}
|
|
fmt.Println("Saving scrape results...")
|
|
app.Profile.SaveTweetTrove(trove)
|
|
fmt.Println("Scraping succeeded.")
|
|
is_for_you_only = false
|
|
}
|
|
|
|
func (app *Application) background_user_likes_scrape() {
|
|
// Avoid crashing the thread if a scrape fails
|
|
defer func() {
|
|
if r := recover(); r != nil {
|
|
// TODO
|
|
fmt.Println("Background Home Timeline thread: panicked!")
|
|
if err, ok := r.(error); ok {
|
|
fmt.Println(err.Error())
|
|
} else {
|
|
fmt.Println(r)
|
|
}
|
|
}
|
|
}()
|
|
|
|
fmt.Println("Starting user likes scrape...")
|
|
|
|
// Do nothing if scraping is currently disabled
|
|
if app.IsScrapingDisabled {
|
|
fmt.Println("Skipping user likes scrape!")
|
|
return
|
|
}
|
|
|
|
fmt.Println("Scraping user likes...")
|
|
trove, err := scraper.GetUserLikes(app.ActiveUser.ID, 50) // TODO: parameterizable
|
|
if err != nil {
|
|
app.ErrorLog.Printf("Background scrape failed: %s", err.Error())
|
|
return
|
|
}
|
|
fmt.Println("Saving scrape results...")
|
|
app.Profile.SaveTweetTrove(trove)
|
|
fmt.Println("Scraping succeeded.")
|
|
}
|
|
|
|
func (app *Application) start_background() {
|
|
fmt.Println("Starting background")
|
|
|
|
// Scrape the home timeline every 3 minutes
|
|
go func() {
|
|
// Initial delay before the first task execution
|
|
time.Sleep(10 * time.Second)
|
|
app.background_scrape()
|
|
|
|
// Create a timer that triggers the background task every 3 minutes
|
|
interval := 3 * time.Minute // TODO: parameterizable
|
|
timer := time.NewTicker(interval)
|
|
defer timer.Stop()
|
|
|
|
for range timer.C {
|
|
app.background_scrape()
|
|
}
|
|
}()
|
|
|
|
// Scrape the logged-in user's likes every 10 minutes
|
|
go func() {
|
|
time.Sleep(15 * time.Second)
|
|
app.background_user_likes_scrape()
|
|
|
|
interval := 10 * time.Minute // TODO: parameterizable
|
|
timer := time.NewTicker(interval)
|
|
defer timer.Stop()
|
|
|
|
for range timer.C {
|
|
app.background_user_likes_scrape()
|
|
}
|
|
}()
|
|
}
|