Add background scraping of logged-in user's Likes
This commit is contained in:
parent
6f66f68a15
commit
cefe8fc939
@ -11,22 +11,26 @@ var is_for_you_only = true // Do one initial scrape of the "for you" feed and th
|
|||||||
func (app *Application) background_scrape() {
|
func (app *Application) background_scrape() {
|
||||||
// Avoid crashing the thread if a scrape fails
|
// Avoid crashing the thread if a scrape fails
|
||||||
defer func() {
|
defer func() {
|
||||||
if err := recover(); err != nil {
|
if r := recover(); r != nil {
|
||||||
// TODO
|
// TODO
|
||||||
fmt.Println("Panicked!")
|
fmt.Println("Background Home Timeline thread: panicked!")
|
||||||
fmt.Printf("%#v\n", err)
|
if err, ok := r.(error); ok {
|
||||||
|
fmt.Println(err.Error())
|
||||||
|
} else {
|
||||||
|
fmt.Println(r)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
fmt.Println("Starting scrape...")
|
fmt.Println("Starting home timeline scrape...")
|
||||||
|
|
||||||
// Do nothing if scraping is currently disabled
|
// Do nothing if scraping is currently disabled
|
||||||
if app.IsScrapingDisabled {
|
if app.IsScrapingDisabled {
|
||||||
fmt.Println("Skipping scrape!")
|
fmt.Println("Skipping home timeline scrape!")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println("Scraping...")
|
fmt.Println("Scraping home timeline...")
|
||||||
trove, err := scraper.GetHomeTimeline("", is_for_you_only)
|
trove, err := scraper.GetHomeTimeline("", is_for_you_only)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
app.ErrorLog.Printf("Background scrape failed: %s", err.Error())
|
app.ErrorLog.Printf("Background scrape failed: %s", err.Error())
|
||||||
@ -38,16 +42,46 @@ func (app *Application) background_scrape() {
|
|||||||
is_for_you_only = false
|
is_for_you_only = false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (app *Application) background_user_likes_scrape() {
|
||||||
|
// Avoid crashing the thread if a scrape fails
|
||||||
|
defer func() {
|
||||||
|
if r := recover(); r != nil {
|
||||||
|
// TODO
|
||||||
|
fmt.Println("Background Home Timeline thread: panicked!")
|
||||||
|
if err, ok := r.(error); ok {
|
||||||
|
fmt.Println(err.Error())
|
||||||
|
} else {
|
||||||
|
fmt.Println(r)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
fmt.Println("Starting user likes scrape...")
|
||||||
|
|
||||||
|
// Do nothing if scraping is currently disabled
|
||||||
|
if app.IsScrapingDisabled {
|
||||||
|
fmt.Println("Skipping user likes scrape!")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fmt.Println("Scraping user likes...")
|
||||||
|
trove, err := scraper.GetUserLikes(app.ActiveUser.ID, 50) // TODO: parameterizable
|
||||||
|
if err != nil {
|
||||||
|
app.ErrorLog.Printf("Background scrape failed: %s", err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fmt.Println("Saving scrape results...")
|
||||||
|
app.Profile.SaveTweetTrove(trove)
|
||||||
|
fmt.Println("Scraping succeeded.")
|
||||||
|
}
|
||||||
|
|
||||||
func (app *Application) start_background() {
|
func (app *Application) start_background() {
|
||||||
// Start a goroutine to run the background task every 3 minutes
|
|
||||||
fmt.Println("Starting background")
|
fmt.Println("Starting background")
|
||||||
|
|
||||||
|
// Scrape the home timeline every 3 minutes
|
||||||
go func() {
|
go func() {
|
||||||
fmt.Println("Starting routine")
|
// Initial delay before the first task execution
|
||||||
|
time.Sleep(10 * time.Second)
|
||||||
// Initial delay before the first task execution (0 seconds here, adjust as needed)
|
|
||||||
initialDelay := 10 * time.Second
|
|
||||||
time.Sleep(initialDelay)
|
|
||||||
|
|
||||||
app.background_scrape()
|
app.background_scrape()
|
||||||
|
|
||||||
// Create a timer that triggers the background task every 3 minutes
|
// Create a timer that triggers the background task every 3 minutes
|
||||||
@ -56,10 +90,21 @@ func (app *Application) start_background() {
|
|||||||
defer timer.Stop()
|
defer timer.Stop()
|
||||||
|
|
||||||
for range timer.C {
|
for range timer.C {
|
||||||
// Execute the background task
|
|
||||||
fmt.Println("Starting routine")
|
|
||||||
|
|
||||||
app.background_scrape()
|
app.background_scrape()
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
// Scrape the logged-in user's likes every 10 minutes
|
||||||
|
go func() {
|
||||||
|
time.Sleep(15 * time.Second)
|
||||||
|
app.background_user_likes_scrape()
|
||||||
|
|
||||||
|
interval := 10 * time.Minute // TODO: parameterizable
|
||||||
|
timer := time.NewTicker(interval)
|
||||||
|
defer timer.Stop()
|
||||||
|
|
||||||
|
for range timer.C {
|
||||||
|
app.background_user_likes_scrape()
|
||||||
|
}
|
||||||
|
}()
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user