me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Some whitespace changes :V

Browse Source
This commit is contained in:
Alessio 2022-03-06 17:07:05 -08:00
parent 1d990e8a40
commit 7edc8ad5d3
20 changed files with 880 additions and 2001 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/twitter/helpers.go
View File

@ -3,14 +3,14 @@ package main
import ( import (
"fmt" "fmt"
"os" "os"
"regexp"
"strconv"
"strings"
"offline_twitter/scraper" "offline_twitter/scraper"
"offline_twitter/terminal_utils" "offline_twitter/terminal_utils"
"strings"
"strconv"
"regexp"
) )
/** /**
* Help message to print if command syntax is incorrect * Help message to print if command syntax is incorrect
*/ */
@ -73,13 +73,13 @@ This application downloads tweets from twitter and saves them in a SQLite databa
won't count toward the limit. won't count toward the limit.
` `
/** /**
* Helper function * Helper function
*/ */
func die(text string, display_help bool, exit_code int) { func die(text string, display_help bool, exit_code int) {
if text != "" { if text != "" {
fmt.Fprint(os.Stderr, terminal_utils.COLOR_RED + text + terminal_utils.COLOR_RESET + "\n") outstring := terminal_utils.COLOR_RED + text + terminal_utils.COLOR_RESET + "\n"
fmt.Fprint(os.Stderr, outstring)
} }
if display_help { if display_help {
fmt.Fprint(os.Stderr, help_message) fmt.Fprint(os.Stderr, help_message)
@ -91,8 +91,8 @@ func die(text string, display_help bool, exit_code int) {
* Print a happy exit message and exit * Print a happy exit message and exit
*/ */
func happy_exit(text string) { func happy_exit(text string) {
fmt.Printf(terminal_utils.COLOR_GREEN + text + terminal_utils.COLOR_RESET + "\n") fmt.Printf(terminal_utils.COLOR_GREEN + text + terminal_utils.COLOR_RESET + "\n")
fmt.Printf(terminal_utils.COLOR_GREEN + "Exiting successfully." + terminal_utils.COLOR_RESET + "\n") fmt.Printf(terminal_utils.COLOR_GREEN + "Exiting successfully." + terminal_utils.COLOR_RESET + "\n")
} }
/** /**

23
cmd/twitter/main.go
View File

@ -1,14 +1,14 @@
package main package main
import ( import (
"os"
"fmt"
"flag" "flag"
"fmt"
"os"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"offline_twitter/scraper"
"offline_twitter/persistence" "offline_twitter/persistence"
"offline_twitter/scraper"
) )
/** /**
@ -87,10 +87,10 @@ func main() {
profile, err = persistence.LoadProfile(*profile_dir) profile, err = persistence.LoadProfile(*profile_dir)
if err != nil { if err != nil {
die("Could not load profile: " + err.Error(), true, 2) die(fmt.Sprintf("Could not load profile: %s", err.Error()), true, 2)
} }
switch (operation) { switch operation {
case "create_profile": case "create_profile":
create_profile(target) create_profile(target)
case "fetch_user": case "fetch_user":
@ -116,7 +116,7 @@ func main() {
case "list_followed": case "list_followed":
list_followed() list_followed()
default: default:
die("Invalid operation: " + operation, true, 3) die(fmt.Sprintf("Invalid operation: %s", operation), true, 3)
} }
} }
@ -148,10 +148,10 @@ func fetch_user(handle scraper.UserHandle) {
err = profile.SaveUser(&user) err = profile.SaveUser(&user)
if err != nil { if err != nil {
die("Error saving user: " + err.Error(), false, 4) die(fmt.Sprintf("Error saving user: %s", err.Error()), false, 4)
} }
download_user_content(handle); download_user_content(handle)
happy_exit("Saved the user") happy_exit("Saved the user")
} }
@ -169,13 +169,13 @@ func fetch_tweet_only(tweet_identifier string) {
tweet, err := scraper.GetTweet(tweet_id) tweet, err := scraper.GetTweet(tweet_id)
if err != nil { if err != nil {
die("Error fetching tweet: " + err.Error(), false, -1) die(fmt.Sprintf("Error fetching tweet: %s", err.Error()), false, -1)
} }
log.Debug(tweet) log.Debug(tweet)
err = profile.SaveTweet(tweet) err = profile.SaveTweet(tweet)
if err != nil { if err != nil {
die("Error saving tweet: " + err.Error(), false, 4) die(fmt.Sprintf("Error saving tweet: %s", err.Error()), false, 4)
} }
happy_exit("Saved the tweet") happy_exit("Saved the tweet")
} }
@ -222,7 +222,6 @@ func fetch_user_feed(handle string, how_many int) {
happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
} }
func download_tweet_content(tweet_identifier string) { func download_tweet_content(tweet_identifier string) {
tweet_id, err := extract_id_from(tweet_identifier) tweet_id, err := extract_id_from(tweet_identifier)
if err != nil { if err != nil {
@ -253,7 +252,7 @@ func download_user_content(handle scraper.UserHandle) {
func search(query string) { func search(query string) {
trove, err := scraper.Search(query, 1000) trove, err := scraper.Search(query, 1000)
if err != nil { if err != nil {
die("Error scraping search results: " + err.Error(), false, -100) die(fmt.Sprintf("Error scraping search results: %s", err.Error()), false, -100)
} }
profile.SaveTweetTrove(trove) profile.SaveTweetTrove(trove)

247
persistence/media_download.go
View File

@ -1,21 +1,21 @@
package persistence package persistence
import ( import (
"fmt" "fmt"
"os" "io/ioutil"
"path" "net/http"
"net/http" "os"
"io/ioutil" "path"
"strings" "strings"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
type MediaDownloader interface { type MediaDownloader interface {
Curl(url string, outpath string) error Curl(url string, outpath string) error
} }
type DefaultDownloader struct {} type DefaultDownloader struct{}
/** /**
* Download a file over HTTP and save it. * Download a file over HTTP and save it.
@ -25,77 +25,75 @@ type DefaultDownloader struct {}
* - outpath: the path on disk to save it to * - outpath: the path on disk to save it to
*/ */
func (d DefaultDownloader) Curl(url string, outpath string) error { func (d DefaultDownloader) Curl(url string, outpath string) error {
println(url) println(url)
resp, err := http.Get(url) resp, err := http.Get(url)
if err != nil { if err != nil {
return err return err
} }
if resp.StatusCode != 200 { if resp.StatusCode != 200 {
return fmt.Errorf("Error %s: %s", url, resp.Status) return fmt.Errorf("Error %s: %s", url, resp.Status)
} }
data, err := ioutil.ReadAll(resp.Body) data, err := ioutil.ReadAll(resp.Body)
if err != nil { if err != nil {
return fmt.Errorf("Error downloading image %s: %s", url, err.Error()) return fmt.Errorf("Error downloading image %s: %s", url, err.Error())
} }
err = os.WriteFile(outpath, data, 0644) err = os.WriteFile(outpath, data, 0644)
if err != nil { if err != nil {
return fmt.Errorf("Error writing to path: %s, url: %s: %s", outpath, url, err.Error()) return fmt.Errorf("Error writing to path: %s, url: %s: %s", outpath, url, err.Error())
} }
return nil return nil
} }
/** /**
* Downloads an Image, and if successful, marks it as downloaded in the DB * Downloads an Image, and if successful, marks it as downloaded in the DB
*/ */
func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownloader) error { func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownloader) error {
outfile := path.Join(p.ProfileDir, "images", img.LocalFilename) outfile := path.Join(p.ProfileDir, "images", img.LocalFilename)
err := downloader.Curl(img.RemoteURL, outfile) err := downloader.Curl(img.RemoteURL, outfile)
if err != nil { if err != nil {
return err return err
} }
img.IsDownloaded = true img.IsDownloaded = true
return p.SaveImage(*img) return p.SaveImage(*img)
} }
/** /**
* Downloads a Video and its thumbnail, and if successful, marks it as downloaded in the DB * Downloads a Video and its thumbnail, and if successful, marks it as downloaded in the DB
*/ */
func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownloader) error { func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownloader) error {
// Download the video // Download the video
outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename) outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename)
err := downloader.Curl(v.RemoteURL, outfile) err := downloader.Curl(v.RemoteURL, outfile)
if err != nil { if err != nil {
return err return err
} }
// Download the thumbnail // Download the thumbnail
outfile = path.Join(p.ProfileDir, "video_thumbnails", v.ThumbnailLocalPath) outfile = path.Join(p.ProfileDir, "video_thumbnails", v.ThumbnailLocalPath)
err = downloader.Curl(v.ThumbnailRemoteUrl, outfile) err = downloader.Curl(v.ThumbnailRemoteUrl, outfile)
if err != nil { if err != nil {
return err return err
} }
v.IsDownloaded = true v.IsDownloaded = true
return p.SaveVideo(*v) return p.SaveVideo(*v)
} }
/** /**
* Downloads an URL thumbnail image, and if successful, marks it as downloaded in the DB * Downloads an URL thumbnail image, and if successful, marks it as downloaded in the DB
*/ */
func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownloader) error { func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownloader) error {
if url.HasCard && url.HasThumbnail { if url.HasCard && url.HasThumbnail {
outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath) outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath)
err := downloader.Curl(url.ThumbnailRemoteUrl, outfile) err := downloader.Curl(url.ThumbnailRemoteUrl, outfile)
if err != nil { if err != nil {
return err return err
} }
} }
url.IsContentDownloaded = true url.IsContentDownloaded = true
return p.SaveUrl(*url) return p.SaveUrl(*url)
} }
/** /**
@ -104,90 +102,89 @@ func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownl
* Wraps the `DownloadTweetContentWithInjector` method with the default (i.e., real) downloader. * Wraps the `DownloadTweetContentWithInjector` method with the default (i.e., real) downloader.
*/ */
func (p Profile) DownloadTweetContentFor(t *scraper.Tweet) error { func (p Profile) DownloadTweetContentFor(t *scraper.Tweet) error {
return p.DownloadTweetContentWithInjector(t, DefaultDownloader{}) return p.DownloadTweetContentWithInjector(t, DefaultDownloader{})
} }
/** /**
* Enable injecting a custom MediaDownloader (i.e., for testing) * Enable injecting a custom MediaDownloader (i.e., for testing)
*/ */
func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader MediaDownloader) error { func (p Profile) DownloadTweetContentWithInjector(t *scraper.Tweet, downloader MediaDownloader) error {
// Check if content needs to be downloaded; if not, just return // Check if content needs to be downloaded; if not, just return
if !p.CheckTweetContentDownloadNeeded(*t) { if !p.CheckTweetContentDownloadNeeded(*t) {
return nil return nil
} }
for i := range t.Images { for i := range t.Images {
err := p.download_tweet_image(&t.Images[i], downloader) err := p.download_tweet_image(&t.Images[i], downloader)
if err != nil { if err != nil {
return err return err
} }
} }
for i := range t.Videos { for i := range t.Videos {
err := p.download_tweet_video(&t.Videos[i], downloader) err := p.download_tweet_video(&t.Videos[i], downloader)
if err != nil { if err != nil {
return err return err
} }
} }
for i := range t.Urls { for i := range t.Urls {
err := p.download_link_thumbnail(&t.Urls[i], downloader) err := p.download_link_thumbnail(&t.Urls[i], downloader)
if err != nil { if err != nil {
return err return err
} }
} }
t.IsContentDownloaded = true t.IsContentDownloaded = true
return p.SaveTweet(*t) return p.SaveTweet(*t)
} }
/** /**
* Download a user's banner and profile images * Download a user's banner and profile images
*/ */
func (p Profile) DownloadUserContentFor(u *scraper.User) error { func (p Profile) DownloadUserContentFor(u *scraper.User) error {
return p.DownloadUserContentWithInjector(u, DefaultDownloader{}) return p.DownloadUserContentWithInjector(u, DefaultDownloader{})
} }
/** /**
* Enable injecting a custom MediaDownloader (i.e., for testing) * Enable injecting a custom MediaDownloader (i.e., for testing)
*/ */
func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader MediaDownloader) error { func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader MediaDownloader) error {
if !p.CheckUserContentDownloadNeeded(*u) { if !p.CheckUserContentDownloadNeeded(*u) {
return nil return nil
} }
var outfile string var outfile string
var target_url string var target_url string
if u.ProfileImageUrl == "" { if u.ProfileImageUrl == "" {
outfile = path.Join(p.ProfileDir, "profile_images", path.Base(scraper.DEFAULT_PROFILE_IMAGE_URL)) outfile = path.Join(p.ProfileDir, "profile_images", path.Base(scraper.DEFAULT_PROFILE_IMAGE_URL))
target_url = scraper.DEFAULT_PROFILE_IMAGE_URL target_url = scraper.DEFAULT_PROFILE_IMAGE_URL
} else { } else {
outfile = path.Join(p.ProfileDir, "profile_images", u.ProfileImageLocalPath) outfile = path.Join(p.ProfileDir, "profile_images", u.ProfileImageLocalPath)
target_url = u.ProfileImageUrl target_url = u.ProfileImageUrl
} }
err := downloader.Curl(target_url, outfile) err := downloader.Curl(target_url, outfile)
if err != nil { if err != nil {
return err return err
} }
// Skip it if there's no banner image // Skip it if there's no banner image
if u.BannerImageLocalPath != "" { if u.BannerImageLocalPath != "" {
outfile = path.Join(p.ProfileDir, "profile_images", u.BannerImageLocalPath) outfile = path.Join(p.ProfileDir, "profile_images", u.BannerImageLocalPath)
err = downloader.Curl(u.BannerImageUrl, outfile) err = downloader.Curl(u.BannerImageUrl, outfile)
if err != nil && strings.Contains(err.Error(), "404 Not Found") { if err != nil && strings.Contains(err.Error(), "404 Not Found") {
// Try adding "600x200". Not sure why this does this but sometimes it does. // Try adding "600x200". Not sure why this does this but sometimes it does.
err = downloader.Curl(u.BannerImageUrl + "/600x200", outfile) err = downloader.Curl(u.BannerImageUrl+"/600x200", outfile)
} }
if err != nil { if err != nil {
return err return err
} }
} }
u.IsContentDownloaded = true u.IsContentDownloaded = true
return p.SaveUser(u) return p.SaveUser(u)
} }
/** /**
@ -196,16 +193,16 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med
* If this user should have a big profile picture, defer to the regular `DownloadUserContentFor` method. * If this user should have a big profile picture, defer to the regular `DownloadUserContentFor` method.
*/ */
func (p Profile) DownloadUserProfileImageTiny(u *scraper.User) error { func (p Profile) DownloadUserProfileImageTiny(u *scraper.User) error {
if p.IsFollowing(u.Handle) { if p.IsFollowing(u.Handle) {
return p.DownloadUserContentFor(u) return p.DownloadUserContentFor(u)
} }
d := DefaultDownloader{} d := DefaultDownloader{}
outfile := path.Join(p.ProfileDir, "profile_images", u.GetTinyProfileImageLocalPath()) outfile := path.Join(p.ProfileDir, "profile_images", u.GetTinyProfileImageLocalPath())
if file_exists(outfile) { if file_exists(outfile) {
return nil return nil
} }
err := d.Curl(u.GetTinyProfileImageUrl(), outfile) err := d.Curl(u.GetTinyProfileImageUrl(), outfile)
return err return err
} }

119
persistence/media_download_test.go
View File

@ -1,95 +1,96 @@
package persistence_test package persistence_test
import ( import (
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
type FakeDownloader struct {} type FakeDownloader struct{}
func (d FakeDownloader) Curl(url string, outpath string) error { return nil } func (d FakeDownloader) Curl(url string, outpath string) error { return nil }
func test_all_downloaded(tweet scraper.Tweet, yes_or_no bool, t *testing.T) { func test_all_downloaded(tweet scraper.Tweet, yes_or_no bool, t *testing.T) {
error_msg := map[bool]string{ error_msg := map[bool]string{
true: "Expected to be downloaded, but it wasn't", true: "Expected to be downloaded, but it wasn't",
false: "Expected not to be downloaded, but it was", false: "Expected not to be downloaded, but it was",
}[yes_or_no] }[yes_or_no]
assert.Len(t, tweet.Images, 2) assert.Len(t, tweet.Images, 2)
assert.Len(t, tweet.Videos, 1) assert.Len(t, tweet.Videos, 1)
for _, img := range tweet.Images { for _, img := range tweet.Images {
if img.IsDownloaded != yes_or_no { if img.IsDownloaded != yes_or_no {
t.Errorf("%s: ImageID %d", error_msg, img.ID) t.Errorf("%s: ImageID %d", error_msg, img.ID)
} }
} }
for _, vid := range tweet.Videos { for _, vid := range tweet.Videos {
if vid.IsDownloaded != yes_or_no { if vid.IsDownloaded != yes_or_no {
t.Errorf("Expected not to be downloaded, but it was: VideoID %d", vid.ID) t.Errorf("Expected not to be downloaded, but it was: VideoID %d", vid.ID)
} }
} }
if tweet.IsContentDownloaded != yes_or_no { if tweet.IsContentDownloaded != yes_or_no {
t.Errorf("%s: the tweet", error_msg) t.Errorf("%s: the tweet", error_msg)
} }
} }
/** /**
* Downloading a Tweet's contents should mark the Tweet as downloaded * Downloading a Tweet's contents should mark the Tweet as downloaded
*/ */
func TestDownloadTweetContent(t *testing.T) { func TestDownloadTweetContent(t *testing.T) {
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_dummy_tweet() tweet := create_dummy_tweet()
// Persist the tweet // Persist the tweet
err := profile.SaveTweet(tweet) err := profile.SaveTweet(tweet)
require.NoError(t, err) require.NoError(t, err)
// Make sure everything is marked "not downloaded" // Make sure everything is marked "not downloaded"
test_all_downloaded(tweet, false, t) test_all_downloaded(tweet, false, t)
// Do the (fake) downloading // Do the (fake) downloading
err = profile.DownloadTweetContentWithInjector(&tweet, FakeDownloader{}) err = profile.DownloadTweetContentWithInjector(&tweet, FakeDownloader{})
require.NoError(t, err) require.NoError(t, err)
// It should all be marked "yes downloaded" now // It should all be marked "yes downloaded" now
test_all_downloaded(tweet, true, t) test_all_downloaded(tweet, true, t)
// Reload the Tweet (check db); should also be "yes downloaded" // Reload the Tweet (check db); should also be "yes downloaded"
new_tweet, err := profile.GetTweetById(tweet.ID) new_tweet, err := profile.GetTweetById(tweet.ID)
require.NoError(t, err) require.NoError(t, err)
test_all_downloaded(new_tweet, true, t) test_all_downloaded(new_tweet, true, t)
} }
/** /**
* Downloading a User's contents should mark the User as downloaded * Downloading a User's contents should mark the User as downloaded
*/ */
func TestDownloadUserContent(t *testing.T) { func TestDownloadUserContent(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
user := create_dummy_user() user := create_dummy_user()
// Persist the User // Persist the User
err := profile.SaveUser(&user) err := profile.SaveUser(&user)
require.NoError(t, err) require.NoError(t, err)
// Make sure the User is marked "not downloaded" // Make sure the User is marked "not downloaded"
assert.False(user.IsContentDownloaded) assert.False(user.IsContentDownloaded)
// Do the (fake) downloading // Do the (fake) downloading
err = profile.DownloadUserContentWithInjector(&user, FakeDownloader{}) err = profile.DownloadUserContentWithInjector(&user, FakeDownloader{})
require.NoError(t, err) require.NoError(t, err)
// The User should now be marked "yes downloaded" // The User should now be marked "yes downloaded"
assert.True(user.IsContentDownloaded) assert.True(user.IsContentDownloaded)
// Reload the User (check db); should also be "yes downloaded" // Reload the User (check db); should also be "yes downloaded"
new_user, err := profile.GetUserByID(user.ID) new_user, err := profile.GetUserByID(user.ID)
require.NoError(t, err) require.NoError(t, err)
assert.True(new_user.IsContentDownloaded) assert.True(new_user.IsContentDownloaded)
} }

220
persistence/media_queries.go
View File

@ -1,9 +1,9 @@
package persistence package persistence
import ( import (
"time" "time"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
/** /**
@ -13,16 +13,16 @@ import (
* - img: the Image to save * - img: the Image to save
*/ */
func (p Profile) SaveImage(img scraper.Image) error { func (p Profile) SaveImage(img scraper.Image) error {
_, err := p.DB.Exec(` _, err := p.DB.Exec(`
insert into images (id, tweet_id, width, height, remote_url, local_filename, is_downloaded) insert into images (id, tweet_id, width, height, remote_url, local_filename, is_downloaded)
values (?, ?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?, ?)
on conflict do update on conflict do update
set is_downloaded=(is_downloaded or ?) set is_downloaded=(is_downloaded or ?)
`, `,
img.ID, img.TweetID, img.Width, img.Height, img.RemoteURL, img.LocalFilename, img.IsDownloaded, img.ID, img.TweetID, img.Width, img.Height, img.RemoteURL, img.LocalFilename, img.IsDownloaded,
img.IsDownloaded, img.IsDownloaded,
) )
return err return err
} }
/** /**
@ -32,7 +32,7 @@ func (p Profile) SaveImage(img scraper.Image) error {
* - img: the Video to save * - img: the Video to save
*/ */
func (p Profile) SaveVideo(vid scraper.Video) error { func (p Profile) SaveVideo(vid scraper.Video) error {
_, err := p.DB.Exec(` _, err := p.DB.Exec(`
insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
duration, view_count, is_downloaded, is_gif) duration, view_count, is_downloaded, is_gif)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
@ -40,38 +40,38 @@ func (p Profile) SaveVideo(vid scraper.Video) error {
set is_downloaded=(is_downloaded or ?), set is_downloaded=(is_downloaded or ?),
view_count=max(view_count, ?) view_count=max(view_count, ?)
`, `,
vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath, vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath,
vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsGif, vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsGif,
vid.IsDownloaded, vid.ViewCount, vid.IsDownloaded, vid.ViewCount,
) )
return err return err
} }
/** /**
* Save an Url * Save an Url
*/ */
func (p Profile) SaveUrl(url scraper.Url) error { func (p Profile) SaveUrl(url scraper.Url) error {
_, err := p.DB.Exec(` _, err := p.DB.Exec(`
insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height,
thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded) thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update on conflict do update
set is_content_downloaded=(is_content_downloaded or ?) set is_content_downloaded=(is_content_downloaded or ?)
`, `,
url.TweetID, url.Domain, url.Text, url.ShortText, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailWidth, url.TweetID, url.Domain, url.Text, url.ShortText, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailWidth,
url.ThumbnailHeight, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded, url.ThumbnailHeight, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded,
url.IsContentDownloaded, url.IsContentDownloaded,
) )
return err return err
} }
/** /**
* Save a Poll * Save a Poll
*/ */
func (p Profile) SavePoll(poll scraper.Poll) error { func (p Profile) SavePoll(poll scraper.Poll) error {
_, err := p.DB.Exec(` _, err := p.DB.Exec(`
insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4,
choice4_votes, voting_duration, voting_ends_at, last_scraped_at) choice4_votes, voting_duration, voting_ends_at, last_scraped_at)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
@ -82,137 +82,135 @@ func (p Profile) SavePoll(poll scraper.Poll) error {
choice4_votes=?, choice4_votes=?,
last_scraped_at=? last_scraped_at=?
`, `,
poll.ID, poll.TweetID, poll.NumChoices, poll.Choice1, poll.Choice1_Votes, poll.Choice2, poll.Choice2_Votes, poll.Choice3, poll.ID, poll.TweetID, poll.NumChoices, poll.Choice1, poll.Choice1_Votes, poll.Choice2, poll.Choice2_Votes, poll.Choice3,
poll.Choice3_Votes, poll.Choice4, poll.Choice4_Votes, poll.VotingDuration, poll.VotingEndsAt.Unix(), poll.LastUpdatedAt.Unix(), poll.Choice3_Votes, poll.Choice4, poll.Choice4_Votes, poll.VotingDuration, poll.VotingEndsAt.Unix(), poll.LastUpdatedAt.Unix(),
poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt.Unix(), poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt.Unix(),
) )
return err return err
} }
/** /**
* Get the list of images for a tweet * Get the list of images for a tweet
*/ */
func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err error) { func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err error) {
stmt, err := p.DB.Prepare("select id, width, height, remote_url, local_filename, is_downloaded from images where tweet_id=?") stmt, err := p.DB.Prepare("select id, width, height, remote_url, local_filename, is_downloaded from images where tweet_id=?")
if err != nil { if err != nil {
return return
} }
defer stmt.Close() defer stmt.Close()
rows, err := stmt.Query(t.ID) rows, err := stmt.Query(t.ID)
if err != nil { if err != nil {
return return
} }
var img scraper.Image var img scraper.Image
for rows.Next() { for rows.Next() {
err = rows.Scan(&img.ID, &img.Width, &img.Height, &img.RemoteURL, &img.LocalFilename, &img.IsDownloaded) err = rows.Scan(&img.ID, &img.Width, &img.Height, &img.RemoteURL, &img.LocalFilename, &img.IsDownloaded)
if err != nil { if err != nil {
return return
} }
img.TweetID = t.ID img.TweetID = t.ID
imgs = append(imgs, img) imgs = append(imgs, img)
} }
return return
} }
/** /**
* Get the list of videos for a tweet * Get the list of videos for a tweet
*/ */
func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) { func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) {
stmt, err := p.DB.Prepare(` stmt, err := p.DB.Prepare(`
select id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration, view_count, select id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration, view_count,
is_downloaded, is_gif is_downloaded, is_gif
from videos from videos
where tweet_id = ? where tweet_id = ?
`) `)
if err != nil { if err != nil {
return return
} }
defer stmt.Close() defer stmt.Close()
rows, err := stmt.Query(t.ID) rows, err := stmt.Query(t.ID)
if err != nil { if err != nil {
return return
} }
var vid scraper.Video var vid scraper.Video
for rows.Next() { for rows.Next() {
err = rows.Scan(&vid.ID, &vid.Width, &vid.Height, &vid.RemoteURL, &vid.LocalFilename, &vid.ThumbnailRemoteUrl, err = rows.Scan(&vid.ID, &vid.Width, &vid.Height, &vid.RemoteURL, &vid.LocalFilename, &vid.ThumbnailRemoteUrl,
&vid.ThumbnailLocalPath, &vid.Duration, &vid.ViewCount, &vid.IsDownloaded, &vid.IsGif) &vid.ThumbnailLocalPath, &vid.Duration, &vid.ViewCount, &vid.IsDownloaded, &vid.IsGif)
if err != nil { if err != nil {
return return
} }
vid.TweetID = t.ID vid.TweetID = t.ID
vids = append(vids, vid) vids = append(vids, vid)
} }
return return
} }
/** /**
* Get the list of Urls for a Tweet * Get the list of Urls for a Tweet
*/ */
func (p Profile) GetUrlsForTweet(t scraper.Tweet) (urls []scraper.Url, err error) { func (p Profile) GetUrlsForTweet(t scraper.Tweet) (urls []scraper.Url, err error) {
stmt, err := p.DB.Prepare(` stmt, err := p.DB.Prepare(`
select domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, thumbnail_remote_url, select domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, thumbnail_remote_url,
thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded
from urls from urls
where tweet_id = ? where tweet_id = ?
order by rowid order by rowid
`) `)
if err != nil { if err != nil {
return return
} }
defer stmt.Close() defer stmt.Close()
rows, err := stmt.Query(t.ID) rows, err := stmt.Query(t.ID)
if err != nil { if err != nil {
return return
} }
var url scraper.Url var url scraper.Url
for rows.Next() { for rows.Next() {
err = rows.Scan(&url.Domain, &url.Text, &url.ShortText, &url.Title, &url.Description, &url.CreatorID, &url.SiteID, err = rows.Scan(&url.Domain, &url.Text, &url.ShortText, &url.Title, &url.Description, &url.CreatorID, &url.SiteID,
&url.ThumbnailWidth, &url.ThumbnailHeight, &url.ThumbnailRemoteUrl, &url.ThumbnailLocalPath, &url.HasCard, &url.ThumbnailWidth, &url.ThumbnailHeight, &url.ThumbnailRemoteUrl, &url.ThumbnailLocalPath, &url.HasCard,
&url.HasThumbnail, &url.IsContentDownloaded) &url.HasThumbnail, &url.IsContentDownloaded)
if err != nil { if err != nil {
return return
} }
url.TweetID = t.ID url.TweetID = t.ID
urls = append(urls, url) urls = append(urls, url)
} }
return return
} }
/** /**
* Get the list of Polls for a Tweet * Get the list of Polls for a Tweet
*/ */
func (p Profile) GetPollsForTweet(t scraper.Tweet) (polls []scraper.Poll, err error) { func (p Profile) GetPollsForTweet(t scraper.Tweet) (polls []scraper.Poll, err error) {
stmt, err := p.DB.Prepare(` stmt, err := p.DB.Prepare(`
select id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, choice4_votes, select id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, choice4_votes,
voting_duration, voting_ends_at, last_scraped_at voting_duration, voting_ends_at, last_scraped_at
from polls from polls
where tweet_id = ? where tweet_id = ?
`) `)
if err != nil { if err != nil {
return return
} }
defer stmt.Close() defer stmt.Close()
rows, err := stmt.Query(t.ID) rows, err := stmt.Query(t.ID)
if err != nil { if err != nil {
return return
} }
var poll scraper.Poll var poll scraper.Poll
var voting_ends_at int var voting_ends_at int
var last_scraped_at int var last_scraped_at int
for rows.Next() { for rows.Next() {
err = rows.Scan(&poll.ID, &poll.NumChoices, &poll.Choice1, &poll.Choice1_Votes, &poll.Choice2, &poll.Choice2_Votes, &poll.Choice3, err = rows.Scan(&poll.ID, &poll.NumChoices, &poll.Choice1, &poll.Choice1_Votes, &poll.Choice2, &poll.Choice2_Votes, &poll.Choice3,
&poll.Choice3_Votes, &poll.Choice4, &poll.Choice4_Votes, &poll.VotingDuration, &voting_ends_at, &last_scraped_at) &poll.Choice3_Votes, &poll.Choice4, &poll.Choice4_Votes, &poll.VotingDuration, &voting_ends_at, &last_scraped_at)
if err != nil { if err != nil {
return return
} }
poll.TweetID = t.ID poll.TweetID = t.ID
poll.VotingEndsAt = time.Unix(int64(voting_ends_at), 0) poll.VotingEndsAt = time.Unix(int64(voting_ends_at), 0)
poll.LastUpdatedAt = time.Unix(int64(last_scraped_at), 0) poll.LastUpdatedAt = time.Unix(int64(last_scraped_at), 0)
polls = append(polls, poll) polls = append(polls, poll)
} }
return return
} }

351
persistence/media_queries_test.go
View File

@ -2,281 +2,278 @@ package persistence_test
import ( import (
"testing" "testing"
"math/rand"
"time"
"github.com/go-test/deep" "math/rand"
"github.com/stretchr/testify/require" "time"
"offline_twitter/scraper" "github.com/go-test/deep"
"github.com/stretchr/testify/require"
"offline_twitter/scraper"
) )
/** /**
* Create an Image, save it, reload it, and make sure it comes back the same * Create an Image, save it, reload it, and make sure it comes back the same
*/ */
func TestSaveAndLoadImage(t *testing.T) { func TestSaveAndLoadImage(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_stable_tweet() tweet := create_stable_tweet()
// Create a fresh Image to test on // Create a fresh Image to test on
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
img := create_image_from_id(rand.Int()) img := create_image_from_id(rand.Int())
img.TweetID = tweet.ID img.TweetID = tweet.ID
// Save the Image // Save the Image
err := profile.SaveImage(img) err := profile.SaveImage(img)
require.NoError(err) require.NoError(err)
// Reload the Image // Reload the Image
imgs, err := profile.GetImagesForTweet(tweet) imgs, err := profile.GetImagesForTweet(tweet)
require.NoError(err) require.NoError(err)
var new_img scraper.Image var new_img scraper.Image
for index := range imgs { for index := range imgs {
if imgs[index].ID == img.ID { if imgs[index].ID == img.ID {
new_img = imgs[index] new_img = imgs[index]
} }
} }
require.Equal(img.ID, new_img.ID, "Could not find image for some reason") require.Equal(img.ID, new_img.ID, "Could not find image for some reason")
if diff := deep.Equal(img, new_img); diff != nil { if diff := deep.Equal(img, new_img); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
* Change an Image, save the changes, reload it, and check if it comes back the same * Change an Image, save the changes, reload it, and check if it comes back the same
*/ */
func TestModifyImage(t *testing.T) { func TestModifyImage(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_stable_tweet() tweet := create_stable_tweet()
img := tweet.Images[0] img := tweet.Images[0]
require.Equal(scraper.ImageID(-1), img.ID, "Got the wrong image back") require.Equal(scraper.ImageID(-1), img.ID, "Got the wrong image back")
img.IsDownloaded = true img.IsDownloaded = true
// Save the changes // Save the changes
err := profile.SaveImage(img) err := profile.SaveImage(img)
require.NoError(err) require.NoError(err)
// Reload it // Reload it
imgs, err := profile.GetImagesForTweet(tweet) imgs, err := profile.GetImagesForTweet(tweet)
require.NoError(err) require.NoError(err)
new_img := imgs[0] new_img := imgs[0]
require.Equal(imgs[0], new_img, "Got the wrong image back") require.Equal(imgs[0], new_img, "Got the wrong image back")
if diff := deep.Equal(img, new_img); diff != nil { if diff := deep.Equal(img, new_img); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
* Create an Video, save it, reload it, and make sure it comes back the same * Create an Video, save it, reload it, and make sure it comes back the same
*/ */
func TestSaveAndLoadVideo(t *testing.T) { func TestSaveAndLoadVideo(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_stable_tweet() tweet := create_stable_tweet()
// Create a fresh Video to test on // Create a fresh Video to test on
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
vid := create_video_from_id(rand.Int()) vid := create_video_from_id(rand.Int())
vid.TweetID = tweet.ID vid.TweetID = tweet.ID
vid.IsGif = true; vid.IsGif = true
// Save the Video // Save the Video
err := profile.SaveVideo(vid) err := profile.SaveVideo(vid)
require.NoError(err) require.NoError(err)
// Reload the Video // Reload the Video
vids, err := profile.GetVideosForTweet(tweet) vids, err := profile.GetVideosForTweet(tweet)
require.NoError(err) require.NoError(err)
var new_vid scraper.Video var new_vid scraper.Video
for index := range vids { for index := range vids {
if vids[index].ID == vid.ID { if vids[index].ID == vid.ID {
new_vid = vids[index] new_vid = vids[index]
} }
} }
require.Equal(vid.ID, new_vid.ID, "Could not find video for some reason") require.Equal(vid.ID, new_vid.ID, "Could not find video for some reason")
if diff := deep.Equal(vid, new_vid); diff != nil { if diff := deep.Equal(vid, new_vid); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
* Change an Video, save the changes, reload it, and check if it comes back the same * Change an Video, save the changes, reload it, and check if it comes back the same
*/ */
func TestModifyVideo(t *testing.T) { func TestModifyVideo(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_stable_tweet() tweet := create_stable_tweet()
vid := tweet.Videos[0] vid := tweet.Videos[0]
require.Equal(scraper.VideoID(-1), vid.ID, "Got the wrong video back") require.Equal(scraper.VideoID(-1), vid.ID, "Got the wrong video back")
vid.IsDownloaded = true vid.IsDownloaded = true
vid.ViewCount = 23000 vid.ViewCount = 23000
// Save the changes // Save the changes
err := profile.SaveVideo(vid) err := profile.SaveVideo(vid)
require.NoError(err) require.NoError(err)
// Reload it // Reload it
vids, err := profile.GetVideosForTweet(tweet) vids, err := profile.GetVideosForTweet(tweet)
require.NoError(err) require.NoError(err)
new_vid := vids[0] new_vid := vids[0]
require.Equal(vid.ID, new_vid.ID, "Got the wrong video back") require.Equal(vid.ID, new_vid.ID, "Got the wrong video back")
if diff := deep.Equal(vid, new_vid); diff != nil { if diff := deep.Equal(vid, new_vid); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
* Create an Url, save it, reload it, and make sure it comes back the same * Create an Url, save it, reload it, and make sure it comes back the same
*/ */
func TestSaveAndLoadUrl(t *testing.T) { func TestSaveAndLoadUrl(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_stable_tweet() tweet := create_stable_tweet()
// Create a fresh Url to test on // Create a fresh Url to test on
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
url := create_url_from_id(rand.Int()) url := create_url_from_id(rand.Int())
url.TweetID = tweet.ID url.TweetID = tweet.ID
// Save the Url // Save the Url
err := profile.SaveUrl(url) err := profile.SaveUrl(url)
require.NoError(err) require.NoError(err)
// Reload the Url // Reload the Url
urls, err := profile.GetUrlsForTweet(tweet) urls, err := profile.GetUrlsForTweet(tweet)
require.NoError(err) require.NoError(err)
var new_url scraper.Url var new_url scraper.Url
for index := range urls { for index := range urls {
if urls[index].Text == url.Text { if urls[index].Text == url.Text {
new_url = urls[index] new_url = urls[index]
} }
} }
require.Equal(url.Text, new_url.Text, "Could not find the url for some reason") require.Equal(url.Text, new_url.Text, "Could not find the url for some reason")
if diff := deep.Equal(url, new_url); diff != nil { if diff := deep.Equal(url, new_url); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
* Change an Url, save the changes, reload it, and check if it comes back the same * Change an Url, save the changes, reload it, and check if it comes back the same
*/ */
func TestModifyUrl(t *testing.T) { func TestModifyUrl(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_stable_tweet() tweet := create_stable_tweet()
url := tweet.Urls[0] url := tweet.Urls[0]
require.Equal("-1text", url.Text, "Got the wrong url back") require.Equal("-1text", url.Text, "Got the wrong url back")
url.IsContentDownloaded = true url.IsContentDownloaded = true
// Save the changes // Save the changes
err := profile.SaveUrl(url) err := profile.SaveUrl(url)
require.NoError(err) require.NoError(err)
// Reload it // Reload it
urls, err := profile.GetUrlsForTweet(tweet) urls, err := profile.GetUrlsForTweet(tweet)
require.NoError(err) require.NoError(err)
new_url := urls[0] new_url := urls[0]
require.Equal("-1text", url.Text, "Got the wrong url back") require.Equal("-1text", url.Text, "Got the wrong url back")
if diff := deep.Equal(url, new_url); diff != nil { if diff := deep.Equal(url, new_url); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
* Create a Poll, save it, reload it, and make sure it comes back the same * Create a Poll, save it, reload it, and make sure it comes back the same
*/ */
func TestSaveAndLoadPoll(t *testing.T) { func TestSaveAndLoadPoll(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_stable_tweet() tweet := create_stable_tweet()
poll := create_poll_from_id(rand.Int()) poll := create_poll_from_id(rand.Int())
poll.TweetID = tweet.ID poll.TweetID = tweet.ID
// Save the Poll // Save the Poll
err := profile.SavePoll(poll) err := profile.SavePoll(poll)
require.NoError(err) require.NoError(err)
// Reload the Poll // Reload the Poll
polls, err := profile.GetPollsForTweet(tweet) polls, err := profile.GetPollsForTweet(tweet)
require.NoError(err) require.NoError(err)
var new_poll scraper.Poll var new_poll scraper.Poll
for index := range polls { for index := range polls {
if polls[index].ID == poll.ID { if polls[index].ID == poll.ID {
new_poll = polls[index] new_poll = polls[index]
} }
} }
require.Equal(poll.ID, new_poll.ID, "Could not find poll for some reason") require.Equal(poll.ID, new_poll.ID, "Could not find poll for some reason")
if diff := deep.Equal(poll, new_poll); diff != nil { if diff := deep.Equal(poll, new_poll); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
* Change an Poll, save the changes, reload it, and check if it comes back the same * Change an Poll, save the changes, reload it, and check if it comes back the same
*/ */
func TestModifyPoll(t *testing.T) { func TestModifyPoll(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestMediaQueries" profile_path := "test_profiles/TestMediaQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_stable_tweet() tweet := create_stable_tweet()
poll := tweet.Polls[0] poll := tweet.Polls[0]
require.Equal("-1", poll.Choice1, "Got the wrong Poll back") require.Equal("-1", poll.Choice1, "Got the wrong Poll back")
poll.Choice1_Votes = 1200 // Increment it by 200 votes poll.Choice1_Votes = 1200 // Increment it by 200 votes
// Save the changes // Save the changes
err := profile.SavePoll(poll) err := profile.SavePoll(poll)
require.NoError(err) require.NoError(err)
// Reload it // Reload it
polls, err := profile.GetPollsForTweet(tweet) polls, err := profile.GetPollsForTweet(tweet)
require.NoError(err) require.NoError(err)
new_poll := polls[0] new_poll := polls[0]
require.Equal("-1", new_poll.Choice1, "Got the wrong poll back") require.Equal("-1", new_poll.Choice1, "Got the wrong poll back")
if diff := deep.Equal(poll, new_poll); diff != nil { if diff := deep.Equal(poll, new_poll); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }

21
persistence/profile.go
View File

@ -13,12 +13,12 @@ import (
//go:embed schema.sql //go:embed schema.sql
var sql_init string var sql_init string
type Settings struct {} type Settings struct{}
type Profile struct { type Profile struct {
ProfileDir string ProfileDir string
Settings Settings Settings Settings
DB *sql.DB DB *sql.DB
} }
/** /**
@ -27,11 +27,11 @@ type Profile struct {
type ErrTargetAlreadyExists struct { type ErrTargetAlreadyExists struct {
target string target string
} }
func (err ErrTargetAlreadyExists) Error() string { func (err ErrTargetAlreadyExists) Error() string {
return fmt.Sprintf("Target already exists: %s", err.target) return fmt.Sprintf("Target already exists: %s", err.target)
} }
/** /**
* Create a new profile in the given location. * Create a new profile in the given location.
* Fails if target location already exists (i.e., is a file or directory). * Fails if target location already exists (i.e., is a file or directory).
@ -124,7 +124,6 @@ func NewProfile(target_dir string) (Profile, error) {
return Profile{target_dir, settings, db}, nil return Profile{target_dir, settings, db}, nil
} }
/** /**
* Loads the profile at the given location. Fails if the given directory is not a Profile. * Loads the profile at the given location. Fails if the given directory is not a Profile.
* *
@ -139,9 +138,9 @@ func LoadProfile(profile_dir string) (Profile, error) {
sqlite_file := path.Join(profile_dir, "twitter.db") sqlite_file := path.Join(profile_dir, "twitter.db")
for _, file := range []string{ for _, file := range []string{
settings_file, settings_file,
sqlite_file, sqlite_file,
} { } {
if !file_exists(file) { if !file_exists(file) {
return Profile{}, fmt.Errorf("Invalid profile, could not find file: %s", file) return Profile{}, fmt.Errorf("Invalid profile, could not find file: %s", file)
} }
@ -157,15 +156,15 @@ func LoadProfile(profile_dir string) (Profile, error) {
return Profile{}, err return Profile{}, err
} }
db, err := sql.Open("sqlite3", sqlite_file + "?_foreign_keys=on&_journal_mode=WAL") db, err := sql.Open("sqlite3", sqlite_file+"?_foreign_keys=on&_journal_mode=WAL")
if err != nil { if err != nil {
return Profile{}, err return Profile{}, err
} }
ret := Profile{ ret := Profile{
ProfileDir: profile_dir, ProfileDir: profile_dir,
Settings: settings, Settings: settings,
DB: db, DB: db,
} }
err = ret.check_and_update_version() err = ret.check_and_update_version()

12
persistence/profile_test.go
View File

@ -2,8 +2,9 @@ package persistence_test
import ( import (
"testing" "testing"
"os"
"errors" "errors"
"os"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -23,7 +24,6 @@ func file_exists(path string) bool {
} }
} }
/** /**
* Should refuse to create a Profile if the target already exists (i.e., is a file or directory). * Should refuse to create a Profile if the target already exists (i.e., is a file or directory).
*/ */
@ -44,7 +44,6 @@ func TestNewProfileInvalidPath(t *testing.T) {
assert.True(t, is_right_type, "Expected 'ErrTargetAlreadyExists' error, got %T instead", err) assert.True(t, is_right_type, "Expected 'ErrTargetAlreadyExists' error, got %T instead", err)
} }
/** /**
* Should correctly create a new Profile * Should correctly create a new Profile
*/ */
@ -61,7 +60,7 @@ func TestNewProfile(t *testing.T) {
profile, err := persistence.NewProfile(profile_path) profile, err := persistence.NewProfile(profile_path)
require.NoError(err) require.NoError(err)
assert.Equal(profile_path,profile.ProfileDir) assert.Equal(profile_path, profile.ProfileDir)
// Check files were created // Check files were created
contents, err := os.ReadDir(profile_path) contents, err := os.ReadDir(profile_path)
@ -70,8 +69,8 @@ func TestNewProfile(t *testing.T) {
expected_files := []struct { expected_files := []struct {
filename string filename string
isDir bool isDir bool
} { }{
{"images", true}, {"images", true},
{"link_preview_images", true}, {"link_preview_images", true},
{"profile_images", true}, {"profile_images", true},
@ -92,7 +91,6 @@ func TestNewProfile(t *testing.T) {
assert.Equal(persistence.ENGINE_DATABASE_VERSION, version) assert.Equal(persistence.ENGINE_DATABASE_VERSION, version)
} }
/** /**
* Should correctly load the Profile * Should correctly load the Profile
*/ */

1
persistence/retweet_queries.go
View File

@ -20,7 +20,6 @@ func (p Profile) SaveRetweet(r scraper.Retweet) error {
return err return err
} }
/** /**
* Retrieve a Retweet by ID * Retrieve a Retweet by ID
*/ */

1
persistence/retweet_queries_test.go
View File

@ -8,7 +8,6 @@ import (
"github.com/go-test/deep" "github.com/go-test/deep"
) )
func TestSaveAndLoadRetweet(t *testing.T) { func TestSaveAndLoadRetweet(t *testing.T) {
require := require.New(t) require := require.New(t)

287
persistence/tweet_queries.go
View File

@ -1,21 +1,21 @@
package persistence package persistence
import ( import (
"time" "database/sql"
"strings" "strings"
"database/sql" "time"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
func (p Profile) SaveTweet(t scraper.Tweet) error { func (p Profile) SaveTweet(t scraper.Tweet) error {
db := p.DB db := p.DB
tx, err := db.Begin() tx, err := db.Begin()
if err != nil { if err != nil {
return err return err
} }
_, err = db.Exec(` _, err = db.Exec(`
insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id,
quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded, quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded,
is_conversation_scraped, last_scraped_at) is_conversation_scraped, last_scraped_at)
@ -30,74 +30,74 @@ func (p Profile) SaveTweet(t scraper.Tweet) error {
is_conversation_scraped=(is_conversation_scraped or ?), is_conversation_scraped=(is_conversation_scraped or ?),
last_scraped_at=max(last_scraped_at, ?) last_scraped_at=max(last_scraped_at, ?)
`, `,
t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID, t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID,
t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions),
strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt.Unix(), strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt.Unix(),
t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped,
t.LastScrapedAt.Unix(), t.LastScrapedAt.Unix(),
) )
if err != nil { if err != nil {
return err return err
} }
for _, url := range t.Urls { for _, url := range t.Urls {
err := p.SaveUrl(url) err := p.SaveUrl(url)
if err != nil { if err != nil {
return err return err
} }
} }
for _, image := range t.Images { for _, image := range t.Images {
err := p.SaveImage(image) err := p.SaveImage(image)
if err != nil { if err != nil {
return err return err
} }
} }
for _, video := range t.Videos { for _, video := range t.Videos {
err := p.SaveVideo(video) err := p.SaveVideo(video)
if err != nil { if err != nil {
return err return err
} }
} }
for _, hashtag := range t.Hashtags { for _, hashtag := range t.Hashtags {
_, err := db.Exec("insert into hashtags (tweet_id, text) values (?, ?) on conflict do nothing", t.ID, hashtag) _, err := db.Exec("insert into hashtags (tweet_id, text) values (?, ?) on conflict do nothing", t.ID, hashtag)
if err != nil { if err != nil {
return err return err
} }
} }
for _, poll := range t.Polls { for _, poll := range t.Polls {
err := p.SavePoll(poll) err := p.SavePoll(poll)
if err != nil { if err != nil {
return err return err
} }
} }
err = tx.Commit() err = tx.Commit()
if err != nil { if err != nil {
return err return err
} }
return nil return nil
} }
func (p Profile) IsTweetInDatabase(id scraper.TweetID) bool { func (p Profile) IsTweetInDatabase(id scraper.TweetID) bool {
db := p.DB db := p.DB
var dummy string var dummy string
err := db.QueryRow("select 1 from tweets where id = ?", id).Scan(&dummy) err := db.QueryRow("select 1 from tweets where id = ?", id).Scan(&dummy)
if err != nil { if err != nil {
if err != sql.ErrNoRows { if err != sql.ErrNoRows {
// A real error // A real error
panic(err) panic(err)
} }
return false return false
} }
return true return true
} }
func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
db := p.DB db := p.DB
stmt, err := db.Prepare(` stmt, err := db.Prepare(`
select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id,
mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded, mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded,
is_conversation_scraped, last_scraped_at is_conversation_scraped, last_scraped_at
@ -105,104 +105,103 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
where id = ? where id = ?
`) `)
if err != nil { if err != nil {
return scraper.Tweet{}, err return scraper.Tweet{}, err
} }
defer stmt.Close() defer stmt.Close()
var t scraper.Tweet var t scraper.Tweet
var postedAt int var postedAt int
var last_scraped_at int var last_scraped_at int
var mentions string var mentions string
var reply_mentions string var reply_mentions string
var hashtags string var hashtags string
row := stmt.QueryRow(id) row := stmt.QueryRow(id)
err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID, err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID,
&t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded, &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded,
&t.IsConversationScraped, &last_scraped_at) &t.IsConversationScraped, &last_scraped_at)
if err != nil { if err != nil {
return t, err return t, err
} }
t.PostedAt = time.Unix(int64(postedAt), 0) // args are `seconds` and `nanoseconds` t.PostedAt = time.Unix(int64(postedAt), 0) // args are `seconds` and `nanoseconds`
t.LastScrapedAt = time.Unix(int64(last_scraped_at), 0) t.LastScrapedAt = time.Unix(int64(last_scraped_at), 0)
t.Mentions = []scraper.UserHandle{} t.Mentions = []scraper.UserHandle{}
for _, m := range strings.Split(mentions, ",") { for _, m := range strings.Split(mentions, ",") {
if m != "" { if m != "" {
t.Mentions = append(t.Mentions, scraper.UserHandle(m)) t.Mentions = append(t.Mentions, scraper.UserHandle(m))
} }
} }
t.ReplyMentions = []scraper.UserHandle{} t.ReplyMentions = []scraper.UserHandle{}
for _, m := range strings.Split(reply_mentions, ",") { for _, m := range strings.Split(reply_mentions, ",") {
if m != "" { if m != "" {
t.ReplyMentions = append(t.ReplyMentions, scraper.UserHandle(m)) t.ReplyMentions = append(t.ReplyMentions, scraper.UserHandle(m))
} }
} }
t.Hashtags = []string{} t.Hashtags = []string{}
for _, h := range strings.Split(hashtags, ",") { for _, h := range strings.Split(hashtags, ",") {
if h != "" { if h != "" {
t.Hashtags = append(t.Hashtags, h) t.Hashtags = append(t.Hashtags, h)
} }
} }
imgs, err := p.GetImagesForTweet(t) imgs, err := p.GetImagesForTweet(t)
if err != nil { if err != nil {
return t, err return t, err
} }
t.Images = imgs t.Images = imgs
vids, err := p.GetVideosForTweet(t) vids, err := p.GetVideosForTweet(t)
if err != nil { if err != nil {
return t, err return t, err
} }
t.Videos = vids t.Videos = vids
polls, err := p.GetPollsForTweet(t) polls, err := p.GetPollsForTweet(t)
if err != nil { if err != nil {
return t, err return t, err
} }
t.Polls = polls t.Polls = polls
urls, err := p.GetUrlsForTweet(t) urls, err := p.GetUrlsForTweet(t)
t.Urls = urls t.Urls = urls
return t, err return t, err
} }
/** /**
* Populate the `User` field on a tweet with an actual User * Populate the `User` field on a tweet with an actual User
*/ */
func (p Profile) LoadUserFor(t *scraper.Tweet) error { func (p Profile) LoadUserFor(t *scraper.Tweet) error {
if t.User != nil { if t.User != nil {
// Already there, no need to load it // Already there, no need to load it
return nil return nil
} }
user, err := p.GetUserByID(t.UserID) user, err := p.GetUserByID(t.UserID)
if err != nil { if err != nil {
return err return err
} }
t.User = &user t.User = &user
return nil return nil
} }
/** /**
* Return `false` if the tweet is in the DB and has had its content downloaded, `false` otherwise * Return `false` if the tweet is in the DB and has had its content downloaded, `false` otherwise
*/ */
func (p Profile) CheckTweetContentDownloadNeeded(tweet scraper.Tweet) bool { func (p Profile) CheckTweetContentDownloadNeeded(tweet scraper.Tweet) bool {
row := p.DB.QueryRow(`select is_content_downloaded from tweets where id = ?`, tweet.ID) row := p.DB.QueryRow(`select is_content_downloaded from tweets where id = ?`, tweet.ID)
var is_content_downloaded bool var is_content_downloaded bool
err := row.Scan(&is_content_downloaded) err := row.Scan(&is_content_downloaded)
if err != nil { if err != nil {
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
return true return true
} else { } else {
panic(err) panic(err)
} }
} }
return !is_content_downloaded return !is_content_downloaded
} }

263
persistence/tweet_queries_test.go
View File

@ -1,59 +1,58 @@
package persistence_test package persistence_test
import ( import (
"testing" "testing"
"time" "time"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
"github.com/go-test/deep" "github.com/go-test/deep"
) )
/** /**
* Create a Tweet, save it, reload it, and make sure it comes back the same * Create a Tweet, save it, reload it, and make sure it comes back the same
*/ */
func TestSaveAndLoadTweet(t *testing.T) { func TestSaveAndLoadTweet(t *testing.T) {
profile_path := "test_profiles/TestTweetQueries" profile_path := "test_profiles/TestTweetQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_dummy_tweet() tweet := create_dummy_tweet()
tweet.IsContentDownloaded = true tweet.IsContentDownloaded = true
// Save the tweet // Save the tweet
err := profile.SaveTweet(tweet) err := profile.SaveTweet(tweet)
require.NoError(t, err) require.NoError(t, err)
// Reload the tweet // Reload the tweet
new_tweet, err := profile.GetTweetById(tweet.ID) new_tweet, err := profile.GetTweetById(tweet.ID)
require.NoError(t, err) require.NoError(t, err)
if diff := deep.Equal(tweet, new_tweet); diff != nil { if diff := deep.Equal(tweet, new_tweet); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
* Same as above, but with a tombstone * Same as above, but with a tombstone
*/ */
func TestSaveAndLoadTombstone(t *testing.T) { func TestSaveAndLoadTombstone(t *testing.T) {
profile_path := "test_profiles/TestTweetQueries" profile_path := "test_profiles/TestTweetQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_dummy_tombstone() tweet := create_dummy_tombstone()
// Save the tweet // Save the tweet
err := profile.SaveTweet(tweet) err := profile.SaveTweet(tweet)
require.NoError(t, err) require.NoError(t, err)
// Reload the tweet // Reload the tweet
new_tweet, err := profile.GetTweetById(tweet.ID) new_tweet, err := profile.GetTweetById(tweet.ID)
require.NoError(t, err) require.NoError(t, err)
if diff := deep.Equal(tweet, new_tweet); diff != nil { if diff := deep.Equal(tweet, new_tweet); diff != nil {
t.Error(diff) t.Error(diff)
} }
} }
/** /**
@ -65,152 +64,152 @@ func TestSaveAndLoadTombstone(t *testing.T) {
* - is_content_downloaded should only go from "no" to "yes" * - is_content_downloaded should only go from "no" to "yes"
*/ */
func TestNoWorseningTweet(t *testing.T) { func TestNoWorseningTweet(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestTweetQueries" profile_path := "test_profiles/TestTweetQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_dummy_tweet() tweet := create_dummy_tweet()
tweet.IsContentDownloaded = true tweet.IsContentDownloaded = true
tweet.IsStub = false tweet.IsStub = false
tweet.IsConversationScraped = true tweet.IsConversationScraped = true
tweet.LastScrapedAt = time.Unix(1000, 0) tweet.LastScrapedAt = time.Unix(1000, 0)
// Save the tweet // Save the tweet
err := profile.SaveTweet(tweet) err := profile.SaveTweet(tweet)
require.NoError(err) require.NoError(err)
// Worsen the tweet and re-save it // Worsen the tweet and re-save it
tweet.IsContentDownloaded = false tweet.IsContentDownloaded = false
tweet.IsStub = true tweet.IsStub = true
tweet.IsConversationScraped = false tweet.IsConversationScraped = false
tweet.LastScrapedAt = time.Unix(500, 0) tweet.LastScrapedAt = time.Unix(500, 0)
err = profile.SaveTweet(tweet) err = profile.SaveTweet(tweet)
require.NoError(err) require.NoError(err)
// Reload the tweet // Reload the tweet
new_tweet, err := profile.GetTweetById(tweet.ID) new_tweet, err := profile.GetTweetById(tweet.ID)
require.NoError(err) require.NoError(err)
assert.False(new_tweet.IsStub, "Should have preserved non-stub status") assert.False(new_tweet.IsStub, "Should have preserved non-stub status")
assert.True(new_tweet.IsContentDownloaded, "Should have preserved is-content-downloaded status") assert.True(new_tweet.IsContentDownloaded, "Should have preserved is-content-downloaded status")
assert.True(new_tweet.IsConversationScraped, "Should have preserved is-conversation-scraped status") assert.True(new_tweet.IsConversationScraped, "Should have preserved is-conversation-scraped status")
assert.Equal(int64(1000), new_tweet.LastScrapedAt.Unix(), "Should have preserved last-scraped-at time") assert.Equal(int64(1000), new_tweet.LastScrapedAt.Unix(), "Should have preserved last-scraped-at time")
} }
func TestModifyTweet(t *testing.T) { func TestModifyTweet(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestTweetQueries" profile_path := "test_profiles/TestTweetQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_dummy_tweet() tweet := create_dummy_tweet()
tweet.NumLikes = 1000 tweet.NumLikes = 1000
tweet.NumRetweets = 2000 tweet.NumRetweets = 2000
tweet.NumReplies = 3000 tweet.NumReplies = 3000
tweet.NumQuoteTweets = 4000 tweet.NumQuoteTweets = 4000
tweet.IsStub = true tweet.IsStub = true
tweet.IsContentDownloaded = false tweet.IsContentDownloaded = false
tweet.IsConversationScraped = false tweet.IsConversationScraped = false
tweet.LastScrapedAt = time.Unix(1000, 0) tweet.LastScrapedAt = time.Unix(1000, 0)
err := profile.SaveTweet(tweet) err := profile.SaveTweet(tweet)
require.NoError(err) require.NoError(err)
tweet.NumLikes = 1500 tweet.NumLikes = 1500
tweet.NumRetweets = 2500 tweet.NumRetweets = 2500
tweet.NumReplies = 3500 tweet.NumReplies = 3500
tweet.NumQuoteTweets = 4500 tweet.NumQuoteTweets = 4500
tweet.IsStub = false tweet.IsStub = false
tweet.IsContentDownloaded = true tweet.IsContentDownloaded = true
tweet.IsConversationScraped = true tweet.IsConversationScraped = true
tweet.LastScrapedAt = time.Unix(2000, 0) tweet.LastScrapedAt = time.Unix(2000, 0)
err = profile.SaveTweet(tweet) err = profile.SaveTweet(tweet)
require.NoError(err) require.NoError(err)
// Reload the tweet // Reload the tweet
new_tweet, err := profile.GetTweetById(tweet.ID) new_tweet, err := profile.GetTweetById(tweet.ID)
require.NoError(err) require.NoError(err)
assert.Equal(1500, new_tweet.NumLikes) assert.Equal(1500, new_tweet.NumLikes)
assert.Equal(2500, new_tweet.NumRetweets) assert.Equal(2500, new_tweet.NumRetweets)
assert.Equal(3500, new_tweet.NumReplies) assert.Equal(3500, new_tweet.NumReplies)
assert.Equal(4500, new_tweet.NumQuoteTweets) assert.Equal(4500, new_tweet.NumQuoteTweets)
assert.False(new_tweet.IsStub) assert.False(new_tweet.IsStub)
assert.True(new_tweet.IsContentDownloaded) assert.True(new_tweet.IsContentDownloaded)
assert.True(new_tweet.IsConversationScraped) assert.True(new_tweet.IsConversationScraped)
assert.Equal(int64(2000), new_tweet.LastScrapedAt.Unix()) assert.Equal(int64(2000), new_tweet.LastScrapedAt.Unix())
} }
/** /**
* Should correctly report whether the User exists in the database * Should correctly report whether the User exists in the database
*/ */
func TestIsTweetInDatabase(t *testing.T) { func TestIsTweetInDatabase(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestTweetQueries" profile_path := "test_profiles/TestTweetQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_dummy_tweet() tweet := create_dummy_tweet()
exists := profile.IsTweetInDatabase(tweet.ID) exists := profile.IsTweetInDatabase(tweet.ID)
require.False(exists) require.False(exists)
err := profile.SaveTweet(tweet) err := profile.SaveTweet(tweet)
require.NoError(err) require.NoError(err)
exists = profile.IsTweetInDatabase(tweet.ID) exists = profile.IsTweetInDatabase(tweet.ID)
assert.True(t, exists) assert.True(t, exists)
} }
/** /**
* Should correctly populate the `User` field on a Tweet * Should correctly populate the `User` field on a Tweet
*/ */
func TestLoadUserForTweet(t *testing.T) { func TestLoadUserForTweet(t *testing.T) {
require := require.New(t) require := require.New(t)
profile_path := "test_profiles/TestTweetQueries" profile_path := "test_profiles/TestTweetQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_dummy_tweet() tweet := create_dummy_tweet()
// Save the tweet // Save the tweet
err := profile.SaveTweet(tweet) err := profile.SaveTweet(tweet)
require.NoError(err) require.NoError(err)
require.Nil(tweet.User, "`User` field is already there for some reason") require.Nil(tweet.User, "`User` field is already there for some reason")
err = profile.LoadUserFor(&tweet) err = profile.LoadUserFor(&tweet)
require.NoError(err) require.NoError(err)
require.NotNil(tweet.User, "Did not load a user. It is still nil.") require.NotNil(tweet.User, "Did not load a user. It is still nil.")
} }
/** /**
* Test all the combinations for whether a tweet needs its content downloaded * Test all the combinations for whether a tweet needs its content downloaded
*/ */
func TestCheckTweetContentDownloadNeeded(t *testing.T) { func TestCheckTweetContentDownloadNeeded(t *testing.T) {
assert := assert.New(t) assert := assert.New(t)
profile_path := "test_profiles/TestTweetQueries" profile_path := "test_profiles/TestTweetQueries"
profile := create_or_load_profile(profile_path) profile := create_or_load_profile(profile_path)
tweet := create_dummy_tweet() tweet := create_dummy_tweet()
tweet.IsContentDownloaded = false tweet.IsContentDownloaded = false
// Non-saved tweets should need to be downloaded // Non-saved tweets should need to be downloaded
assert.True(profile.CheckTweetContentDownloadNeeded(tweet)) assert.True(profile.CheckTweetContentDownloadNeeded(tweet))
// Save the tweet // Save the tweet
err := profile.SaveTweet(tweet) err := profile.SaveTweet(tweet)
require.NoError(t, err) require.NoError(t, err)
// Should still need a download since `is_content_downloaded` is false // Should still need a download since `is_content_downloaded` is false
assert.True(profile.CheckTweetContentDownloadNeeded(tweet)) assert.True(profile.CheckTweetContentDownloadNeeded(tweet))
// Try again but this time with `is_content_downloaded` = true // Try again but this time with `is_content_downloaded` = true
tweet.IsContentDownloaded = true tweet.IsContentDownloaded = true
err = profile.SaveTweet(tweet) err = profile.SaveTweet(tweet)
require.NoError(t, err) require.NoError(t, err)
// Should no longer need a download // Should no longer need a download
assert.False(profile.CheckTweetContentDownloadNeeded(tweet)) assert.False(profile.CheckTweetContentDownloadNeeded(tweet))
} }

28
persistence/user_queries.go
View File

@ -15,21 +15,21 @@ import (
* - u: the User * - u: the User
*/ */
func (p Profile) SaveUser(u *scraper.User) error { func (p Profile) SaveUser(u *scraper.User) error {
if u.IsNeedingFakeID { if u.IsNeedingFakeID {
err := p.DB.QueryRow("select id from users where lower(handle) = lower(?)", u.Handle).Scan(&u.ID) err := p.DB.QueryRow("select id from users where lower(handle) = lower(?)", u.Handle).Scan(&u.ID)
if err == sql.ErrNoRows { if err == sql.ErrNoRows {
// We need to continue-- create a new fake user // We need to continue-- create a new fake user
u.ID = p.NextFakeUserID() u.ID = p.NextFakeUserID()
} else if err == nil { } else if err == nil {
// We're done; everything is fine (ID has already been scanned into the User) // We're done; everything is fine (ID has already been scanned into the User)
return nil return nil
} else { } else {
// A real error occurred // A real error occurred
panic(fmt.Sprintf("Error checking for existence of fake user with handle %q: %s", u.Handle, err.Error())) panic(fmt.Sprintf("Error checking for existence of fake user with handle %q: %s", u.Handle, err.Error()))
} }
} }
_, err := p.DB.Exec(` _, err := p.DB.Exec(`
insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private,
is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path,
pinned_tweet_id, is_content_downloaded, is_id_fake) pinned_tweet_id, is_content_downloaded, is_id_fake)

391
scraper/test_responses/api_v2/tweet_with_poll.json
View File

@ -1,390 +1 @@
{ {"result":{"__typename":"Tweet","rest_id":"1485692111106285571","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"card":{"rest_id":"card://1485692110472892424","legacy":{"binding_values":[{"key":"choice1_label","value":{"string_value":"1","type":"STRING"}},{"key":"choice2_label","value":{"string_value":"2","type":"STRING"}},{"key":"end_datetime_utc","value":{"string_value":"2022-01-25T19:12:56Z","type":"STRING"}},{"key":"counts_are_final","value":{"boolean_value":false,"type":"BOOLEAN"}},{"key":"choice2_count","value":{"string_value":"702","type":"STRING"}},{"key":"choice1_count","value":{"string_value":"891","type":"STRING"}},{"key":"choice4_label","value":{"string_value":"E","type":"STRING"}},{"key":"last_updated_datetime_utc","value":{"string_value":"2022-01-24T20:20:38Z","type":"STRING"}},{"key":"duration_minutes","value":{"string_value":"1440","type":"STRING"}},{"key":"choice3_count","value":{"string_value":"459","type":"STRING"}},{"key":"choice4_count","value":{"string_value":"1801","type":"STRING"}},{"key":"choice3_label","value":{"string_value":"C","type":"STRING"}},{"key":"api","value":{"string_value":"capi://passthrough/1","type":"STRING"}},{"key":"card_url","value":{"scribe_key":"card_url","string_value":"https://twitter.com","type":"STRING"}}],"card_platform":{"platform":{"audience":{"name":"production"},"device":{"name":"Swift","version":"12"}}},"name":"poll4choice_text_only","url":"card://1485692110472892424","user_refs":[]}},"legacy":{"created_at":"Mon Jan 24 19:12:56 +0000 2022","conversation_control":{"policy":"Community","conversation_owner":{"legacy":{"screen_name":"michaelmalice"}}},"conversation_id_str":"1485692111106285571","display_text_range":[0,158],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":71,"favorited":false,"full_text":"Which of these would most make you feel a disconnect from someone else?\n\n1) They don't like music\n2) They don't like pets\nC) They don't read\nE) They are vegan","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":12,"reply_count":11,"retweet_count":16,"retweeted":false,"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","user_id_str":"44067298","id_str":"1485692111106285571"}}}
"result":
{
"__typename": "Tweet",
"rest_id": "1485692111106285571",
"core":
{
"user_results":
{
"result":
{
"__typename": "User",
"id": "VXNlcjo0NDA2NzI5OA==",
"rest_id": "44067298",
"affiliates_highlighted_label":
{},
"has_nft_avatar": false,
"legacy":
{
"created_at": "Tue Jun 02 05:35:52 +0000 2009",
"default_profile": false,
"default_profile_image": false,
"description": "Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model",
"entities":
{
"description":
{
"urls":
[]
},
"url":
{
"urls":
[
{
"display_url": "amzn.to/3oInafv",
"expanded_url": "https://amzn.to/3oInafv",
"url": "https://t.co/7VDFOOtFK2",
"indices":
[
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 3840,
"followers_count": 334571,
"friends_count": 964,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 1434,
"location": "Austin",
"media_count": 9504,
"name": "Michael Malice",
"normal_followers_count": 334571,
"pinned_tweet_ids_str":
[
"1477347403023982596"
],
"profile_banner_extensions":
{
"mediaColor":
{
"r":
{
"ok":
{
"palette":
[
{
"percentage": 60.59,
"rgb":
{
"blue": 0,
"green": 0,
"red": 0
}
},
{
"percentage": 18.77,
"rgb":
{
"blue": 64,
"green": 60,
"red": 156
}
},
{
"percentage": 3.62,
"rgb":
{
"blue": 31,
"green": 29,
"red": 77
}
},
{
"percentage": 3.22,
"rgb":
{
"blue": 215,
"green": 199,
"red": 138
}
},
{
"percentage": 2.83,
"rgb":
{
"blue": 85,
"green": 79,
"red": 215
}
}
]
}
}
}
},
"profile_banner_url": "https://pbs.twimg.com/profile_banners/44067298/1615134676",
"profile_image_extensions":
{
"mediaColor":
{
"r":
{
"ok":
{
"palette":
[
{
"percentage": 50.78,
"rgb":
{
"blue": 249,
"green": 247,
"red": 246
}
},
{
"percentage": 17.4,
"rgb":
{
"blue": 51,
"green": 51,
"red": 205
}
},
{
"percentage": 9.43,
"rgb":
{
"blue": 124,
"green": 139,
"red": 210
}
},
{
"percentage": 6.38,
"rgb":
{
"blue": 47,
"green": 63,
"red": 116
}
},
{
"percentage": 3.17,
"rgb":
{
"blue": 65,
"green": 45,
"red": 46
}
}
]
}
}
}
},
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg",
"profile_interstitial_type": "",
"protected": false,
"screen_name": "michaelmalice",
"statuses_count": 138682,
"translator_type": "none",
"url": "https://t.co/7VDFOOtFK2",
"verified": true,
"withheld_in_countries":
[]
},
"super_follow_eligible": false,
"super_followed_by": false,
"super_following": false
}
}
},
"card":
{
"rest_id": "card://1485692110472892424",
"legacy":
{
"binding_values":
[
{
"key": "choice1_label",
"value":
{
"string_value": "1",
"type": "STRING"
}
},
{
"key": "choice2_label",
"value":
{
"string_value": "2",
"type": "STRING"
}
},
{
"key": "end_datetime_utc",
"value":
{
"string_value": "2022-01-25T19:12:56Z",
"type": "STRING"
}
},
{
"key": "counts_are_final",
"value":
{
"boolean_value": false,
"type": "BOOLEAN"
}
},
{
"key": "choice2_count",
"value":
{
"string_value": "702",
"type": "STRING"
}
},
{
"key": "choice1_count",
"value":
{
"string_value": "891",
"type": "STRING"
}
},
{
"key": "choice4_label",
"value":
{
"string_value": "E",
"type": "STRING"
}
},
{
"key": "last_updated_datetime_utc",
"value":
{
"string_value": "2022-01-24T20:20:38Z",
"type": "STRING"
}
},
{
"key": "duration_minutes",
"value":
{
"string_value": "1440",
"type": "STRING"
}
},
{
"key": "choice3_count",
"value":
{
"string_value": "459",
"type": "STRING"
}
},
{
"key": "choice4_count",
"value":
{
"string_value": "1801",
"type": "STRING"
}
},
{
"key": "choice3_label",
"value":
{
"string_value": "C",
"type": "STRING"
}
},
{
"key": "api",
"value":
{
"string_value": "capi://passthrough/1",
"type": "STRING"
}
},
{
"key": "card_url",
"value":
{
"scribe_key": "card_url",
"string_value": "https://twitter.com",
"type": "STRING"
}
}
],
"card_platform":
{
"platform":
{
"audience":
{
"name": "production"
},
"device":
{
"name": "Swift",
"version": "12"
}
}
},
"name": "poll4choice_text_only",
"url": "card://1485692110472892424",
"user_refs":
[]
}
},
"legacy":
{
"created_at": "Mon Jan 24 19:12:56 +0000 2022",
"conversation_control":
{
"policy": "Community",
"conversation_owner":
{
"legacy":
{
"screen_name": "michaelmalice"
}
}
},
"conversation_id_str": "1485692111106285571",
"display_text_range":
[
0,
158
],
"entities":
{
"user_mentions":
[],
"urls":
[],
"hashtags":
[],
"symbols":
[]
},
"favorite_count": 71,
"favorited": false,
"full_text": "Which of these would most make you feel a disconnect from someone else?\n\n1) They don't like music\n2) They don't like pets\nC) They don't read\nE) They are vegan",
"is_quote_status": false,
"lang": "en",
"possibly_sensitive": false,
"possibly_sensitive_editable": true,
"quote_count": 12,
"reply_count": 11,
"retweet_count": 16,
"retweeted": false,
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"user_id_str": "44067298",
"id_str": "1485692111106285571"
}
}
}

21
scraper/test_responses/single_tweets/tweet_with_html_entities.json
View File

@ -1,20 +1 @@
{ {"created_at":"Thu Dec 23 20:55:48 +0000 2021","id_str":"1474121585510563845","full_text":"By the 1970s the elite consensus was that \"the hunt for atomic spies\" had been a grotesque over-reaction to minor leaks that cost the lives of the Rosenbergs &amp; ruined many innocents. Only when the USSR fell was it discovered that they &amp; other spies had given away ALL the secrets","display_text_range":[0,288],"entities":{},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","user_id_str":"1239676915386068993","retweet_count":239,"favorite_count":1118,"reply_count":26,"quote_count":26,"conversation_id_str":"1474121585510563845","lang":"en"}
"created_at": "Thu Dec 23 20:55:48 +0000 2021",
"id_str": "1474121585510563845",
"full_text": "By the 1970s the elite consensus was that \"the hunt for atomic spies\" had been a grotesque over-reaction to minor leaks that cost the lives of the Rosenbergs &amp; ruined many innocents. Only when the USSR fell was it discovered that they &amp; other spies had given away ALL the secrets",
"display_text_range":
[
0,
288
],
"entities":
{},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"user_id_str": "1239676915386068993",
"retweet_count": 239,
"favorite_count": 1118,
"reply_count": 26,
"quote_count": 26,
"conversation_id_str": "1474121585510563845",
"lang": "en"
}

368
scraper/test_responses/single_tweets/tweet_with_quoted_tweet_and_url.json
View File

File diff suppressed because one or more lines are too long

332
scraper/test_responses/tombstones/tombstone_deleted.json
View File

@ -1,331 +1 @@
{ {"globalObjects":{"tweets":{"1454524255127887878":{"created_at":"Sat Oct 30 19:03:00 +0000 2021","id_str":"1454524255127887878","full_text":"@TastefulTyrant Halloween is often the easiest night of the year but women do thirst trap, too.","display_text_range":[16,95],"entities":{"user_mentions":[{"screen_name":"TastefulTyrant","name":"ᴛᴀꜱᴛᴇꜰᴜʟ ᴛʏʀᴀɴᴛ","id_str":"1218687933391298560","indices":[0,15]}]},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","in_reply_to_status_id_str":"1454521654781136902","in_reply_to_user_id_str":"1218687933391298560","in_reply_to_screen_name":"TastefulTyrant","user_id_str":"887434912529338375","retweet_count":0,"favorite_count":12,"reply_count":0,"quote_count":0,"conversation_id_str":"1454521654781136902","lang":"en"}},"users":{"887434912529338375":{"id_str":"887434912529338375","name":"Covfefe Anon","screen_name":"CovfefeAnon","location":"","description":"Not to be confused with 2001 Nobel Peace Prize winner Kofi Annan.\n\n54th Clause of the Magna Carta absolutist.\n\nCommentary from an NRx perspective.","entities":{"description":{}},"followers_count":8386,"fast_followers_count":0,"normal_followers_count":8386,"friends_count":497,"listed_count":59,"created_at":"Tue Jul 18 22:12:25 +0000 2017","favourites_count":175661,"statuses_count":26334,"media_count":1755,"profile_image_url_https":"https://pbs.twimg.com/profile_images/1392509603116617731/TDrNeUiZ_normal.jpg","profile_banner_url":"https://pbs.twimg.com/profile_banners/887434912529338375/1598514714","profile_image_extensions_alt_text":null,"profile_image_extensions_media_color":{"palette":[{"rgb":{"red":127,"green":125,"blue":102},"percentage":34.13},{"rgb":{"red":68,"green":50,"blue":44},"percentage":26.45},{"rgb":{"red":167,"green":170,"blue":176},"percentage":12.16},{"rgb":{"red":102,"green":47,"blue":31},"percentage":6.4},{"rgb":{"red":43,"green":52,"blue":65},"percentage":3.54}]},"profile_image_extensions_media_availability":null,"profile_image_extensions":{"mediaStats":{"r":{"missing":null},"ttl":-1}},"profile_banner_extensions_alt_text":null,"profile_banner_extensions_media_availability":null,"profile_banner_extensions_media_color":{"palette":[{"rgb":{"red":254,"green":254,"blue":254},"percentage":44.66},{"rgb":{"red":122,"green":116,"blue":123},"percentage":24.0},{"rgb":{"red":131,"green":164,"blue":104},"percentage":18.44},{"rgb":{"red":50,"green":50,"blue":50},"percentage":6.56},{"rgb":{"red":114,"green":156,"blue":99},"percentage":2.85}]},"profile_banner_extensions":{"mediaStats":{"r":{"missing":null},"ttl":-1}},"profile_link_color":"1B95E0","pinned_tweet_ids":[1005906691324596224],"pinned_tweet_ids_str":["1005906691324596224"],"advertiser_account_type":"promotable_user","advertiser_account_service_levels":["analytics"],"profile_interstitial_type":"","business_profile_state":"none","translator_type":"none","withheld_in_countries":[],"ext":{"highlightedLabel":{"r":{"ok":{}},"ttl":-1}}}},"moments":{},"cards":{},"places":{},"media":{},"broadcasts":{},"topics":{},"lists":{}},"timeline":{"id":"Conversation-1454521654781136902","instructions":[{"addEntries":{"entries":[{"entryId":"tombstone-7768850382073638905","sortIndex":"7768850382073638905","content":{"item":{"content":{"tombstone":{"displayType":"Inline","tombstoneInfo":{"text":"","richText":{"text":"This Tweet was deleted by the Tweet author. Learn more","entities":[{"fromIndex":44,"toIndex":54,"ref":{"url":{"urlType":"ExternalUrl","url":"https://help.twitter.com/rules-and-policies/notices-on-twitter"}}}],"rtl":false}}}}}}},{"entryId":"tweet-1454524255127887878","sortIndex":"7768847781726887929","content":{"item":{"content":{"tweet":{"id":"1454524255127887878","displayType":"Tweet"}}}}}]}},{"terminateTimeline":{"direction":"Top"}}],"responseObjects":{"feedbackActions":{}}}}
"globalObjects":
{
"tweets":
{
"1454524255127887878":
{
"created_at": "Sat Oct 30 19:03:00 +0000 2021",
"id_str": "1454524255127887878",
"full_text": "@TastefulTyrant Halloween is often the easiest night of the year but women do thirst trap, too.",
"display_text_range":
[
16,
95
],
"entities":
{
"user_mentions":
[
{
"screen_name": "TastefulTyrant",
"name": "ᴛᴀꜱᴛᴇꜰᴜʟ ᴛʏʀᴀɴᴛ",
"id_str": "1218687933391298560",
"indices":
[
0,
15
]
}
]
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"in_reply_to_status_id_str": "1454521654781136902",
"in_reply_to_user_id_str": "1218687933391298560",
"in_reply_to_screen_name": "TastefulTyrant",
"user_id_str": "887434912529338375",
"retweet_count": 0,
"favorite_count": 12,
"reply_count": 0,
"quote_count": 0,
"conversation_id_str": "1454521654781136902",
"lang": "en"
}
},
"users":
{
"887434912529338375":
{
"id_str": "887434912529338375",
"name": "Covfefe Anon",
"screen_name": "CovfefeAnon",
"location": "",
"description": "Not to be confused with 2001 Nobel Peace Prize winner Kofi Annan.\n\n54th Clause of the Magna Carta absolutist.\n\nCommentary from an NRx perspective.",
"entities":
{
"description":
{}
},
"followers_count": 8386,
"fast_followers_count": 0,
"normal_followers_count": 8386,
"friends_count": 497,
"listed_count": 59,
"created_at": "Tue Jul 18 22:12:25 +0000 2017",
"favourites_count": 175661,
"statuses_count": 26334,
"media_count": 1755,
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1392509603116617731/TDrNeUiZ_normal.jpg",
"profile_banner_url": "https://pbs.twimg.com/profile_banners/887434912529338375/1598514714",
"profile_image_extensions_alt_text": null,
"profile_image_extensions_media_color":
{
"palette":
[
{
"rgb":
{
"red": 127,
"green": 125,
"blue": 102
},
"percentage": 34.13
},
{
"rgb":
{
"red": 68,
"green": 50,
"blue": 44
},
"percentage": 26.45
},
{
"rgb":
{
"red": 167,
"green": 170,
"blue": 176
},
"percentage": 12.16
},
{
"rgb":
{
"red": 102,
"green": 47,
"blue": 31
},
"percentage": 6.4
},
{
"rgb":
{
"red": 43,
"green": 52,
"blue": 65
},
"percentage": 3.54
}
]
},
"profile_image_extensions_media_availability": null,
"profile_image_extensions":
{
"mediaStats":
{
"r":
{
"missing": null
},
"ttl": -1
}
},
"profile_banner_extensions_alt_text": null,
"profile_banner_extensions_media_availability": null,
"profile_banner_extensions_media_color":
{
"palette":
[
{
"rgb":
{
"red": 254,
"green": 254,
"blue": 254
},
"percentage": 44.66
},
{
"rgb":
{
"red": 122,
"green": 116,
"blue": 123
},
"percentage": 24.0
},
{
"rgb":
{
"red": 131,
"green": 164,
"blue": 104
},
"percentage": 18.44
},
{
"rgb":
{
"red": 50,
"green": 50,
"blue": 50
},
"percentage": 6.56
},
{
"rgb":
{
"red": 114,
"green": 156,
"blue": 99
},
"percentage": 2.85
}
]
},
"profile_banner_extensions":
{
"mediaStats":
{
"r":
{
"missing": null
},
"ttl": -1
}
},
"profile_link_color": "1B95E0",
"pinned_tweet_ids":
[
1005906691324596224
],
"pinned_tweet_ids_str":
[
"1005906691324596224"
],
"advertiser_account_type": "promotable_user",
"advertiser_account_service_levels":
[
"analytics"
],
"profile_interstitial_type": "",
"business_profile_state": "none",
"translator_type": "none",
"withheld_in_countries":
[],
"ext":
{
"highlightedLabel":
{
"r":
{
"ok":
{}
},
"ttl": -1
}
}
}
},
"moments":
{},
"cards":
{},
"places":
{},
"media":
{},
"broadcasts":
{},
"topics":
{},
"lists":
{}
},
"timeline":
{
"id": "Conversation-1454521654781136902",
"instructions":
[
{
"addEntries":
{
"entries":
[
{
"entryId": "tombstone-7768850382073638905",
"sortIndex": "7768850382073638905",
"content":
{
"item":
{
"content":
{
"tombstone":
{
"displayType": "Inline",
"tombstoneInfo":
{
"text": "",
"richText":
{
"text": "This Tweet was deleted by the Tweet author. Learn more",
"entities":
[
{
"fromIndex": 44,
"toIndex": 54,
"ref":
{
"url":
{
"urlType": "ExternalUrl",
"url": "https://help.twitter.com/rules-and-policies/notices-on-twitter"
}
}
}
],
"rtl": false
}
}
}
}
}
}
},
{
"entryId": "tweet-1454524255127887878",
"sortIndex": "7768847781726887929",
"content":
{
"item":
{
"content":
{
"tweet":
{
"id": "1454524255127887878",
"displayType": "Tweet"
}
}
}
}
}
]
}
},
{
"terminateTimeline":
{
"direction": "Top"
}
}
],
"responseObjects":
{
"feedbackActions":
{}
}
}
}

22
terminal_utils/colors.go
View File

@ -1,15 +1,17 @@
package terminal_utils package terminal_utils
/** /**
* Colors for terminal output * Colors for terminal output
*/ */
const COLOR_RESET = "\033[0m"
const COLOR_RED = "\033[31m" const (
const COLOR_GREEN = "\033[32m" COLOR_RESET = "\033[0m"
const COLOR_YELLOW = "\033[33m" COLOR_RED = "\033[31m"
const COLOR_BLUE = "\033[34m" COLOR_GREEN = "\033[32m"
const COLOR_PURPLE = "\033[35m" COLOR_YELLOW = "\033[33m"
const COLOR_CYAN = "\033[36m" COLOR_BLUE = "\033[34m"
const COLOR_GRAY = "\033[37m" COLOR_PURPLE = "\033[35m"
const COLOR_WHITE = "\033[97m" COLOR_CYAN = "\033[36m"
COLOR_GRAY = "\033[37m"
COLOR_WHITE = "\033[97m"
)

46
terminal_utils/formatting.go
View File

@ -1,46 +1,44 @@
package terminal_utils package terminal_utils
import ( import (
"time" "strings"
"strings" "time"
) )
/** /**
* Format a timestamp in human-readable form. * Format a timestamp in human-readable form.
*/ */
func FormatDate(t time.Time) string { func FormatDate(t time.Time) string {
return t.Format("Jan 2, 2006 15:04:05") return t.Format("Jan 2, 2006 15:04:05")
} }
/** /**
* Wrap lines to fixed width, while respecting word breaks * Wrap lines to fixed width, while respecting word breaks
*/ */
func WrapParagraph(paragraph string, width int) []string { func WrapParagraph(paragraph string, width int) []string {
var lines []string var lines []string
i := 0 i := 0
for i < len(paragraph) - width { for i < len(paragraph)-width {
// Find a word break at the end of the line to avoid splitting up words // Find a word break at the end of the line to avoid splitting up words
end := i + width end := i + width
for end > i && paragraph[end] != ' ' { // Look for a space, starting at the end for end > i && paragraph[end] != ' ' { // Look for a space, starting at the end
end -= 1 end -= 1
} }
lines = append(lines, paragraph[i:end]) lines = append(lines, paragraph[i:end])
i = end + 1 i = end + 1
} }
lines = append(lines, paragraph[i:]) lines = append(lines, paragraph[i:])
return lines return lines
} }
/** /**
* Return the text as a wrapped, indented block * Return the text as a wrapped, indented block
*/ */
func WrapText(text string, width int) string { func WrapText(text string, width int) string {
paragraphs := strings.Split(text, "\n") paragraphs := strings.Split(text, "\n")
var lines []string var lines []string
for _, paragraph := range paragraphs { for _, paragraph := range paragraphs {
lines = append(lines, WrapParagraph(paragraph, width)...) lines = append(lines, WrapParagraph(paragraph, width)...)
} }
return strings.Join(lines, "\n ") return strings.Join(lines, "\n ")
} }

112
terminal_utils/formatting_test.go
View File

@ -1,79 +1,77 @@
package terminal_utils_test package terminal_utils_test
import ( import (
"testing" "testing"
"reflect"
"offline_twitter/terminal_utils" "reflect"
"offline_twitter/terminal_utils"
) )
func TestWrapParagraph(t *testing.T) { func TestWrapParagraph(t *testing.T) {
test_cases := []struct{ test_cases := []struct {
Text string Text string
Expected []string Expected []string
} { }{
{ {
"These are public health officials who are making decisions about your lifestyle because they know more about health, " + "These are public health officials who are making decisions about your lifestyle because they know more about health, " +
"fitness and well-being than you do", "fitness and well-being than you do",
[]string{ []string{
"These are public health officials who are making decisions", "These are public health officials who are making decisions",
"about your lifestyle because they know more about health,", "about your lifestyle because they know more about health,",
"fitness and well-being than you do", "fitness and well-being than you do",
}, },
}, },
{ {
`Things I learned in law school:`, `Things I learned in law school:`,
[]string{`Things I learned in law school:`}, []string{`Things I learned in law school:`},
}, },
{ {
`Every student is smarter than you except the ones in your group project.`, `Every student is smarter than you except the ones in your group project.`,
[]string{ []string{
`Every student is smarter than you except the ones in your`, `Every student is smarter than you except the ones in your`,
`group project.`, `group project.`,
}, },
}, },
} }
for _, testcase := range test_cases { for _, testcase := range test_cases {
result := terminal_utils.WrapParagraph(testcase.Text, 60) result := terminal_utils.WrapParagraph(testcase.Text, 60)
if !reflect.DeepEqual(result, testcase.Expected) { if !reflect.DeepEqual(result, testcase.Expected) {
t.Errorf("Expected:\n%s\nGot:\n%s\n", testcase.Expected, result) t.Errorf("Expected:\n%s\nGot:\n%s\n", testcase.Expected, result)
} }
} }
} }
func TestWrapText(t *testing.T) { func TestWrapText(t *testing.T) {
test_cases := []struct{ test_cases := []struct {
Text string Text string
Expected string Expected string
} { }{
{ {
"These are public health officials who are making decisions about your lifestyle because they know more about health, " + "These are public health officials who are making decisions about your lifestyle because they know more about health, " +
"fitness and well-being than you do", "fitness and well-being than you do",
`These are public health officials who are making decisions `These are public health officials who are making decisions
about your lifestyle because they know more about health, about your lifestyle because they know more about health,
fitness and well-being than you do`, fitness and well-being than you do`,
}, },
{ {
`Things I learned in law school: `Things I learned in law school:
Falling behind early gives you more time to catch up. Falling behind early gives you more time to catch up.
Never use a long word when a diminutive one will suffice. Never use a long word when a diminutive one will suffice.
Every student is smarter than you except the ones in your group project. Every student is smarter than you except the ones in your group project.
If you try & fail, doesn’t matter. Try again & fail better`, If you try & fail, doesn’t matter. Try again & fail better`,
`Things I learned in law school: `Things I learned in law school:
Falling behind early gives you more time to catch up. Falling behind early gives you more time to catch up.
Never use a long word when a diminutive one will suffice. Never use a long word when a diminutive one will suffice.
Every student is smarter than you except the ones in your Every student is smarter than you except the ones in your
group project. group project.
If you try & fail, doesn’t matter. Try again & fail better`, If you try & fail, doesn’t matter. Try again & fail better`,
}, },
} }
for _, testcase := range test_cases { for _, testcase := range test_cases {
result := terminal_utils.WrapText(testcase.Text, 60) result := terminal_utils.WrapText(testcase.Text, 60)
if result != testcase.Expected { if result != testcase.Expected {
t.Errorf("Expected:\n%s\nGot:\n%s\n", testcase.Expected, result) t.Errorf("Expected:\n%s\nGot:\n%s\n", testcase.Expected, result)
} }
} }
} }