me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Some whitespace changes :V

Browse Source
This commit is contained in:
Alessio 2022-03-06 17:07:05 -08:00
parent 1d990e8a40
commit 7edc8ad5d3
20 changed files with 880 additions and 2001 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
cmd/twitter/helpers.go
View File

@ -3,14 +3,14 @@ package main
import ( import (
"fmt" "fmt"
"os" "os"
"regexp"
"strconv"
"strings"
"offline_twitter/scraper" "offline_twitter/scraper"
"offline_twitter/terminal_utils" "offline_twitter/terminal_utils"
"strings"
"strconv"
"regexp"
) )
/** /**
* Help message to print if command syntax is incorrect * Help message to print if command syntax is incorrect
*/ */
@ -73,13 +73,13 @@ This application downloads tweets from twitter and saves them in a SQLite databa
won't count toward the limit. won't count toward the limit.
` `
/** /**
* Helper function * Helper function
*/ */
func die(text string, display_help bool, exit_code int) { func die(text string, display_help bool, exit_code int) {
if text != "" { if text != "" {
fmt.Fprint(os.Stderr, terminal_utils.COLOR_RED + text + terminal_utils.COLOR_RESET + "\n") outstring := terminal_utils.COLOR_RED + text + terminal_utils.COLOR_RESET + "\n"
fmt.Fprint(os.Stderr, outstring)
} }
if display_help { if display_help {
fmt.Fprint(os.Stderr, help_message) fmt.Fprint(os.Stderr, help_message)

23
cmd/twitter/main.go
View File

@ -1,14 +1,14 @@
package main package main
import ( import (
"os"
"fmt"
"flag" "flag"
"fmt"
"os"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"offline_twitter/scraper"
"offline_twitter/persistence" "offline_twitter/persistence"
"offline_twitter/scraper"
) )
/** /**
@ -87,10 +87,10 @@ func main() {
profile, err = persistence.LoadProfile(*profile_dir) profile, err = persistence.LoadProfile(*profile_dir)
if err != nil { if err != nil {
die("Could not load profile: " + err.Error(), true, 2) die(fmt.Sprintf("Could not load profile: %s", err.Error()), true, 2)
} }
switch (operation) { switch operation {
case "create_profile": case "create_profile":
create_profile(target) create_profile(target)
case "fetch_user": case "fetch_user":
@ -116,7 +116,7 @@ func main() {
case "list_followed": case "list_followed":
list_followed() list_followed()
default: default:
die("Invalid operation: " + operation, true, 3) die(fmt.Sprintf("Invalid operation: %s", operation), true, 3)
} }
} }
@ -148,10 +148,10 @@ func fetch_user(handle scraper.UserHandle) {
err = profile.SaveUser(&user) err = profile.SaveUser(&user)
if err != nil { if err != nil {
die("Error saving user: " + err.Error(), false, 4) die(fmt.Sprintf("Error saving user: %s", err.Error()), false, 4)
} }
download_user_content(handle); download_user_content(handle)
happy_exit("Saved the user") happy_exit("Saved the user")
} }
@ -169,13 +169,13 @@ func fetch_tweet_only(tweet_identifier string) {
tweet, err := scraper.GetTweet(tweet_id) tweet, err := scraper.GetTweet(tweet_id)
if err != nil { if err != nil {
die("Error fetching tweet: " + err.Error(), false, -1) die(fmt.Sprintf("Error fetching tweet: %s", err.Error()), false, -1)
} }
log.Debug(tweet) log.Debug(tweet)
err = profile.SaveTweet(tweet) err = profile.SaveTweet(tweet)
if err != nil { if err != nil {
die("Error saving tweet: " + err.Error(), false, 4) die(fmt.Sprintf("Error saving tweet: %s", err.Error()), false, 4)
} }
happy_exit("Saved the tweet") happy_exit("Saved the tweet")
} }
@ -222,7 +222,6 @@ func fetch_user_feed(handle string, how_many int) {
happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users))) happy_exit(fmt.Sprintf("Saved %d tweets, %d retweets and %d users", len(trove.Tweets), len(trove.Retweets), len(trove.Users)))
} }
func download_tweet_content(tweet_identifier string) { func download_tweet_content(tweet_identifier string) {
tweet_id, err := extract_id_from(tweet_identifier) tweet_id, err := extract_id_from(tweet_identifier)
if err != nil { if err != nil {
@ -253,7 +252,7 @@ func download_user_content(handle scraper.UserHandle) {
func search(query string) { func search(query string) {
trove, err := scraper.Search(query, 1000) trove, err := scraper.Search(query, 1000)
if err != nil { if err != nil {
die("Error scraping search results: " + err.Error(), false, -100) die(fmt.Sprintf("Error scraping search results: %s", err.Error()), false, -100)
} }
profile.SaveTweetTrove(trove) profile.SaveTweetTrove(trove)

11
persistence/media_download.go
View File

@ -2,10 +2,10 @@ package persistence
import ( import (
"fmt" "fmt"
"io/ioutil"
"net/http"
"os" "os"
"path" "path"
"net/http"
"io/ioutil"
"strings" "strings"
"offline_twitter/scraper" "offline_twitter/scraper"
@ -15,7 +15,7 @@ type MediaDownloader interface {
Curl(url string, outpath string) error Curl(url string, outpath string) error
} }
type DefaultDownloader struct {} type DefaultDownloader struct{}
/** /**
* Download a file over HTTP and save it. * Download a file over HTTP and save it.
@ -46,7 +46,6 @@ func (d DefaultDownloader) Curl(url string, outpath string) error {
return nil return nil
} }
/** /**
* Downloads an Image, and if successful, marks it as downloaded in the DB * Downloads an Image, and if successful, marks it as downloaded in the DB
*/ */
@ -60,7 +59,6 @@ func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownlo
return p.SaveImage(*img) return p.SaveImage(*img)
} }
/** /**
* Downloads a Video and its thumbnail, and if successful, marks it as downloaded in the DB * Downloads a Video and its thumbnail, and if successful, marks it as downloaded in the DB
*/ */
@ -107,7 +105,6 @@ func (p Profile) DownloadTweetContentFor(t *scraper.Tweet) error {
return p.DownloadTweetContentWithInjector(t, DefaultDownloader{}) return p.DownloadTweetContentWithInjector(t, DefaultDownloader{})
} }
/** /**
* Enable injecting a custom MediaDownloader (i.e., for testing) * Enable injecting a custom MediaDownloader (i.e., for testing)
*/ */
@ -179,7 +176,7 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med
if err != nil && strings.Contains(err.Error(), "404 Not Found") { if err != nil && strings.Contains(err.Error(), "404 Not Found") {
// Try adding "600x200". Not sure why this does this but sometimes it does. // Try adding "600x200". Not sure why this does this but sometimes it does.
err = downloader.Curl(u.BannerImageUrl + "/600x200", outfile) err = downloader.Curl(u.BannerImageUrl+"/600x200", outfile)
} }
if err != nil { if err != nil {
return err return err

3
persistence/media_download_test.go
View File

@ -9,7 +9,8 @@ import (
"offline_twitter/scraper" "offline_twitter/scraper"
) )
type FakeDownloader struct {} type FakeDownloader struct{}
func (d FakeDownloader) Curl(url string, outpath string) error { return nil } func (d FakeDownloader) Curl(url string, outpath string) error { return nil }
func test_all_downloaded(tweet scraper.Tweet, yes_or_no bool, t *testing.T) { func test_all_downloaded(tweet scraper.Tweet, yes_or_no bool, t *testing.T) {

2
persistence/media_queries.go
View File

@ -90,7 +90,6 @@ func (p Profile) SavePoll(poll scraper.Poll) error {
return err return err
} }
/** /**
* Get the list of images for a tweet * Get the list of images for a tweet
*/ */
@ -117,7 +116,6 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e
return return
} }
/** /**
* Get the list of videos for a tweet * Get the list of videos for a tweet
*/ */

7
persistence/media_queries_test.go
View File

@ -2,6 +2,7 @@ package persistence_test
import ( import (
"testing" "testing"
"math/rand" "math/rand"
"time" "time"
@ -11,7 +12,6 @@ import (
"offline_twitter/scraper" "offline_twitter/scraper"
) )
/** /**
* Create an Image, save it, reload it, and make sure it comes back the same * Create an Image, save it, reload it, and make sure it comes back the same
*/ */
@ -78,7 +78,6 @@ func TestModifyImage(t *testing.T) {
} }
} }
/** /**
* Create an Video, save it, reload it, and make sure it comes back the same * Create an Video, save it, reload it, and make sure it comes back the same
*/ */
@ -93,7 +92,7 @@ func TestSaveAndLoadVideo(t *testing.T) {
rand.Seed(time.Now().UnixNano()) rand.Seed(time.Now().UnixNano())
vid := create_video_from_id(rand.Int()) vid := create_video_from_id(rand.Int())
vid.TweetID = tweet.ID vid.TweetID = tweet.ID
vid.IsGif = true; vid.IsGif = true
// Save the Video // Save the Video
err := profile.SaveVideo(vid) err := profile.SaveVideo(vid)
@ -147,7 +146,6 @@ func TestModifyVideo(t *testing.T) {
} }
} }
/** /**
* Create an Url, save it, reload it, and make sure it comes back the same * Create an Url, save it, reload it, and make sure it comes back the same
*/ */
@ -215,7 +213,6 @@ func TestModifyUrl(t *testing.T) {
} }
} }
/** /**
* Create a Poll, save it, reload it, and make sure it comes back the same * Create a Poll, save it, reload it, and make sure it comes back the same
*/ */

7
persistence/profile.go
View File

@ -13,7 +13,7 @@ import (
//go:embed schema.sql //go:embed schema.sql
var sql_init string var sql_init string
type Settings struct {} type Settings struct{}
type Profile struct { type Profile struct {
ProfileDir string ProfileDir string
@ -27,11 +27,11 @@ type Profile struct {
type ErrTargetAlreadyExists struct { type ErrTargetAlreadyExists struct {
target string target string
} }
func (err ErrTargetAlreadyExists) Error() string { func (err ErrTargetAlreadyExists) Error() string {
return fmt.Sprintf("Target already exists: %s", err.target) return fmt.Sprintf("Target already exists: %s", err.target)
} }
/** /**
* Create a new profile in the given location. * Create a new profile in the given location.
* Fails if target location already exists (i.e., is a file or directory). * Fails if target location already exists (i.e., is a file or directory).
@ -124,7 +124,6 @@ func NewProfile(target_dir string) (Profile, error) {
return Profile{target_dir, settings, db}, nil return Profile{target_dir, settings, db}, nil
} }
/** /**
* Loads the profile at the given location. Fails if the given directory is not a Profile. * Loads the profile at the given location. Fails if the given directory is not a Profile.
* *
@ -157,7 +156,7 @@ func LoadProfile(profile_dir string) (Profile, error) {
return Profile{}, err return Profile{}, err
} }
db, err := sql.Open("sqlite3", sqlite_file + "?_foreign_keys=on&_journal_mode=WAL") db, err := sql.Open("sqlite3", sqlite_file+"?_foreign_keys=on&_journal_mode=WAL")
if err != nil { if err != nil {
return Profile{}, err return Profile{}, err
} }

10
persistence/profile_test.go
View File

@ -2,8 +2,9 @@ package persistence_test
import ( import (
"testing" "testing"
"os"
"errors" "errors"
"os"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
@ -23,7 +24,6 @@ func file_exists(path string) bool {
} }
} }
/** /**
* Should refuse to create a Profile if the target already exists (i.e., is a file or directory). * Should refuse to create a Profile if the target already exists (i.e., is a file or directory).
*/ */
@ -44,7 +44,6 @@ func TestNewProfileInvalidPath(t *testing.T) {
assert.True(t, is_right_type, "Expected 'ErrTargetAlreadyExists' error, got %T instead", err) assert.True(t, is_right_type, "Expected 'ErrTargetAlreadyExists' error, got %T instead", err)
} }
/** /**
* Should correctly create a new Profile * Should correctly create a new Profile
*/ */
@ -61,7 +60,7 @@ func TestNewProfile(t *testing.T) {
profile, err := persistence.NewProfile(profile_path) profile, err := persistence.NewProfile(profile_path)
require.NoError(err) require.NoError(err)
assert.Equal(profile_path,profile.ProfileDir) assert.Equal(profile_path, profile.ProfileDir)
// Check files were created // Check files were created
contents, err := os.ReadDir(profile_path) contents, err := os.ReadDir(profile_path)
@ -71,7 +70,7 @@ func TestNewProfile(t *testing.T) {
expected_files := []struct { expected_files := []struct {
filename string filename string
isDir bool isDir bool
} { }{
{"images", true}, {"images", true},
{"link_preview_images", true}, {"link_preview_images", true},
{"profile_images", true}, {"profile_images", true},
@ -92,7 +91,6 @@ func TestNewProfile(t *testing.T) {
assert.Equal(persistence.ENGINE_DATABASE_VERSION, version) assert.Equal(persistence.ENGINE_DATABASE_VERSION, version)
} }
/** /**
* Should correctly load the Profile * Should correctly load the Profile
*/ */

1
persistence/retweet_queries.go
View File

@ -20,7 +20,6 @@ func (p Profile) SaveRetweet(r scraper.Retweet) error {
return err return err
} }
/** /**
* Retrieve a Retweet by ID * Retrieve a Retweet by ID
*/ */

1
persistence/retweet_queries_test.go
View File

@ -8,7 +8,6 @@ import (
"github.com/go-test/deep" "github.com/go-test/deep"
) )
func TestSaveAndLoadRetweet(t *testing.T) { func TestSaveAndLoadRetweet(t *testing.T) {
require := require.New(t) require := require.New(t)

5
persistence/tweet_queries.go
View File

@ -1,9 +1,9 @@
package persistence package persistence
import ( import (
"time"
"strings"
"database/sql" "database/sql"
"strings"
"time"
"offline_twitter/scraper" "offline_twitter/scraper"
) )
@ -171,7 +171,6 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) {
return t, err return t, err
} }
/** /**
* Populate the `User` field on a tweet with an actual User * Populate the `User` field on a tweet with an actual User
*/ */

1
persistence/tweet_queries_test.go
View File

@ -10,7 +10,6 @@ import (
"github.com/go-test/deep" "github.com/go-test/deep"
) )
/** /**
* Create a Tweet, save it, reload it, and make sure it comes back the same * Create a Tweet, save it, reload it, and make sure it comes back the same
*/ */

391
scraper/test_responses/api_v2/tweet_with_poll.json
View File

@ -1,390 +1 @@
{ {"result":{"__typename":"Tweet","rest_id":"1485692111106285571","core":{"user_results":{"result":{"__typename":"User","id":"VXNlcjo0NDA2NzI5OA==","rest_id":"44067298","affiliates_highlighted_label":{},"has_nft_avatar":false,"legacy":{"created_at":"Tue Jun 02 05:35:52 +0000 2009","default_profile":false,"default_profile_image":false,"description":"Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model","entities":{"description":{"urls":[]},"url":{"urls":[{"display_url":"amzn.to/3oInafv","expanded_url":"https://amzn.to/3oInafv","url":"https://t.co/7VDFOOtFK2","indices":[0,23]}]}},"fast_followers_count":0,"favourites_count":3840,"followers_count":334571,"friends_count":964,"has_custom_timelines":false,"is_translator":false,"listed_count":1434,"location":"Austin","media_count":9504,"name":"Michael Malice","normal_followers_count":334571,"pinned_tweet_ids_str":["1477347403023982596"],"profile_banner_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":60.59,"rgb":{"blue":0,"green":0,"red":0}},{"percentage":18.77,"rgb":{"blue":64,"green":60,"red":156}},{"percentage":3.62,"rgb":{"blue":31,"green":29,"red":77}},{"percentage":3.22,"rgb":{"blue":215,"green":199,"red":138}},{"percentage":2.83,"rgb":{"blue":85,"green":79,"red":215}}]}}}},"profile_banner_url":"https://pbs.twimg.com/profile_banners/44067298/1615134676","profile_image_extensions":{"mediaColor":{"r":{"ok":{"palette":[{"percentage":50.78,"rgb":{"blue":249,"green":247,"red":246}},{"percentage":17.4,"rgb":{"blue":51,"green":51,"red":205}},{"percentage":9.43,"rgb":{"blue":124,"green":139,"red":210}},{"percentage":6.38,"rgb":{"blue":47,"green":63,"red":116}},{"percentage":3.17,"rgb":{"blue":65,"green":45,"red":46}}]}}}},"profile_image_url_https":"https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg","profile_interstitial_type":"","protected":false,"screen_name":"michaelmalice","statuses_count":138682,"translator_type":"none","url":"https://t.co/7VDFOOtFK2","verified":true,"withheld_in_countries":[]},"super_follow_eligible":false,"super_followed_by":false,"super_following":false}}},"card":{"rest_id":"card://1485692110472892424","legacy":{"binding_values":[{"key":"choice1_label","value":{"string_value":"1","type":"STRING"}},{"key":"choice2_label","value":{"string_value":"2","type":"STRING"}},{"key":"end_datetime_utc","value":{"string_value":"2022-01-25T19:12:56Z","type":"STRING"}},{"key":"counts_are_final","value":{"boolean_value":false,"type":"BOOLEAN"}},{"key":"choice2_count","value":{"string_value":"702","type":"STRING"}},{"key":"choice1_count","value":{"string_value":"891","type":"STRING"}},{"key":"choice4_label","value":{"string_value":"E","type":"STRING"}},{"key":"last_updated_datetime_utc","value":{"string_value":"2022-01-24T20:20:38Z","type":"STRING"}},{"key":"duration_minutes","value":{"string_value":"1440","type":"STRING"}},{"key":"choice3_count","value":{"string_value":"459","type":"STRING"}},{"key":"choice4_count","value":{"string_value":"1801","type":"STRING"}},{"key":"choice3_label","value":{"string_value":"C","type":"STRING"}},{"key":"api","value":{"string_value":"capi://passthrough/1","type":"STRING"}},{"key":"card_url","value":{"scribe_key":"card_url","string_value":"https://twitter.com","type":"STRING"}}],"card_platform":{"platform":{"audience":{"name":"production"},"device":{"name":"Swift","version":"12"}}},"name":"poll4choice_text_only","url":"card://1485692110472892424","user_refs":[]}},"legacy":{"created_at":"Mon Jan 24 19:12:56 +0000 2022","conversation_control":{"policy":"Community","conversation_owner":{"legacy":{"screen_name":"michaelmalice"}}},"conversation_id_str":"1485692111106285571","display_text_range":[0,158],"entities":{"user_mentions":[],"urls":[],"hashtags":[],"symbols":[]},"favorite_count":71,"favorited":false,"full_text":"Which of these would most make you feel a disconnect from someone else?\n\n1) They don't like music\n2) They don't like pets\nC) They don't read\nE) They are vegan","is_quote_status":false,"lang":"en","possibly_sensitive":false,"possibly_sensitive_editable":true,"quote_count":12,"reply_count":11,"retweet_count":16,"retweeted":false,"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","user_id_str":"44067298","id_str":"1485692111106285571"}}}
"result":
{
"__typename": "Tweet",
"rest_id": "1485692111106285571",
"core":
{
"user_results":
{
"result":
{
"__typename": "User",
"id": "VXNlcjo0NDA2NzI5OA==",
"rest_id": "44067298",
"affiliates_highlighted_label":
{},
"has_nft_avatar": false,
"legacy":
{
"created_at": "Tue Jun 02 05:35:52 +0000 2009",
"default_profile": false,
"default_profile_image": false,
"description": "Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model",
"entities":
{
"description":
{
"urls":
[]
},
"url":
{
"urls":
[
{
"display_url": "amzn.to/3oInafv",
"expanded_url": "https://amzn.to/3oInafv",
"url": "https://t.co/7VDFOOtFK2",
"indices":
[
0,
23
]
}
]
}
},
"fast_followers_count": 0,
"favourites_count": 3840,
"followers_count": 334571,
"friends_count": 964,
"has_custom_timelines": false,
"is_translator": false,
"listed_count": 1434,
"location": "Austin",
"media_count": 9504,
"name": "Michael Malice",
"normal_followers_count": 334571,
"pinned_tweet_ids_str":
[
"1477347403023982596"
],
"profile_banner_extensions":
{
"mediaColor":
{
"r":
{
"ok":
{
"palette":
[
{
"percentage": 60.59,
"rgb":
{
"blue": 0,
"green": 0,
"red": 0
}
},
{
"percentage": 18.77,
"rgb":
{
"blue": 64,
"green": 60,
"red": 156
}
},
{
"percentage": 3.62,
"rgb":
{
"blue": 31,
"green": 29,
"red": 77
}
},
{
"percentage": 3.22,
"rgb":
{
"blue": 215,
"green": 199,
"red": 138
}
},
{
"percentage": 2.83,
"rgb":
{
"blue": 85,
"green": 79,
"red": 215
}
}
]
}
}
}
},
"profile_banner_url": "https://pbs.twimg.com/profile_banners/44067298/1615134676",
"profile_image_extensions":
{
"mediaColor":
{
"r":
{
"ok":
{
"palette":
[
{
"percentage": 50.78,
"rgb":
{
"blue": 249,
"green": 247,
"red": 246
}
},
{
"percentage": 17.4,
"rgb":
{
"blue": 51,
"green": 51,
"red": 205
}
},
{
"percentage": 9.43,
"rgb":
{
"blue": 124,
"green": 139,
"red": 210
}
},
{
"percentage": 6.38,
"rgb":
{
"blue": 47,
"green": 63,
"red": 116
}
},
{
"percentage": 3.17,
"rgb":
{
"blue": 65,
"green": 45,
"red": 46
}
}
]
}
}
}
},
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1415820415314931715/_VVX4GI8_normal.jpg",
"profile_interstitial_type": "",
"protected": false,
"screen_name": "michaelmalice",
"statuses_count": 138682,
"translator_type": "none",
"url": "https://t.co/7VDFOOtFK2",
"verified": true,
"withheld_in_countries":
[]
},
"super_follow_eligible": false,
"super_followed_by": false,
"super_following": false
}
}
},
"card":
{
"rest_id": "card://1485692110472892424",
"legacy":
{
"binding_values":
[
{
"key": "choice1_label",
"value":
{
"string_value": "1",
"type": "STRING"
}
},
{
"key": "choice2_label",
"value":
{
"string_value": "2",
"type": "STRING"
}
},
{
"key": "end_datetime_utc",
"value":
{
"string_value": "2022-01-25T19:12:56Z",
"type": "STRING"
}
},
{
"key": "counts_are_final",
"value":
{
"boolean_value": false,
"type": "BOOLEAN"
}
},
{
"key": "choice2_count",
"value":
{
"string_value": "702",
"type": "STRING"
}
},
{
"key": "choice1_count",
"value":
{
"string_value": "891",
"type": "STRING"
}
},
{
"key": "choice4_label",
"value":
{
"string_value": "E",
"type": "STRING"
}
},
{
"key": "last_updated_datetime_utc",
"value":
{
"string_value": "2022-01-24T20:20:38Z",
"type": "STRING"
}
},
{
"key": "duration_minutes",
"value":
{
"string_value": "1440",
"type": "STRING"
}
},
{
"key": "choice3_count",
"value":
{
"string_value": "459",
"type": "STRING"
}
},
{
"key": "choice4_count",
"value":
{
"string_value": "1801",
"type": "STRING"
}
},
{
"key": "choice3_label",
"value":
{
"string_value": "C",
"type": "STRING"
}
},
{
"key": "api",
"value":
{
"string_value": "capi://passthrough/1",
"type": "STRING"
}
},
{
"key": "card_url",
"value":
{
"scribe_key": "card_url",
"string_value": "https://twitter.com",
"type": "STRING"
}
}
],
"card_platform":
{
"platform":
{
"audience":
{
"name": "production"
},
"device":
{
"name": "Swift",
"version": "12"
}
}
},
"name": "poll4choice_text_only",
"url": "card://1485692110472892424",
"user_refs":
[]
}
},
"legacy":
{
"created_at": "Mon Jan 24 19:12:56 +0000 2022",
"conversation_control":
{
"policy": "Community",
"conversation_owner":
{
"legacy":
{
"screen_name": "michaelmalice"
}
}
},
"conversation_id_str": "1485692111106285571",
"display_text_range":
[
0,
158
],
"entities":
{
"user_mentions":
[],
"urls":
[],
"hashtags":
[],
"symbols":
[]
},
"favorite_count": 71,
"favorited": false,
"full_text": "Which of these would most make you feel a disconnect from someone else?\n\n1) They don't like music\n2) They don't like pets\nC) They don't read\nE) They are vegan",
"is_quote_status": false,
"lang": "en",
"possibly_sensitive": false,
"possibly_sensitive_editable": true,
"quote_count": 12,
"reply_count": 11,
"retweet_count": 16,
"retweeted": false,
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"user_id_str": "44067298",
"id_str": "1485692111106285571"
}
}
}

21
scraper/test_responses/single_tweets/tweet_with_html_entities.json
View File

@ -1,20 +1 @@
{ {"created_at":"Thu Dec 23 20:55:48 +0000 2021","id_str":"1474121585510563845","full_text":"By the 1970s the elite consensus was that \"the hunt for atomic spies\" had been a grotesque over-reaction to minor leaks that cost the lives of the Rosenbergs &amp; ruined many innocents. Only when the USSR fell was it discovered that they &amp; other spies had given away ALL the secrets","display_text_range":[0,288],"entities":{},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","user_id_str":"1239676915386068993","retweet_count":239,"favorite_count":1118,"reply_count":26,"quote_count":26,"conversation_id_str":"1474121585510563845","lang":"en"}
"created_at": "Thu Dec 23 20:55:48 +0000 2021",
"id_str": "1474121585510563845",
"full_text": "By the 1970s the elite consensus was that \"the hunt for atomic spies\" had been a grotesque over-reaction to minor leaks that cost the lives of the Rosenbergs &amp; ruined many innocents. Only when the USSR fell was it discovered that they &amp; other spies had given away ALL the secrets",
"display_text_range":
[
0,
288
],
"entities":
{},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"user_id_str": "1239676915386068993",
"retweet_count": 239,
"favorite_count": 1118,
"reply_count": 26,
"quote_count": 26,
"conversation_id_str": "1474121585510563845",
"lang": "en"
}

368
scraper/test_responses/single_tweets/tweet_with_quoted_tweet_and_url.json
View File

File diff suppressed because one or more lines are too long

332
scraper/test_responses/tombstones/tombstone_deleted.json
View File

@ -1,331 +1 @@
{ {"globalObjects":{"tweets":{"1454524255127887878":{"created_at":"Sat Oct 30 19:03:00 +0000 2021","id_str":"1454524255127887878","full_text":"@TastefulTyrant Halloween is often the easiest night of the year but women do thirst trap, too.","display_text_range":[16,95],"entities":{"user_mentions":[{"screen_name":"TastefulTyrant","name":"ᴛᴀꜱᴛᴇꜰᴜʟ ᴛʏʀᴀɴᴛ","id_str":"1218687933391298560","indices":[0,15]}]},"source":"<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>","in_reply_to_status_id_str":"1454521654781136902","in_reply_to_user_id_str":"1218687933391298560","in_reply_to_screen_name":"TastefulTyrant","user_id_str":"887434912529338375","retweet_count":0,"favorite_count":12,"reply_count":0,"quote_count":0,"conversation_id_str":"1454521654781136902","lang":"en"}},"users":{"887434912529338375":{"id_str":"887434912529338375","name":"Covfefe Anon","screen_name":"CovfefeAnon","location":"","description":"Not to be confused with 2001 Nobel Peace Prize winner Kofi Annan.\n\n54th Clause of the Magna Carta absolutist.\n\nCommentary from an NRx perspective.","entities":{"description":{}},"followers_count":8386,"fast_followers_count":0,"normal_followers_count":8386,"friends_count":497,"listed_count":59,"created_at":"Tue Jul 18 22:12:25 +0000 2017","favourites_count":175661,"statuses_count":26334,"media_count":1755,"profile_image_url_https":"https://pbs.twimg.com/profile_images/1392509603116617731/TDrNeUiZ_normal.jpg","profile_banner_url":"https://pbs.twimg.com/profile_banners/887434912529338375/1598514714","profile_image_extensions_alt_text":null,"profile_image_extensions_media_color":{"palette":[{"rgb":{"red":127,"green":125,"blue":102},"percentage":34.13},{"rgb":{"red":68,"green":50,"blue":44},"percentage":26.45},{"rgb":{"red":167,"green":170,"blue":176},"percentage":12.16},{"rgb":{"red":102,"green":47,"blue":31},"percentage":6.4},{"rgb":{"red":43,"green":52,"blue":65},"percentage":3.54}]},"profile_image_extensions_media_availability":null,"profile_image_extensions":{"mediaStats":{"r":{"missing":null},"ttl":-1}},"profile_banner_extensions_alt_text":null,"profile_banner_extensions_media_availability":null,"profile_banner_extensions_media_color":{"palette":[{"rgb":{"red":254,"green":254,"blue":254},"percentage":44.66},{"rgb":{"red":122,"green":116,"blue":123},"percentage":24.0},{"rgb":{"red":131,"green":164,"blue":104},"percentage":18.44},{"rgb":{"red":50,"green":50,"blue":50},"percentage":6.56},{"rgb":{"red":114,"green":156,"blue":99},"percentage":2.85}]},"profile_banner_extensions":{"mediaStats":{"r":{"missing":null},"ttl":-1}},"profile_link_color":"1B95E0","pinned_tweet_ids":[1005906691324596224],"pinned_tweet_ids_str":["1005906691324596224"],"advertiser_account_type":"promotable_user","advertiser_account_service_levels":["analytics"],"profile_interstitial_type":"","business_profile_state":"none","translator_type":"none","withheld_in_countries":[],"ext":{"highlightedLabel":{"r":{"ok":{}},"ttl":-1}}}},"moments":{},"cards":{},"places":{},"media":{},"broadcasts":{},"topics":{},"lists":{}},"timeline":{"id":"Conversation-1454521654781136902","instructions":[{"addEntries":{"entries":[{"entryId":"tombstone-7768850382073638905","sortIndex":"7768850382073638905","content":{"item":{"content":{"tombstone":{"displayType":"Inline","tombstoneInfo":{"text":"","richText":{"text":"This Tweet was deleted by the Tweet author. Learn more","entities":[{"fromIndex":44,"toIndex":54,"ref":{"url":{"urlType":"ExternalUrl","url":"https://help.twitter.com/rules-and-policies/notices-on-twitter"}}}],"rtl":false}}}}}}},{"entryId":"tweet-1454524255127887878","sortIndex":"7768847781726887929","content":{"item":{"content":{"tweet":{"id":"1454524255127887878","displayType":"Tweet"}}}}}]}},{"terminateTimeline":{"direction":"Top"}}],"responseObjects":{"feedbackActions":{}}}}
"globalObjects":
{
"tweets":
{
"1454524255127887878":
{
"created_at": "Sat Oct 30 19:03:00 +0000 2021",
"id_str": "1454524255127887878",
"full_text": "@TastefulTyrant Halloween is often the easiest night of the year but women do thirst trap, too.",
"display_text_range":
[
16,
95
],
"entities":
{
"user_mentions":
[
{
"screen_name": "TastefulTyrant",
"name": "ᴛᴀꜱᴛᴇꜰᴜʟ ᴛʏʀᴀɴᴛ",
"id_str": "1218687933391298560",
"indices":
[
0,
15
]
}
]
},
"source": "<a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>",
"in_reply_to_status_id_str": "1454521654781136902",
"in_reply_to_user_id_str": "1218687933391298560",
"in_reply_to_screen_name": "TastefulTyrant",
"user_id_str": "887434912529338375",
"retweet_count": 0,
"favorite_count": 12,
"reply_count": 0,
"quote_count": 0,
"conversation_id_str": "1454521654781136902",
"lang": "en"
}
},
"users":
{
"887434912529338375":
{
"id_str": "887434912529338375",
"name": "Covfefe Anon",
"screen_name": "CovfefeAnon",
"location": "",
"description": "Not to be confused with 2001 Nobel Peace Prize winner Kofi Annan.\n\n54th Clause of the Magna Carta absolutist.\n\nCommentary from an NRx perspective.",
"entities":
{
"description":
{}
},
"followers_count": 8386,
"fast_followers_count": 0,
"normal_followers_count": 8386,
"friends_count": 497,
"listed_count": 59,
"created_at": "Tue Jul 18 22:12:25 +0000 2017",
"favourites_count": 175661,
"statuses_count": 26334,
"media_count": 1755,
"profile_image_url_https": "https://pbs.twimg.com/profile_images/1392509603116617731/TDrNeUiZ_normal.jpg",
"profile_banner_url": "https://pbs.twimg.com/profile_banners/887434912529338375/1598514714",
"profile_image_extensions_alt_text": null,
"profile_image_extensions_media_color":
{
"palette":
[
{
"rgb":
{
"red": 127,
"green": 125,
"blue": 102
},
"percentage": 34.13
},
{
"rgb":
{
"red": 68,
"green": 50,
"blue": 44
},
"percentage": 26.45
},
{
"rgb":
{
"red": 167,
"green": 170,
"blue": 176
},
"percentage": 12.16
},
{
"rgb":
{
"red": 102,
"green": 47,
"blue": 31
},
"percentage": 6.4
},
{
"rgb":
{
"red": 43,
"green": 52,
"blue": 65
},
"percentage": 3.54
}
]
},
"profile_image_extensions_media_availability": null,
"profile_image_extensions":
{
"mediaStats":
{
"r":
{
"missing": null
},
"ttl": -1
}
},
"profile_banner_extensions_alt_text": null,
"profile_banner_extensions_media_availability": null,
"profile_banner_extensions_media_color":
{
"palette":
[
{
"rgb":
{
"red": 254,
"green": 254,
"blue": 254
},
"percentage": 44.66
},
{
"rgb":
{
"red": 122,
"green": 116,
"blue": 123
},
"percentage": 24.0
},
{
"rgb":
{
"red": 131,
"green": 164,
"blue": 104
},
"percentage": 18.44
},
{
"rgb":
{
"red": 50,
"green": 50,
"blue": 50
},
"percentage": 6.56
},
{
"rgb":
{
"red": 114,
"green": 156,
"blue": 99
},
"percentage": 2.85
}
]
},
"profile_banner_extensions":
{
"mediaStats":
{
"r":
{
"missing": null
},
"ttl": -1
}
},
"profile_link_color": "1B95E0",
"pinned_tweet_ids":
[
1005906691324596224
],
"pinned_tweet_ids_str":
[
"1005906691324596224"
],
"advertiser_account_type": "promotable_user",
"advertiser_account_service_levels":
[
"analytics"
],
"profile_interstitial_type": "",
"business_profile_state": "none",
"translator_type": "none",
"withheld_in_countries":
[],
"ext":
{
"highlightedLabel":
{
"r":
{
"ok":
{}
},
"ttl": -1
}
}
}
},
"moments":
{},
"cards":
{},
"places":
{},
"media":
{},
"broadcasts":
{},
"topics":
{},
"lists":
{}
},
"timeline":
{
"id": "Conversation-1454521654781136902",
"instructions":
[
{
"addEntries":
{
"entries":
[
{
"entryId": "tombstone-7768850382073638905",
"sortIndex": "7768850382073638905",
"content":
{
"item":
{
"content":
{
"tombstone":
{
"displayType": "Inline",
"tombstoneInfo":
{
"text": "",
"richText":
{
"text": "This Tweet was deleted by the Tweet author. Learn more",
"entities":
[
{
"fromIndex": 44,
"toIndex": 54,
"ref":
{
"url":
{
"urlType": "ExternalUrl",
"url": "https://help.twitter.com/rules-and-policies/notices-on-twitter"
}
}
}
],
"rtl": false
}
}
}
}
}
}
},
{
"entryId": "tweet-1454524255127887878",
"sortIndex": "7768847781726887929",
"content":
{
"item":
{
"content":
{
"tweet":
{
"id": "1454524255127887878",
"displayType": "Tweet"
}
}
}
}
}
]
}
},
{
"terminateTimeline":
{
"direction": "Top"
}
}
],
"responseObjects":
{
"feedbackActions":
{}
}
}
}

22
terminal_utils/colors.go
View File

@ -1,15 +1,17 @@
package terminal_utils package terminal_utils
/** /**
* Colors for terminal output * Colors for terminal output
*/ */
const COLOR_RESET = "\033[0m"
const COLOR_RED = "\033[31m" const (
const COLOR_GREEN = "\033[32m" COLOR_RESET = "\033[0m"
const COLOR_YELLOW = "\033[33m" COLOR_RED = "\033[31m"
const COLOR_BLUE = "\033[34m" COLOR_GREEN = "\033[32m"
const COLOR_PURPLE = "\033[35m" COLOR_YELLOW = "\033[33m"
const COLOR_CYAN = "\033[36m" COLOR_BLUE = "\033[34m"
const COLOR_GRAY = "\033[37m" COLOR_PURPLE = "\033[35m"
const COLOR_WHITE = "\033[97m" COLOR_CYAN = "\033[36m"
COLOR_GRAY = "\033[37m"
COLOR_WHITE = "\033[97m"
)

6
terminal_utils/formatting.go
View File

@ -1,8 +1,8 @@
package terminal_utils package terminal_utils
import ( import (
"time"
"strings" "strings"
"time"
) )
/** /**
@ -12,14 +12,13 @@ func FormatDate(t time.Time) string {
return t.Format("Jan 2, 2006 15:04:05") return t.Format("Jan 2, 2006 15:04:05")
} }
/** /**
* Wrap lines to fixed width, while respecting word breaks * Wrap lines to fixed width, while respecting word breaks
*/ */
func WrapParagraph(paragraph string, width int) []string { func WrapParagraph(paragraph string, width int) []string {
var lines []string var lines []string
i := 0 i := 0
for i < len(paragraph) - width { for i < len(paragraph)-width {
// Find a word break at the end of the line to avoid splitting up words // Find a word break at the end of the line to avoid splitting up words
end := i + width end := i + width
for end > i && paragraph[end] != ' ' { // Look for a space, starting at the end for end > i && paragraph[end] != ' ' { // Look for a space, starting at the end
@ -32,7 +31,6 @@ func WrapParagraph(paragraph string, width int) []string {
return lines return lines
} }
/** /**
* Return the text as a wrapped, indented block * Return the text as a wrapped, indented block
*/ */

18
terminal_utils/formatting_test.go
View File

@ -2,17 +2,17 @@ package terminal_utils_test
import ( import (
"testing" "testing"
"reflect" "reflect"
"offline_twitter/terminal_utils" "offline_twitter/terminal_utils"
) )
func TestWrapParagraph(t *testing.T) { func TestWrapParagraph(t *testing.T) {
test_cases := []struct{ test_cases := []struct {
Text string Text string
Expected []string Expected []string
} { }{
{ {
"These are public health officials who are making decisions about your lifestyle because they know more about health, " + "These are public health officials who are making decisions about your lifestyle because they know more about health, " +
"fitness and well-being than you do", "fitness and well-being than you do",
@ -42,27 +42,25 @@ func TestWrapParagraph(t *testing.T) {
} }
} }
func TestWrapText(t *testing.T) { func TestWrapText(t *testing.T) {
test_cases := []struct{ test_cases := []struct {
Text string Text string
Expected string Expected string
} { }{
{ {
"These are public health officials who are making decisions about your lifestyle because they know more about health, " + "These are public health officials who are making decisions about your lifestyle because they know more about health, " +
"fitness and well-being than you do", "fitness and well-being than you do",
`These are public health officials who are making decisions `These are public health officials who are making decisions
about your lifestyle because they know more about health, about your lifestyle because they know more about health,
fitness and well-being than you do`, fitness and well-being than you do`,
}, },
{ {
`Things I learned in law school: `Things I learned in law school:
Falling behind early gives you more time to catch up. Falling behind early gives you more time to catch up.
Never use a long word when a diminutive one will suffice. Never use a long word when a diminutive one will suffice.
Every student is smarter than you except the ones in your group project. Every student is smarter than you except the ones in your group project.
If you try & fail, doesn’t matter. Try again & fail better`, If you try & fail, doesn’t matter. Try again & fail better`,
`Things I learned in law school: `Things I learned in law school:
Falling behind early gives you more time to catch up. Falling behind early gives you more time to catch up.
Never use a long word when a diminutive one will suffice. Never use a long word when a diminutive one will suffice.
Every student is smarter than you except the ones in your Every student is smarter than you except the ones in your