Add 'gofmt' linter

.golangci.yaml

@@ -27,6 +27,7 @@ linters:
     - wrapcheck
     - lll
     - godox
+    - gofmt
     - errorlint
     - nolintlint
 
@@ -203,9 +204,9 @@ linters-settings:
     keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting
       - XXX
 
-#   gofmt:
+  gofmt:
-#     # simplify code: gofmt with `-s` option, true by default
+    # simplify code: gofmt with `-s` option, true by default
-#     simplify: true
+    simplify: true
 
 #   gofumpt:
 #     # Select the Go version to target. The default is `1.15`.

persistence/user_queries_test.go

@@ -6,9 +6,9 @@ import (
 	"fmt"
 	"math/rand"
 
+	"github.com/go-test/deep"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"github.com/go-test/deep"
 
 	"offline_twitter/scraper"
 )

persistence/utils.go

@@ -3,9 +3,9 @@ package persistence
 import (
 	"errors"
 	"fmt"
+	"os"
 	"regexp"
 	"strings"
-	"os"
 )
 
 var NotInDatabase = errors.New("Not in database")

persistence/versions.go

@@ -7,29 +7,27 @@ import (
 	"offline_twitter/terminal_utils"
 )
 
-
 const ENGINE_DATABASE_VERSION = 11
 
-
 type VersionMismatchError struct {
 	EngineVersion   int
 	DatabaseVersion int
 }
+
 func (e VersionMismatchError) Error() string {
 	return fmt.Sprintf(
-`This profile was created with database schema version %d, which is newer than this application's database schema version, %d.
+		`This profile was created with database schema version %d, which is newer than this application's database schema version, %d.
 Please upgrade this application to a newer version to use this profile.  Or downgrade the profile's schema version, somehow.`,
 		e.DatabaseVersion, e.EngineVersion,
 	)
 }
 
-
 /**
  * The Nth entry is the migration that moves you from version N to version N+1.
  * `len(MIGRATIONS)` should always equal `ENGINE_DATABASE_VERSION`.
  */
 var MIGRATIONS = []string{
-`create table polls (rowid integer primary key,
+	`create table polls (rowid integer primary key,
     id integer unique not null check(typeof(id) = 'integer'),
     tweet_id integer not null,
     num_choices integer not null,
@@ -50,25 +48,25 @@ var MIGRATIONS = []string{
 
     foreign key(tweet_id) references tweets(id)
 	);`,
-`alter table tweets add column is_conversation_scraped boolean default 0;
+	`alter table tweets add column is_conversation_scraped boolean default 0;
 	alter table tweets add column last_scraped_at integer not null default 0`,
-`update tombstone_types set tombstone_text = 'This Tweet is from a suspended account' where rowid = 2;
+	`update tombstone_types set tombstone_text = 'This Tweet is from a suspended account' where rowid = 2;
 	insert into tombstone_types (rowid, short_name, tombstone_text)
 	                     values (5, 'violated', 'This Tweet violated the Twitter Rules'),
 	                            (6, 'no longer exists', 'This Tweet is from an account that no longer exists')`,
-`alter table videos add column thumbnail_remote_url text not null default "missing";
+	`alter table videos add column thumbnail_remote_url text not null default "missing";
 	alter table videos add column thumbnail_local_filename text not null default "missing"`,
-`alter table videos add column duration integer not null default 0;
+	`alter table videos add column duration integer not null default 0;
 	alter table videos add column view_count integer not null default 0`,
-`alter table users add column is_banned boolean default 0`,
+	`alter table users add column is_banned boolean default 0`,
-`alter table urls add column short_text text not null default ""`,
+	`alter table urls add column short_text text not null default ""`,
-`insert into tombstone_types (rowid, short_name, tombstone_text) values (7, 'age-restricted', 'Age-restricted adult content. '
+	`insert into tombstone_types (rowid, short_name, tombstone_text) values (7, 'age-restricted', 'Age-restricted adult content. '
 	|| 'This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter')`,
-`alter table users add column is_followed boolean default 0`,
+	`alter table users add column is_followed boolean default 0`,
-`create table fake_user_sequence(latest_fake_id integer not null);
+	`create table fake_user_sequence(latest_fake_id integer not null);
 	insert into fake_user_sequence values(0x4000000000000000);
 	alter table users add column is_id_fake boolean default 0;`,
-`delete from urls where rowid in (select urls.rowid from tweets join urls on tweets.id = urls.tweet_id where urls.text like
+	`delete from urls where rowid in (select urls.rowid from tweets join urls on tweets.id = urls.tweet_id where urls.text like
 	'https://twitter.com/%/status/' || tweets.quoted_tweet_id || "%")`,
 }
 

persistence/versions_test.go

@@ -2,12 +2,13 @@ package persistence_test
 
 import (
 	"testing"
+
 	"os"
 
 	"github.com/stretchr/testify/require"
 
-	"offline_twitter/scraper"
 	"offline_twitter/persistence"
+	"offline_twitter/scraper"
 )
 
 func TestVersionUpgrade(t *testing.T) {
@@ -25,7 +26,7 @@ func TestVersionUpgrade(t *testing.T) {
 	require.False(profile.IsTweetInDatabase(test_tweet_id), "Test tweet shouldn't be in db yet")
 
 	persistence.MIGRATIONS = append(persistence.MIGRATIONS, test_migration)
-	err := profile.UpgradeFromXToY(persistence.ENGINE_DATABASE_VERSION, persistence.ENGINE_DATABASE_VERSION + 1)
+	err := profile.UpgradeFromXToY(persistence.ENGINE_DATABASE_VERSION, persistence.ENGINE_DATABASE_VERSION+1)
 	require.NoError(err)
 
 	require.True(profile.IsTweetInDatabase(test_tweet_id), "Migration should have created the tweet, but it didn't")

scraper/api_types.go

@@ -1,16 +1,15 @@
 package scraper
 
 import (
+	"encoding/json"
 	"fmt"
 	"html"
-	"time"
-	"strings"
-	"encoding/json"
-	"strconv"
 	"sort"
+	"strconv"
+	"strings"
+	"time"
 )
 
-
 type APIMedia struct {
 	ID            int64  `json:"id_str,string"`
 	MediaURLHttps string `json:"media_url_https"`
@@ -26,6 +25,7 @@ type SortableVariants []struct {
 	Bitrate int    `json:"bitrate,omitempty"`
 	URL     string `json:"url"`
 }
+
 func (v SortableVariants) Len() int           { return len(v) }
 func (v SortableVariants) Swap(i, j int)      { v[i], v[j] = v[j], v[i] }
 func (v SortableVariants) Less(i, j int) bool { return v[i].Bitrate > v[j].Bitrate }
@@ -183,7 +183,7 @@ func (t *APITweet) NormalizeContent() {
 		t.RetweetedStatusID = int64(id)
 	}
 
-	if (len(t.DisplayTextRange) == 2) {
+	if len(t.DisplayTextRange) == 2 {
 		t.Entities.ReplyMentions = strings.TrimSpace(string([]rune(t.FullText)[0:t.DisplayTextRange[0]]))
 		t.FullText = string([]rune(t.FullText)[t.DisplayTextRange[0]:t.DisplayTextRange[1]])
 	}
@@ -217,7 +217,6 @@ func (t APITweet) String() string {
 	return string(data)
 }
 
-
 type APIUser struct {
 	CreatedAt   string `json:"created_at"`
 	Description string `json:"description"`
@@ -246,7 +245,6 @@ type APIUser struct {
 	DoesntExist          bool
 }
 
-
 type UserResponse struct {
 	Data struct {
 		User struct {
@@ -260,6 +258,7 @@ type UserResponse struct {
 		Code    int    `json:"code"`
 	} `json:"errors"`
 }
+
 func (u UserResponse) ConvertToAPIUser() APIUser {
 	ret := u.Data.User.Legacy
 	ret.ID = u.Data.User.ID
@@ -303,10 +302,13 @@ type Entry struct {
 		} `json:"operation"`
 	} `json:"content"`
 }
+
 func (e Entry) GetTombstoneText() string {
 	return e.Content.Item.Content.Tombstone.TombstoneInfo.RichText.Text
 }
+
 type SortableEntries []Entry
+
 func (e SortableEntries) Len() int           { return len(e) }
 func (e SortableEntries) Swap(i, j int)      { e[i], e[j] = e[j], e[i] }
 func (e SortableEntries) Less(i, j int) bool { return e[i].SortIndex > e[j].SortIndex }
@@ -338,6 +340,7 @@ var tombstone_types = map[string]string{
 	"Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, " +
 		"you’ll need to log in to Twitter. Learn more": "age-restricted",
 }
+
 /**
  * Insert tweets into GlobalObjects for each tombstone.  Returns a list of users that need to
  * be fetched for tombstones.
@@ -380,7 +383,7 @@ func (t *TweetResponse) HandleTombstones() []UserHandle {
 			// Try to reconstruct the tombstone tweet
 			var tombstoned_tweet APITweet
 			tombstoned_tweet.ID = int64(i) // Set a default to prevent clobbering other tombstones
-			if i + 1 < len(entries) && entries[i+1].Content.Item.Content.Tweet.ID != 0 {
+			if i+1 < len(entries) && entries[i+1].Content.Item.Content.Tweet.ID != 0 {
 				next_tweet_id := entries[i+1].Content.Item.Content.Tweet.ID
 				api_tweet, ok := t.GlobalObjects.Tweets[fmt.Sprint(next_tweet_id)]
 				if !ok {
@@ -390,7 +393,7 @@ func (t *TweetResponse) HandleTombstones() []UserHandle {
 				tombstoned_tweet.UserID = api_tweet.InReplyToUserID
 				ret = append(ret, UserHandle(api_tweet.InReplyToScreenName))
 			}
-			if i - 1 >= 0 && entries[i-1].Content.Item.Content.Tweet.ID != 0 {
+			if i-1 >= 0 && entries[i-1].Content.Item.Content.Tweet.ID != 0 {
 				prev_tweet_id := entries[i-1].Content.Item.Content.Tweet.ID
 				_, ok := t.GlobalObjects.Tweets[fmt.Sprint(prev_tweet_id)]
 				if !ok {
@@ -416,7 +419,7 @@ func (t *TweetResponse) HandleTombstones() []UserHandle {
 func (t *TweetResponse) GetCursor() string {
 	entries := t.Timeline.Instructions[0].AddEntries.Entries
 	if len(entries) > 0 {
-		last_entry := entries[len(entries) - 1]
+		last_entry := entries[len(entries)-1]
 		if strings.Contains(last_entry.EntryID, "cursor") {
 			return last_entry.Content.Operation.Cursor.Value
 		}
@@ -424,7 +427,7 @@ func (t *TweetResponse) GetCursor() string {
 
 	// Next, try the other format ("replaceEntry")
 	instructions := t.Timeline.Instructions
-	last_replace_entry := instructions[len(instructions) - 1].ReplaceEntry.Entry
+	last_replace_entry := instructions[len(instructions)-1].ReplaceEntry.Entry
 	if strings.Contains(last_replace_entry.EntryID, "cursor") {
 		return last_replace_entry.Content.Operation.Cursor.Value
 	}
@@ -450,7 +453,6 @@ func (t *TweetResponse) IsEndOfFeed() bool {
 	return true
 }
 
-
 func idstr_to_int(idstr string) int64 {
 	id, err := strconv.Atoi(idstr)
 	if err != nil {

scraper/api_types_test.go

@@ -1,9 +1,9 @@
 package scraper_test
 
 import (
-	"testing"
-	"os"
 	"encoding/json"
+	"os"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -11,7 +11,6 @@ import (
 	. "offline_twitter/scraper"
 )
 
-
 func TestNormalizeContent(t *testing.T) {
 	assert := assert.New(t)
 	test_cases := []struct {
@@ -21,7 +20,7 @@ func TestNormalizeContent(t *testing.T) {
 		in_reply_to_id      TweetID
 		retweeted_status_id TweetID
 		reply_mentions      string
-	} {
+	}{
 		{"test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json", "", 0, 1395882872729477131, 0, "@michaelmalice"},
 		{"test_responses/single_tweets/tweet_with_image.json", "this saddens me every time", 0, 0, 0, ""},
 		{"test_responses/single_tweets/tweet_that_is_a_reply.json", "Noted", 0, 1396194494710788100, 0, "@RvaTeddy @michaelmalice"},
@@ -48,7 +47,7 @@ func TestNormalizeContent(t *testing.T) {
 		}
 		var tweet APITweet
 		err = json.Unmarshal(data, &tweet)
-		assert.NoError(err, "Failed at " + v.filename)
+		assert.NoError(err, "Failed at "+v.filename)
 
 		tweet.NormalizeContent()
 
@@ -60,7 +59,6 @@ func TestNormalizeContent(t *testing.T) {
 	}
 }
 
-
 func TestUserProfileToAPIUser(t *testing.T) {
 	assert := assert.New(t)
 	data, err := os.ReadFile("test_responses/michael_malice_user_profile.json")
@@ -76,7 +74,6 @@ func TestUserProfileToAPIUser(t *testing.T) {
 	assert.Equal(user_resp.Data.User.Legacy.FollowersCount, result.FollowersCount)
 }
 
-
 func TestGetCursor(t *testing.T) {
 	assert := assert.New(t)
 	data, err := os.ReadFile("test_responses/midriffs_anarchist_cookbook.json")
@@ -91,13 +88,12 @@ func TestGetCursor(t *testing.T) {
 		tweet_resp.GetCursor())
 }
 
-
 func TestIsEndOfFeed(t *testing.T) {
 	assert := assert.New(t)
 	test_cases := []struct {
 		filename       string
 		is_end_of_feed bool
-	} {
+	}{
 		{"test_responses/michael_malice_feed.json", false},
 		{"test_responses/kwiber_end_of_feed.json", true},
 	}
@@ -113,7 +109,6 @@ func TestIsEndOfFeed(t *testing.T) {
 	}
 }
 
-
 func TestHandleTombstonesHidden(t *testing.T) {
 	assert := assert.New(t)
 	data, err := os.ReadFile("test_responses/tombstones/tombstone_hidden_1.json")

scraper/api_types_v2.go

@@ -37,6 +37,7 @@ type APIV2Card struct {
 		Url  string `json:"url"`
 	} `json:"legacy"`
 }
+
 func (card APIV2Card) ParseAsUrl() Url {
 	values := make(map[string]CardValue)
 	for _, obj := range card.Legacy.BindingValues {
@@ -121,6 +122,7 @@ type APIV2UserResult struct {
 		} `json:"result"`
 	} `json:"user_results"`
 }
+
 func (u APIV2UserResult) ToUser() User {
 	user, err := ParseSingleUser(u.UserResults.Result.Legacy)
 	if err != nil {
@@ -149,11 +151,12 @@ type APIV2Result struct {
 		Tweet _Result `json:"tweet"`
 	} `json:"result"`
 }
+
 func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove {
 	ret := NewTweetTrove()
 
 	// Start by checking if this is a null entry in a feed
-	if api_result.Result.Tombstone != nil && ignore_null_entries{
+	if api_result.Result.Tombstone != nil && ignore_null_entries {
 		// TODO: this is becoming really spaghetti.  Why do we need a separate execution path for this?
 		return ret
 	}
@@ -245,6 +248,7 @@ type APIV2Tweet struct {
 	RetweetedStatusResult *APIV2Result `json:"retweeted_status_result"`
 	APITweet
 }
+
 func (api_v2_tweet APIV2Tweet) ToTweetTrove() TweetTrove {
 	ret := NewTweetTrove()
 
@@ -253,7 +257,6 @@ func (api_v2_tweet APIV2Tweet) ToTweetTrove() TweetTrove {
 		orig_tweet_trove := api_v2_tweet.RetweetedStatusResult.ToTweetTrove(false)
 		ret.MergeWith(orig_tweet_trove)
 
-
 		retweet := Retweet{}
 		var err error
 		retweet.RetweetID = TweetID(api_v2_tweet.ID)
@@ -289,13 +292,12 @@ type APIV2Entry struct {
 		EntryType  string `json:"entryType"`
 		Value      string `json:"value"`
 		CursorType string `json:"cursorType"`
-
 	} `json:"content"`
 }
 
 type APIV2Instruction struct {
 	Type    string       `json:"type"`
-	Entries []APIV2Entry`json:"entries"`
+	Entries []APIV2Entry `json:"entries"`
 }
 
 type APIV2Response struct {
@@ -324,7 +326,7 @@ func (api_response APIV2Response) GetMainInstruction() *APIV2Instruction {
 
 func (api_response APIV2Response) GetCursorBottom() string {
 	entries := api_response.GetMainInstruction().Entries
-	last_entry := entries[len(entries) - 1]
+	last_entry := entries[len(entries)-1]
 	if last_entry.Content.CursorType != "Bottom" {
 		panic("No bottom cursor found")
 	}
@@ -363,7 +365,6 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) {
 	return ret, nil
 }
 
-
 func get_graphql_user_timeline_url(user_id UserID, cursor string) string {
 	if cursor != "" {
 		return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22cursor%22%3A%22" + url.QueryEscape(cursor) + "%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_responsive_web_uc_gql_enabled%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" // nolint:lll  // It's a URL, come on

scraper/api_types_v2_test.go

@@ -1,10 +1,10 @@
 package scraper_test
 
 import (
-	"testing"
-	"os"
 	"encoding/json"
 	"fmt"
+	"os"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -34,7 +34,7 @@ func TestAPIV2ParseUser(t *testing.T) {
 	assert.Equal(user.ID, UserID(44067298))
 	assert.Equal(user.DisplayName, "Michael Malice")
 	assert.Equal(user.Handle, UserHandle("michaelmalice"))
-	assert.Equal(user.Bio, "Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & " +
+	assert.Equal(user.Bio, "Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & "+
 		"Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model")
 	assert.Equal(user.FollowingCount, 964)
 	assert.Equal(user.FollowersCount, 334571)
@@ -70,7 +70,7 @@ func TestAPIV2ParseTweet(t *testing.T) {
 	assert.True(ok)
 	assert.Equal(tweet.ID, TweetID(1485708879174508550))
 	assert.Equal(tweet.UserID, UserID(44067298))
-	assert.Equal(tweet.Text, "If Boris Johnson is driven out of office, it wouldn't mark the first time the Tories had four PMs in a " +
+	assert.Equal(tweet.Text, "If Boris Johnson is driven out of office, it wouldn't mark the first time the Tories had four PMs in a "+
 		"row\nThey had previously governed the UK for 13 years with 4 PMs, from 1951-1964")
 	assert.Equal(tweet.PostedAt.Unix(), int64(1643055574))
 	assert.Equal(tweet.QuotedTweetID, TweetID(0))
@@ -133,7 +133,7 @@ func TestAPIV2ParseTweetWithQuotedTweet(t *testing.T) {
 	assert.True(ok)
 	assert.Equal(TweetID(1485690410899021826), quote_tweet.ID)
 	assert.Equal(TweetID(1485690069079846915), quote_tweet.QuotedTweetID)
-	assert.Equal("Hatred is powerless in and of itself despite all the agitprop to the contrary\nHatred didnt stop Trump's election, " +
+	assert.Equal("Hatred is powerless in and of itself despite all the agitprop to the contrary\nHatred didnt stop Trump's election, "+
 		"for example", quote_tweet.Text)
 
 	// Should be 2 users: quoter and quoted
@@ -200,7 +200,6 @@ func TestAPIV2ParseRetweet(t *testing.T) {
 	assert.Equal(UserID(44067298), retweeting_user.ID)
 	assert.Equal(UserHandle("michaelmalice"), retweeting_user.Handle)
 
-
 	// Should be 1 retweet
 	assert.Equal(1, len(trove.Retweets))
 	retweet, ok := trove.Retweets[1485699748514476037]
@@ -270,7 +269,6 @@ func TestAPIV2ParseRetweetedQuoteTweet(t *testing.T) {
 	assert.Equal(UserID(599817378), retweet.RetweetedByID)
 }
 
-
 /**
  * Parse tweet with quoted tombstone
  */
@@ -300,13 +298,12 @@ func TestAPIV2ParseTweetWithQuotedTombstone(t *testing.T) {
 	assert.True(ok)
 	assert.Equal(TweetID(1485774025347371008), tombstoned_tweet.ID)
 	assert.Equal("no longer exists", tombstoned_tweet.TombstoneType)
-	assert.True (tombstoned_tweet.IsStub)
+	assert.True(tombstoned_tweet.IsStub)
 	assert.Equal(UserHandle("coltnkat"), tombstoned_tweet.UserHandle)
 
 	assert.Equal(0, len(trove.Retweets))
 }
 
-
 /**
  * Parse a tweet with a link
  */
@@ -326,7 +323,7 @@ func TestAPIV2ParseTweetWithURL(t *testing.T) {
 	assert.Equal(1, len(trove.Tweets))
 	tweet, ok := trove.Tweets[1485695695025803264]
 	assert.True(ok)
-	assert.Equal("This led to what I discussed as \"anguish signaling,\" where progs competed in proclaiming their distress both to " +
+	assert.Equal("This led to what I discussed as \"anguish signaling,\" where progs competed in proclaiming their distress both to "+
 		"show they were the Good Guys but also to get the pack to regroup, akin to wolves howling.", tweet.Text)
 
 	assert.Equal(1, len(tweet.Urls))
@@ -335,7 +332,7 @@ func TestAPIV2ParseTweetWithURL(t *testing.T) {
 	assert.Equal("observer.com", url.Domain)
 	assert.Equal("Why Evangelical Progressives Need to Demonstrate Anguish Publicly", url.Title)
 	assert.Equal("https://observer.com/2016/12/why-evangelical-progressives-need-to-demonstrate-anguish-publicly/", url.Text)
-	assert.Equal("The concept of “virtue signaling” gained a great deal of currency in this past year. It’s a way to demonstrate to " +
+	assert.Equal("The concept of “virtue signaling” gained a great deal of currency in this past year. It’s a way to demonstrate to "+
 		"others that one is a good person without having to do anything", url.Description)
 	assert.Equal("https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=600x600", url.ThumbnailRemoteUrl)
 	assert.Equal(600, url.ThumbnailWidth)
@@ -439,10 +436,9 @@ func TestAPIV2ParseTweetWithPoll(t *testing.T) {
 
 	assert.Equal(int64(1643137976), poll.VotingEndsAt.Unix())
 	assert.Equal(int64(1643055638), poll.LastUpdatedAt.Unix())
-	assert.Equal(1440 * 60, poll.VotingDuration)
+	assert.Equal(1440*60, poll.VotingDuration)
 }
 
-
 func TestParseAPIV2UserFeed(t *testing.T) {
 	data, err := os.ReadFile("test_responses/api_v2/user_feed_apiv2.json")
 	if err != nil {
@@ -495,7 +491,6 @@ func TestParseAPIV2UserFeed(t *testing.T) {
 	fmt.Printf("%d Users, %d Tweets, %d Retweets\n", len(tweet_trove.Users), len(tweet_trove.Tweets), len(tweet_trove.Retweets))
 }
 
-
 /**
  * Should correctly identify an "empty" response
  */
@@ -568,7 +563,6 @@ func TestAPIV2TombstoneEntry(t *testing.T) {
 	assert.Len(trove.Retweets, 0)
 }
 
-
 func TestTweetWithWarning(t *testing.T) {
 	assert := assert.New(t)
 	data, err := os.ReadFile("test_responses/api_v2/tweet_with_warning.json")

scraper/image_test.go

@@ -1,9 +1,9 @@
 package scraper_test
 
 import (
-    "testing"
-    "os"
 	"encoding/json"
+	"os"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"

scraper/link_expander.go

@@ -2,11 +2,10 @@ package scraper
 
 import (
 	"fmt"
-	"time"
 	"net/http"
+	"time"
 )
 
-
 /**
  * Return the expanded version of a short URL.  Input must be a real short URL.
  */

scraper/link_expander_test.go

@@ -11,7 +11,6 @@ import (
 	. "offline_twitter/scraper"
 )
 
-
 func TestExpandShortUrl(t *testing.T) {
 	redirecting_to := "redirect target"
 	srvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

scraper/poll.go

@@ -1,9 +1,9 @@
 package scraper
 
 import (
-    "strings"
-    "strconv"
 	"net/url"
+	"strconv"
+	"strings"
 )
 
 type PollID int64

scraper/poll_test.go

@@ -1,9 +1,9 @@
 package scraper_test
 
 import (
-    "testing"
-    "os"
 	"encoding/json"
+	"os"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -24,7 +24,7 @@ func TestParsePoll2Choices(t *testing.T) {
 	poll := ParseAPIPoll(apiCard)
 	assert.Equal(PollID(1457419248461131776), poll.ID)
 	assert.Equal(2, poll.NumChoices)
-    assert.Equal(60 * 60 * 24, poll.VotingDuration)
+	assert.Equal(60*60*24, poll.VotingDuration)
 	assert.Equal(int64(1636397201), poll.VotingEndsAt.Unix())
 	assert.Equal(int64(1636318755), poll.LastUpdatedAt.Unix())
 
@@ -48,7 +48,7 @@ func TestParsePoll4Choices(t *testing.T) {
 	poll := ParseAPIPoll(apiCard)
 	assert.Equal(PollID(1455611588854140929), poll.ID)
 	assert.Equal(4, poll.NumChoices)
-    assert.Equal(60 * 60 * 24, poll.VotingDuration)
+	assert.Equal(60*60*24, poll.VotingDuration)
 	assert.Equal(int64(1635966221), poll.VotingEndsAt.Unix())
 	assert.Equal(int64(1635966226), poll.LastUpdatedAt.Unix())
 	assert.Greater(poll.LastUpdatedAt.Unix(), poll.VotingEndsAt.Unix())

scraper/tweet.go

@@ -1,9 +1,9 @@
 package scraper
 
 import (
-	"time"
 	"fmt"
 	"strings"
+	"time"
 
 	"offline_twitter/terminal_utils"
 )
@@ -42,7 +42,6 @@ type Tweet struct {
 	LastScrapedAt         Timestamp
 }
 
-
 func (t Tweet) String() string {
 	var author string
 	if t.User != nil {
@@ -52,7 +51,7 @@ func (t Tweet) String() string {
 	}
 
 	ret := fmt.Sprintf(
-`%s
+		`%s
 %s
 %s
 Replies: %d      RT: %d      QT: %d      Likes: %d
@@ -67,11 +66,11 @@ Replies: %d      RT: %d      QT: %d      Likes: %d
 	)
 
 	if len(t.Images) > 0 {
-		ret += fmt.Sprintf(terminal_utils.COLOR_GREEN + "images: %d\n" + terminal_utils.COLOR_RESET, len(t.Images))
+		ret += fmt.Sprintf(terminal_utils.COLOR_GREEN+"images: %d\n"+terminal_utils.COLOR_RESET, len(t.Images))
 	}
 	if len(t.Urls) > 0 {
 		ret += "urls: [\n"
-		for _, url := range(t.Urls) {
+		for _, url := range t.Urls {
 			ret += "  " + url.Text + "\n"
 		}
 		ret += "]"
@@ -151,7 +150,6 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
 		}
 	}
 
-
 	// Process videos
 	for _, entity := range apiTweet.ExtendedEntities.Media {
 		if entity.Type != "video" && entity.Type != "animated_gif" {
@@ -181,7 +179,6 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
 	return
 }
 
-
 /**
  * Get a single tweet with no replies from the API.
  *
@@ -206,7 +203,6 @@ func GetTweet(id TweetID) (Tweet, error) {
 	return ParseSingleTweet(single_tweet)
 }
 
-
 /**
  * Return a list of tweets, including the original and the rest of its thread,
  * along with a list of associated users.

scraper/tweet_test.go

@@ -11,7 +11,7 @@ import (
 	. "offline_twitter/scraper"
 )
 
-func load_tweet_from_file(filename string) Tweet{
+func load_tweet_from_file(filename string) Tweet {
 	data, err := os.ReadFile(filename)
 	if err != nil {
 		panic(err)
@@ -28,12 +28,11 @@ func load_tweet_from_file(filename string) Tweet{
 	return tweet
 }
 
-
 func TestParseSingleTweet(t *testing.T) {
 	assert := assert.New(t)
 	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_unicode_chars.json")
 
-	assert.Equal("The fact that @michaelmalice new book ‘The Anarchist Handbook’ is just absolutely destroying on the charts is the " +
+	assert.Equal("The fact that @michaelmalice new book ‘The Anarchist Handbook’ is just absolutely destroying on the charts is the "+
 		"largest white pill I’ve swallowed in years.", tweet.Text)
 	assert.Len(tweet.Mentions, 1)
 	assert.Contains(tweet.Mentions, UserHandle("michaelmalice"))
@@ -73,7 +72,7 @@ func TestParseTweetWithQuotedTweetAndLink(t *testing.T) {
 	assert := assert.New(t)
 	tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_quoted_tweet_and_url.json")
 
-	assert.Equal("This is video he’s talking about. Please watch. Is there a single US politician capable of doing this with the " +
+	assert.Equal("This is video he’s talking about. Please watch. Is there a single US politician capable of doing this with the "+
 		"weasels and rats running American industry today?", tweet.Text)
 	assert.Equal(TweetID(1497997890999898115), tweet.QuotedTweetID)
 
@@ -135,7 +134,7 @@ func TestParseTweetWithMultipleUrls(t *testing.T) {
 
 	assert.False(tweet.Urls[0].HasCard)
 	assert.False(tweet.Urls[1].HasCard)
-	assert.True (tweet.Urls[2].HasCard)
+	assert.True(tweet.Urls[2].HasCard)
 
 	assert.Equal("Biden’s victory came from the suburbs", tweet.Urls[2].Title)
 }
@@ -166,12 +165,11 @@ func TestTweetWithPoll(t *testing.T) {
 	assert.Equal(624, p.Choice2_Votes)
 	assert.Equal(778, p.Choice3_Votes)
 	assert.Equal(1138, p.Choice4_Votes)
-	assert.Equal(1440 * 60, p.VotingDuration)
+	assert.Equal(1440*60, p.VotingDuration)
 	assert.Equal(int64(1638331934), p.VotingEndsAt.Unix())
 	assert.Equal(int64(1638331935), p.LastUpdatedAt.Unix())
 }
 
-
 func TestParseTweetResponse(t *testing.T) {
 	assert := assert.New(t)
 	data, err := os.ReadFile("test_responses/michael_malice_feed.json")
@@ -186,7 +184,7 @@ func TestParseTweetResponse(t *testing.T) {
 	require.NoError(t, err)
 	tweets, retweets, users := trove.Transform()
 
-	assert.Len(tweets, 29 - 3)
+	assert.Len(tweets, 29-3)
 	assert.Len(retweets, 3)
 	assert.Len(users, 9)
 }

scraper/url.go

@@ -2,9 +2,9 @@ package scraper
 
 import (
 	"fmt"
+	"net/url"
 	"path"
 	"regexp"
-	"net/url"
 )
 
 type Url struct {

scraper/url_test.go

@@ -1,9 +1,9 @@
 package scraper_test
 
 import (
-    "testing"
-    "os"
 	"encoding/json"
+	"os"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -24,7 +24,7 @@ func TestParseAPIUrlCard(t *testing.T) {
 	url := ParseAPIUrlCard(apiCard)
 	assert.Equal("reason.com", url.Domain)
 	assert.Equal("L.A. Teachers Union Leader: 'There's No Such Thing As Learning Loss'", url.Title)
-    assert.Equal("\"It’s OK that our babies may not have learned all their times tables,\" says Cecily Myart-Cruz. \"They learned " +
+	assert.Equal("\"It’s OK that our babies may not have learned all their times tables,\" says Cecily Myart-Cruz. \"They learned "+
 		"resilience.\"", url.Description)
 	assert.Equal(600, url.ThumbnailWidth)
 	assert.Equal(315, url.ThumbnailHeight)
@@ -49,7 +49,7 @@ func TestParseAPIUrlCardWithPlayer(t *testing.T) {
 	url := ParseAPIUrlCard(apiCard)
 	assert.Equal("www.youtube.com", url.Domain)
 	assert.Equal("The Politically Incorrect Guide to the Constitution (Starring Tom...", url.Title)
-    assert.Equal("Watch this episode on LBRY/Odysee: https://odysee.com/@capitalresearch:5/the-politically-incorrect-guide-to-the:8" +
+	assert.Equal("Watch this episode on LBRY/Odysee: https://odysee.com/@capitalresearch:5/the-politically-incorrect-guide-to-the:8"+
 		"Watch this episode on Rumble: https://rumble...", url.Description)
 	assert.Equal("https://pbs.twimg.com/card_img/1437849456423194639/_1t0btyt?format=jpg&name=800x320_1", url.ThumbnailRemoteUrl)
 	assert.Equal("_1t0btyt_800x320_1.jpg", url.ThumbnailLocalPath)
@@ -71,7 +71,7 @@ func TestParseAPIUrlCardWithPlayerAndPlaceholderThumbnail(t *testing.T) {
 	url := ParseAPIUrlCard(apiCard)
 	assert.Equal("www.youtube.com", url.Domain)
 	assert.Equal("Did Michael Malice Turn Me into an Anarchist? | Ep 181", url.Title)
-    assert.Equal("SUBSCRIBE TO THE NEW SHOW W/ ELIJAH & SYDNEY: \"YOU ARE HERE\"YT: https://www.youtube.com/youareheredaily____________" +
+	assert.Equal("SUBSCRIBE TO THE NEW SHOW W/ ELIJAH & SYDNEY: \"YOU ARE HERE\"YT: https://www.youtube.com/youareheredaily____________"+
 		"__________________________________________...", url.Description)
 	assert.Equal("https://pbs.twimg.com/cards/player-placeholder.png", url.ThumbnailRemoteUrl)
 	assert.Equal("player-placeholder.png", url.ThumbnailLocalPath)
@@ -102,7 +102,7 @@ func TestParseAPIUrlCardWithoutThumbnail(t *testing.T) {
  * Should check if a url is a tweet url, and if so, parse it
  */
 func TestParseTweetUrl(t *testing.T) {
-    assert:= assert.New(t)
+	assert := assert.New(t)
 
 	// Test valid tweet url
 	url := "https://twitter.com/kanesays23/status/1429583672827465730"

scraper/user.go

@@ -2,9 +2,9 @@ package scraper
 
 import (
 	"fmt"
-    "strings"
-    "regexp"
 	"path"
+	"regexp"
+	"strings"
 
 	"offline_twitter/terminal_utils"
 )
@@ -35,6 +35,7 @@ type User struct {
 	IsPrivate             bool
 	IsVerified            bool
 	IsBanned              bool
+	IsDeleted             bool
 	ProfileImageUrl       string
 	ProfileImageLocalPath string
 	BannerImageUrl        string
@@ -55,7 +56,7 @@ func (u User) String() string {
 		verified = "[\u2713]"
 	}
 	ret := fmt.Sprintf(
-`%s%s
+		`%s%s
 @%s
     %s
 
@@ -83,8 +84,6 @@ Joined %s
 	return ret
 }
 
-
-
 /**
  * Unknown Users with handles are only created by direct GetUser calls (either `twitter fetch_user`
  * subcommand or as part of tombstone user fetching.)
@@ -98,7 +97,7 @@ func GetUnknownUserWithHandle(handle UserHandle) User {
 		FollowersCount:  0,
 		FollowingCount:  0,
 		Location:        "<blank>",
-        Website:"<blank>",
+		Website:         "<blank>",
 		JoinDate:        TimestampFromUnix(0),
 		IsVerified:      false,
 		IsPrivate:       false,
@@ -141,7 +140,6 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) {
 	ret.IsVerified = apiUser.Verified
 	ret.ProfileImageUrl = apiUser.ProfileImageURLHTTPS
 
-
 	if regexp.MustCompile(`_normal\.\w{2,4}`).MatchString(ret.ProfileImageUrl) {
 		ret.ProfileImageUrl = strings.ReplaceAll(ret.ProfileImageUrl, "_normal.", ".")
 	}

scraper/user_feed.go

@@ -1,8 +1,8 @@
 package scraper
 
 import (
-	"fmt"
 	"errors"
+	"fmt"
 )
 
 /**
@@ -33,7 +33,6 @@ func GetUserFeedFor(user_id UserID, min_tweets int) (trove TweetTrove, err error
 	return ParseTweetResponse(tweet_response)
 }
 
-
 func GetUserFeedGraphqlFor(user_id UserID, min_tweets int) (trove TweetTrove, err error) {
 	api := API{}
 	api_response, err := api.GetGraphqlFeedFor(user_id, "")

scraper/user_test.go

@@ -1,10 +1,10 @@
 package scraper_test
 
 import (
-	"testing"
 	"encoding/json"
-	"os"
 	"net/http"
+	"os"
+	"testing"
 
 	"github.com/jarcoal/httpmock"
 	"github.com/stretchr/testify/assert"
@@ -31,7 +31,7 @@ func TestParseSingleUser(t *testing.T) {
 	assert.Equal(UserID(44067298), user.ID)
 	assert.Equal("Michael Malice", user.DisplayName)
 	assert.Equal(UserHandle("michaelmalice"), user.Handle)
-	assert.Equal("Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by " +
+	assert.Equal("Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by "+
 		"Harvey Pekar\nUnderwear Model\nHe/Him ⚑", user.Bio)
 	assert.Equal(941, user.FollowingCount)
 	assert.Equal(208589, user.FollowersCount)
@@ -39,7 +39,7 @@ func TestParseSingleUser(t *testing.T) {
 	assert.Equal("https://amzn.to/3oInafv", user.Website)
 	assert.Equal(int64(1243920952), user.JoinDate.Unix())
 	assert.False(user.IsPrivate)
-	assert.True (user.IsVerified)
+	assert.True(user.IsVerified)
 	assert.False(user.IsBanned)
 	assert.Equal("https://pbs.twimg.com/profile_images/1064051934812913664/Lbwdb_C9.jpg", user.ProfileImageUrl)
 	assert.Equal("https://pbs.twimg.com/profile_images/1064051934812913664/Lbwdb_C9_normal.jpg", user.GetTinyProfileImageUrl())

scraper/video.go

@@ -2,8 +2,8 @@ package scraper
 
 import (
 	"fmt"
-    "sort"
 	"path"
+	"sort"
 )
 
 type VideoID int64

scraper/video_test.go

@@ -1,9 +1,9 @@
 package scraper_test
 
 import (
-    "testing"
-    "os"
 	"encoding/json"
+	"os"
+	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"