Add db operations for Spaces

doc/TODO.txt

@@ -29,15 +29,12 @@ Video size reduction:
 - add "bitrate" and "available bitrates" fields
 - option to upgrade or downgrade video quality
 
-TODO https://twitter.com/michaelmalice/status/1461031030278742020 => tombstone should say "account no longer exists" but it says "Tweet is unavailable"
-
 TODO: Problem tweets
 - https://twitter.com/TyCardon/status/1480640777281839106: "Media has been disabled in response to report by copyright owner"
 - https://twitter.com/CovfefeAnon/status/1427693931047837702: parent thread not loading (possible GUI problem)
-- "account no longer exists" tombstone not being collected
+- "account no longer exists" tombstone not being collected => https://twitter.com/michaelmalice/status/1461031030278742020
 - tweet with warning label not getting scraped right: https://twitter.com/michaelmalice/status/1493324611999748098
 - fails to produce any result for the first tweet in the thread => https://twitter.com/CovfefeAnon/status/1498877082838962181
-- Invalid tweet url (mobile.twitter.com) => https://twitter.com/Mememarkets1/status/1499538952315088897
 
 TODO: videos-view-count
 - videos don't parse properly in APIv2

persistence/media_queries.go

@@ -102,6 +102,17 @@ func (p Profile) SavePoll(poll scraper.Poll) error {
 	return nil
 }
 
+/**
+ * Save a Space
+ */
+func (p Profile) SaveSpace(space scraper.Space) error {
+	_, err := p.DB.NamedExec(`insert into spaces (id, short_url) values (:id, :short_url) on conflict do nothing`, space)
+	if err != nil {
+		return fmt.Errorf("Error saving Space (ID %s):\n  %w", space.ID, err)
+	}
+	return nil
+}
+
 /**
  * Get the list of images for a tweet
  */
@@ -151,3 +162,11 @@ func (p Profile) GetPollsForTweet(t scraper.Tweet) (polls []scraper.Poll, err er
 	`, t.ID)
 	return
 }
+
+/**
+ * Get a Space by ID
+ */
+func (p Profile) GetSpace(id scraper.SpaceID) (space scraper.Space, err error) {
+	err = p.DB.Get(&space, `select id, short_url from spaces where id = ?`, id)
+	return
+}

persistence/media_queries_test.go

@@ -277,3 +277,22 @@ func TestModifyPoll(t *testing.T) {
 		t.Error(diff)
 	}
 }
+
+/**
+ * Create a Space, save it, reload it, and make sure it comes back the same
+ */
+func TestSaveAndLoadSpace(t *testing.T) {
+	require := require.New(t)
+	profile_path := "test_profiles/TestMediaQueries"
+	profile := create_or_load_profile(profile_path)
+
+	space := create_dummy_space()
+	err := profile.SaveSpace(space)
+	require.NoError(err)
+
+	new_space, err := profile.GetSpace(space.ID)
+	require.NoError(err)
+	if diff := deep.Equal(space, new_space); diff != nil {
+		t.Error(diff)
+	}
+}

persistence/schema.sql

@@ -112,6 +112,11 @@ create table polls (rowid integer primary key,
     foreign key(tweet_id) references tweets(id)
 );
 
+create table spaces(rowid integer primary key,
+    id text unique not null,
+    short_url text not null
+);
+
 create table images (rowid integer primary key,
     id integer unique not null check(typeof(id) = 'integer'),
     tweet_id integer not null,

persistence/utils_test.go

@@ -289,3 +289,10 @@ func create_dummy_retweet(tweet_id scraper.TweetID) scraper.Retweet {
 		RetweetedAt:   scraper.TimestampFromUnix(20000000),
 	}
 }
+
+func create_dummy_space() scraper.Space {
+	return scraper.Space{
+		ID: scraper.SpaceID(fmt.Sprintf("some_id_%d", rand.Int())),
+		ShortUrl: fmt.Sprintf("short_url_%d", rand.Int()),
+	}
+}

scraper/space.go

@@ -3,8 +3,8 @@ package scraper
 type SpaceID string
 
 type Space struct {
-	ID       SpaceID
+	ID       SpaceID `db:"id"`
-	ShortUrl string
+	ShortUrl string  `db:"short_url"`
 }
 
 func ParseAPISpace(apiCard APICard) Space {