Add downloading of DM embedded images, videos and links

internal/webserver/tpl/tweet_page_includes/chat_view.tpl

@@ -35,9 +35,61 @@
               }}
             </div>
           {{end}}
+          {{range $message.Images}}
+            <img class="dm-embedded-image"
+              {{if .IsDownloaded}}
+                src="/content/images/{{.LocalFilename}}"
+              {{else}}
+                src="{{.RemoteURL}}"
+              {{end}}
+              width="{{.Width}}" height="{{.Height}}"
+              onclick="image_carousel.querySelector('img').src = this.src; image_carousel.showModal();"
+            >
+          {{end}}
+          {{range $message.Videos}}
+            <video controls width="{{.Width}}" height="{{.Height}}"
+              {{if .IsDownloaded}}
+                poster="/content/video_thumbnails/{{.ThumbnailLocalPath}}"
+              {{else}}
+                poster="{{.ThumbnailRemoteUrl}}"
+              {{end}}
+            >
+              {{if .IsDownloaded}}
+                <source src="/content/videos/{{.LocalFilename}}">
+              {{else}}
+                <source src="{{.RemoteURL}}">
+              {{end}}
+            </video>
+          {{end}}
+          {{range $message.Urls}}
+            <a
+              class="embedded-link rounded-gray-outline unstyled-link"
+              target="_blank"
+              href="{{.Text}}"
+              style="max-width: {{if (ne .ThumbnailWidth 0)}}{{.ThumbnailWidth}}px {{else}}fit-content {{end}}"
+            >
+              <img
+                {{if .IsContentDownloaded}}
+                  src="/content/link_preview_images/{{.ThumbnailLocalPath}}"
+                {{else}}
+                  src="{{.ThumbnailRemoteUrl}}"
+                {{end}}
+                class="embedded-link-preview"
+                width="{{.ThumbnailWidth}}" height="{{.ThumbnailHeight}}"
+              />
+              <h3 class="embedded-link-title">{{.Title}}</h3>
+              <p class="embedded-link-description">{{.Description}}</p>
+              <span class="row embedded-link-domain-container">
+                <img class="svg-icon" src="/static/icons/link3.svg" width="24" height="24" />
+                <span class="embedded-link-domain">{{(.GetDomain)}}</span>
+              </span>
+            </a>
+          {{end}}
+          {{if $message.Text}}
             <div class="dm-message-text-container">
               {{template "text-with-entities" $message.Text}}
             </div>
+          {{end}}
         </div>
       </div>
       <div class="dm-message-reactions">

pkg/persistence/dm_queries.go

@@ -67,7 +67,6 @@ func (p Profile) SaveChatRoom(r DMChatRoom) error {
 	if err != nil {
 		return fmt.Errorf("Error saving chat participant: %#v\n  %w", r, err)
 	}
-	// }
 	return nil
 }
 
@@ -100,6 +99,7 @@ func (p Profile) GetChatRoom(id DMChatRoomID) (ret DMChatRoom, err error) {
 }
 
 func (p Profile) SaveChatMessage(m DMMessage) error {
+	// The message itself
 	_, err := p.DB.NamedExec(`
 		insert into chat_messages (id, chat_room_id, sender_id, sent_at, request_id, in_reply_to_id, text, embedded_tweet_id)
 		values (:id, :chat_room_id, :sender_id, :sent_at, :request_id, :in_reply_to_id, :text, :embedded_tweet_id)
@@ -110,6 +110,7 @@ func (p Profile) SaveChatMessage(m DMMessage) error {
 		return fmt.Errorf("Error saving message: %#v\n  %w", m, err)
 	}
 
+	// Reactions
 	for _, reacc := range m.Reactions {
 		fmt.Println(reacc)
 		_, err = p.DB.NamedExec(`
@@ -122,6 +123,59 @@ func (p Profile) SaveChatMessage(m DMMessage) error {
 			return fmt.Errorf("Error saving message reaction (message %d, reacc %d): %#v\n  %w", m.ID, reacc.ID, reacc, err)
 		}
 	}
+
+	// Images
+	for _, img := range m.Images {
+		_, err := p.DB.NamedExec(`
+			insert into chat_message_images (id, chat_message_id, width, height, remote_url, local_filename, is_downloaded)
+			            values (:id, :chat_message_id, :width, :height, :remote_url, :local_filename, :is_downloaded)
+			       on conflict do update
+			               set is_downloaded=(is_downloaded or :is_downloaded)
+			`,
+			img,
+		)
+		if err != nil {
+			return fmt.Errorf("Error saving image (message ID %d):\n  %w", img.DMMessageID, err)
+		}
+	}
+
+	// Videos
+	for _, vid := range m.Videos {
+		_, err := p.DB.NamedExec(`
+			insert into chat_message_videos
+			            (id, chat_message_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
+			             duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif)
+			     values (:id, :chat_message_id, :width, :height, :remote_url, :local_filename, :thumbnail_remote_url,
+			            :thumbnail_local_filename, :duration, :view_count, :is_downloaded, :is_blocked_by_dmca, :is_gif)
+			on conflict do update
+			        set is_downloaded=(is_downloaded or :is_downloaded),
+			            view_count=max(view_count, :view_count),
+			            is_blocked_by_dmca = :is_blocked_by_dmca
+			`,
+			vid,
+		)
+		if err != nil {
+			return fmt.Errorf("Error saving video (message ID %d):\n  %w", vid.DMMessageID, err)
+		}
+	}
+
+	// Urls
+	for _, url := range m.Urls {
+		_, err := p.DB.NamedExec(`
+			insert into chat_message_urls (chat_message_id, domain, text, short_text, title, description, creator_id, site_id,
+			                               thumbnail_width, thumbnail_height, thumbnail_remote_url, thumbnail_local_path, has_card,
+			                               has_thumbnail, is_content_downloaded)
+			     values (:chat_message_id, :domain, :text, :short_text, :title, :description, :creator_id, :site_id, :thumbnail_width,
+			             :thumbnail_height, :thumbnail_remote_url, :thumbnail_local_path, :has_card, :has_thumbnail, :is_content_downloaded
+			            )
+			on conflict do update
+			        set is_content_downloaded=(is_content_downloaded or :is_content_downloaded)
+		`, url)
+		if err != nil {
+			return fmt.Errorf("Error saving Url (message ID %d):\n  %w", url.DMMessageID, err)
+		}
+	}
+
 	return nil
 }
 
@@ -133,9 +187,10 @@ func (p Profile) GetChatMessage(id DMMessageID) (ret DMMessage, err error) {
 		`, id,
 	)
 	if err != nil {
-		return ret, fmt.Errorf("Error getting chat message (%d):\n  %w", id, err)
+		return ret, fmt.Errorf("Error getting chat message %d:\n  %w", id, err)
 	}
 
+	// Reactions
 	reaccs := []DMReaction{}
 	err = p.DB.Select(&reaccs, `
 		select id, message_id, sender_id, sent_at, emoji
@@ -144,12 +199,45 @@ func (p Profile) GetChatMessage(id DMMessageID) (ret DMMessage, err error) {
 		`, id,
 	)
 	if err != nil {
-		return ret, fmt.Errorf("Error getting reactions to chat message (%d):\n  %w", id, err)
+		return ret, fmt.Errorf("Error getting reactions to chat message %d:\n  %w", id, err)
 	}
 	ret.Reactions = make(map[UserID]DMReaction)
 	for _, r := range reaccs {
 		ret.Reactions[r.SenderID] = r
 	}
+
+	// Images
+	err = p.DB.Select(&ret.Images, `
+		select id, chat_message_id, width, height, remote_url, local_filename, is_downloaded
+		  from chat_message_images
+		 where chat_message_id = ?
+	`, ret.ID)
+	if err != nil {
+		return ret, fmt.Errorf("Error getting images for chat messsage %d:\n  %w", id, err)
+	}
+
+	// Videos
+	err = p.DB.Select(&ret.Videos, `
+		select id, chat_message_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
+		       duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif
+		  from chat_message_videos
+		 where chat_message_id = ?
+	`, ret.ID)
+	if err != nil {
+		return ret, fmt.Errorf("Error getting videos for chat messsage %d:\n  %w", id, err)
+	}
+
+	// Urls
+	err = p.DB.Select(&ret.Urls, `
+		select chat_message_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height,
+		       thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded
+		  from chat_message_urls
+		 where chat_message_id = ?
+	`, ret.ID)
+	if err != nil {
+		return ret, fmt.Errorf("Error getting urls for chat messsage %d:\n  %w", id, err)
+	}
+
 	return ret, nil
 }
 
@@ -319,6 +407,57 @@ func (p Profile) GetChatRoomContents(id DMChatRoomID, latest_timestamp int) DMCh
 			ret.Messages[reacc.DMMessageID] = msg
 		}
 
+		// Images
+		var images []Image
+		err = p.DB.Select(&images, `
+			select id, chat_message_id, width, height, remote_url, local_filename, is_downloaded
+			  from chat_message_images
+			 where chat_message_id in (`+strings.Repeat("?,", len(ret.MessageIDs)-1)+`?)
+		`, message_ids_copy...)
+		if err != nil {
+			panic(err)
+		}
+		for _, img := range images {
+			msg := ret.Messages[img.DMMessageID]
+			msg.Images = []Image{img}
+			ret.Messages[msg.ID] = msg
+		}
+
+		// Videos
+		var videos []Video
+		err = p.DB.Select(&videos, `
+			select id, chat_message_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
+			       duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif
+			  from chat_message_videos
+			 where chat_message_id in (`+strings.Repeat("?,", len(ret.MessageIDs)-1)+`?)
+		`, message_ids_copy...)
+		if err != nil {
+			panic(err)
+		}
+		for _, vid := range videos {
+			println("asdfasfasdf")
+			msg := ret.Messages[vid.DMMessageID]
+			msg.Videos = []Video{vid}
+			ret.Messages[msg.ID] = msg
+		}
+
+		// Urls
+		var urls []Url
+		err = p.DB.Select(&urls, `
+			select chat_message_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height,
+			       thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded
+			  from chat_message_urls
+			 where chat_message_id in (`+strings.Repeat("?,", len(ret.MessageIDs)-1)+`?)
+		`, message_ids_copy...)
+		if err != nil {
+			panic(err)
+		}
+		for _, url := range urls {
+			msg := ret.Messages[url.DMMessageID]
+			msg.Urls = []Url{url}
+			ret.Messages[msg.ID] = msg
+		}
+
 		// Fetch all embedded tweets
 		embedded_tweet_ids := []interface{}{}
 		for _, m := range ret.Messages {
@@ -343,7 +482,7 @@ func (p Profile) GetChatRoomContents(id DMChatRoomID, latest_timestamp int) DMCh
 			}
 		}
 
-		// Fetch message previews
+		// Fetch replied-to message previews
 		replied_message_ids := []interface{}{}
 		for _, m := range ret.Messages {
 			if m.InReplyToID != 0 {

pkg/persistence/dm_queries_test.go

@@ -167,11 +167,11 @@ func TestGetChatRoomsPreview(t *testing.T) {
 
 	room, is_ok := chat_view.Rooms[chat_view.RoomIDs[0]]
 	require.True(is_ok)
-	assert.Equal(room.LastMessageID, DMMessageID(1665936253483614212))
+	assert.Equal(room.LastMessageID, DMMessageID(1766595519000760325))
 
 	msg, is_ok := chat_view.Messages[room.LastMessageID]
 	require.True(is_ok)
-	assert.Equal(msg.Text, "Check this out")
+	assert.Equal(msg.Text, "This looks pretty good huh")
 
 	require.Len(room.Participants, 2)
 	for _, user_id := range []UserID{1458284524761075714, 1488963321701171204} {
@@ -207,14 +207,31 @@ func TestGetChatRoomContents(t *testing.T) {
 	}
 
 	// Messages
-	require.Equal(chat_view.MessageIDs, []DMMessageID{1663623062195957773, 1663623203644751885, 1665922180176044037, 1665936253483614212})
+	expected_message_ids := []DMMessageID{
-	require.Len(chat_view.Messages, 4)
+		1663623062195957773, 1663623203644751885, 1665922180176044037, 1665936253483614212,
+		1766248283901776125, 1766255994668191902, 1766595519000760325,
+	}
+	require.Equal(chat_view.MessageIDs, expected_message_ids)
+	require.Len(chat_view.Messages, len(expected_message_ids))
 	for _, msg_id := range chat_view.MessageIDs {
 		msg, is_ok := chat_view.Messages[msg_id]
 		assert.True(is_ok)
 		assert.Equal(msg.ID, msg_id)
 	}
 
+	// Attachments
+	m_img := chat_view.Messages[DMMessageID(1766595519000760325)]
+	require.Len(m_img.Images, 1)
+	assert.Equal(m_img.Images[0].RemoteURL,
+		"https://ton.twitter.com/1.1/ton/data/dm/1766595519000760325/1766595500407459840/ML6pC79A.png")
+	m_vid := chat_view.Messages[DMMessageID(1766248283901776125)]
+	require.Len(m_vid.Videos, 1)
+	assert.Equal(m_vid.Videos[0].RemoteURL,
+		"https://video.twimg.com/dm_video/1766248268416385024/vid/avc1/500x280/edFuZXtEVvem158AjvmJ3SZ_1DdG9cbSoW4fm6cDF1k.mp4?tag=1")
+	m_url := chat_view.Messages[DMMessageID(1766255994668191902)]
+	require.Len(m_url.Urls, 1)
+	assert.Equal(m_url.Urls[0].Text, "https://offline-twitter.com/introduction/data-ownership-and-composability/")
+
 	// Reactions
 	msg_with_reacc := chat_view.Messages[DMMessageID(1663623062195957773)]
 	require.Len(msg_with_reacc.Reactions, 1)

pkg/persistence/dm_trove_queries.go

@@ -1,13 +1,17 @@
 package persistence
 
 import (
+	"errors"
 	"fmt"
+	"path"
 
 	. "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper"
 )
 
 // Convenience function that saves all the objects in a TweetTrove.
 // Panics if anything goes wrong.
+//
+// TODO: a lot of this function contains duplicated code and should be extracted to functions
 func (p Profile) SaveDMTrove(trove DMTrove, should_download bool) {
 	p.SaveTweetTrove(trove.TweetTrove, should_download)
 
@@ -22,5 +26,106 @@ func (p Profile) SaveDMTrove(trove DMTrove, should_download bool) {
 		if err != nil {
 			panic(fmt.Errorf("Error saving chat message: %#v\n  %w", m, err))
 		}
+
+		// TODO: all of this is very duplicated and should be refactored
+		// Copied from media_download.go functions:
+		// - download_tweet_image, download_tweet_video, download_link_thumbnail
+		// - DownloadTweetContentWithInjector
+		// Copied from tweet_queries.go functions:
+		// - CheckTweetContentDownloadNeeded
+
+		// Download content if needed
+		if should_download {
+			downloader := DefaultDownloader{}
+
+			for _, img := range m.Images {
+				// Check if it's already downloaded
+				var is_downloaded bool
+				err := p.DB.Get(&is_downloaded, `select is_downloaded from chat_message_images where id = ?`, img.ID)
+				if err != nil {
+					panic(err)
+				}
+				if is_downloaded {
+					// Already downloaded; skip
+					continue
+				}
+
+				// DUPE: download-image
+				outfile := path.Join(p.ProfileDir, "images", img.LocalFilename)
+				err = downloader.Curl(img.RemoteURL, outfile)
+				if err != nil {
+					panic(fmt.Errorf("downloading image %q on DM message %d:\n  %w", img.RemoteURL, m.ID, err))
+				}
+				_, err = p.DB.NamedExec(`update chat_message_images set is_downloaded = 1 where id = :id`, img)
+				if err != nil {
+					panic(err)
+				}
+			}
+
+			for _, vid := range m.Videos {
+				// Videos can be geoblocked, and the HTTP response isn't in JSON so it's hard to capture
+				if vid.IsGeoblocked {
+					continue
+				}
+
+				// Check if it's already downloaded
+				var is_downloaded bool
+				err := p.DB.Get(&is_downloaded, `select is_downloaded from chat_message_videos where id = ?`, vid.ID)
+				if err != nil {
+					panic(err)
+				}
+				if is_downloaded {
+					// Already downloaded; skip
+					continue
+				}
+
+				// DUPE: download-video
+				// Download the video
+				outfile := path.Join(p.ProfileDir, "videos", vid.LocalFilename)
+				err = downloader.Curl(vid.RemoteURL, outfile)
+
+				if errors.Is(err, ErrorDMCA) {
+					vid.IsDownloaded = false
+					vid.IsBlockedByDMCA = true
+				} else if err != nil {
+					panic(fmt.Errorf("downloading video %q on DM message %d:\n  %w", vid.RemoteURL, m.ID, err))
+				} else {
+					vid.IsDownloaded = true
+				}
+
+				// Download the thumbnail
+				outfile = path.Join(p.ProfileDir, "video_thumbnails", vid.ThumbnailLocalPath)
+				err = downloader.Curl(vid.ThumbnailRemoteUrl, outfile)
+				if err != nil {
+					panic(fmt.Errorf("Error downloading video thumbnail (DMMessageID %d):\n  %w", vid.DMMessageID, err))
+				}
+
+				// Update it in the DB
+				_, err = p.DB.NamedExec(`
+					update chat_message_videos set is_downloaded = :is_downloaded, is_blocked_by_dmca = :is_blocked_by_dmca where id = :id
+				`, vid)
+				if err != nil {
+					panic(err)
+				}
+			}
+
+			for _, url := range m.Urls {
+				// DUPE: download-link-thumbnail
+				if url.HasCard && url.HasThumbnail {
+					outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath)
+					err := downloader.Curl(url.ThumbnailRemoteUrl, outfile)
+					if err != nil {
+						panic(fmt.Errorf("downloading link thumbnail %q on DM message %d:\n  %w", url.ThumbnailRemoteUrl, m.ID, err))
+					}
+				}
+				url.IsContentDownloaded = true
+
+				// Update it in the DB
+				_, err = p.DB.NamedExec(`update chat_message_urls set is_downloaded = :is_downloaded where id = :id`, url)
+				if err != nil {
+					panic(err)
+				}
+			}
+		}
 	}
 }

pkg/persistence/media_download.go

@@ -45,6 +45,7 @@ func (d DefaultDownloader) Curl(url string, outpath string) error {
 }
 
 // Downloads an Image, and if successful, marks it as downloaded in the DB
+// DUPE: download-image
 func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownloader) error {
 	outfile := path.Join(p.ProfileDir, "images", img.LocalFilename)
 	err := downloader.Curl(img.RemoteURL, outfile)
@@ -56,6 +57,7 @@ func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownlo
 }
 
 // Downloads a Video and its thumbnail, and if successful, marks it as downloaded in the DB
+// DUPE: download-video
 func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownloader) error {
 	// Download the video
 	outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename)
@@ -82,6 +84,7 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload
 }
 
 // Downloads an URL thumbnail image, and if successful, marks it as downloaded in the DB
+// DUPE: download-link-thumbnail
 func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownloader) error {
 	if url.HasCard && url.HasThumbnail {
 		outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath)

pkg/persistence/schema.sql

@@ -314,6 +314,60 @@ create table chat_message_reactions (rowid integer primary key,
     foreign key(sender_id) references users(id)
 );
 
+create table chat_message_images (rowid integer primary key,
+    id integer unique not null check(typeof(id) = 'integer'),
+    chat_message_id integer not null,
+    width integer not null,
+    height integer not null,
+    remote_url text not null unique,
+    local_filename text not null unique,
+    is_downloaded boolean default 0,
+
+    foreign key(chat_message_id) references chat_messages(id)
+);
+create index if not exists index_chat_message_images_chat_message_id on chat_message_images (chat_message_id);
+
+create table chat_message_videos (rowid integer primary key,
+    id integer unique not null check(typeof(id) = 'integer'),
+    chat_message_id integer not null,
+    width integer not null,
+    height integer not null,
+    remote_url text not null unique,
+    local_filename text not null unique,
+    thumbnail_remote_url text not null default "missing",
+    thumbnail_local_filename text not null default "missing",
+    duration integer not null default 0,
+    view_count integer not null default 0,
+    is_gif boolean default 0,
+    is_downloaded boolean default 0,
+    is_blocked_by_dmca boolean not null default 0,
+
+    foreign key(chat_message_id) references chat_messages(id)
+);
+create index if not exists index_chat_message_videos_chat_message_id on chat_message_videos (chat_message_id);
+
+create table chat_message_urls (rowid integer primary key,
+    chat_message_id integer not null,
+    domain text,
+    text text not null,
+    short_text text not null default "",
+    title text,
+    description text,
+    creator_id integer,
+    site_id integer,
+    thumbnail_width integer not null,
+    thumbnail_height integer not null,
+    thumbnail_remote_url text,
+    thumbnail_local_path text,
+    has_card boolean,
+    has_thumbnail boolean,
+    is_content_downloaded boolean default 0,
+
+    unique (chat_message_id, text)
+    foreign key(chat_message_id) references chat_messages(id)
+);
+create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id);
+
 
 -- Meta
 -- ----

pkg/persistence/utils_test.go

@@ -358,6 +358,15 @@ func create_dummy_chat_room() DMChatRoom {
 func create_dummy_chat_message() DMMessage {
 	rand.Seed(time.Now().UnixNano())
 	id := DMMessageID(rand.Int())
+	vid := create_video_from_id(int(id))
+	vid.TweetID = TweetID(0)
+	vid.DMMessageID = id
+	img := create_image_from_id(int(id))
+	img.TweetID = TweetID(0)
+	img.DMMessageID = id
+	url := create_url_from_id(int(id))
+	url.TweetID = TweetID(0)
+	url.DMMessageID = id
 	return DMMessage{
 		ID:           id,
 		DMChatRoomID: create_stable_chat_room().ID,
@@ -374,5 +383,8 @@ func create_dummy_chat_message() DMMessage {
 				Emoji:       "🤔",
 			},
 		},
+		Videos: []Video{vid},
+		Images: []Image{img},
+		Urls:   []Url{url},
 	}
 }

pkg/persistence/versions.go

@@ -244,6 +244,59 @@ var MIGRATIONS = []string{
 		create index if not exists index_list_users_user_id on list_users (user_id);
 		insert into lists(rowid, name) values (1, "Offline Follows");
 		insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;`,
+	`create table chat_message_images (rowid integer primary key,
+		    id integer unique not null check(typeof(id) = 'integer'),
+		    chat_message_id integer not null,
+		    width integer not null,
+		    height integer not null,
+		    remote_url text not null unique,
+		    local_filename text not null unique,
+		    is_downloaded boolean default 0,
+
+		    foreign key(chat_message_id) references chat_messages(id)
+		);
+		create index if not exists index_chat_message_images_chat_message_id on chat_message_images (chat_message_id);
+
+		create table chat_message_videos (rowid integer primary key,
+		    id integer unique not null check(typeof(id) = 'integer'),
+		    chat_message_id integer not null,
+		    width integer not null,
+		    height integer not null,
+		    remote_url text not null unique,
+		    local_filename text not null unique,
+		    thumbnail_remote_url text not null default "missing",
+		    thumbnail_local_filename text not null default "missing",
+		    duration integer not null default 0,
+		    view_count integer not null default 0,
+		    is_gif boolean default 0,
+		    is_downloaded boolean default 0,
+		    is_blocked_by_dmca boolean not null default 0,
+
+		    foreign key(chat_message_id) references chat_messages(id)
+		);
+		create index if not exists index_chat_message_videos_chat_message_id on chat_message_videos (chat_message_id);
+
+		create table chat_message_urls (rowid integer primary key,
+		    chat_message_id integer not null,
+		    domain text,
+		    text text not null,
+		    short_text text not null default "",
+		    title text,
+		    description text,
+		    creator_id integer,
+		    site_id integer,
+		    thumbnail_width integer not null,
+		    thumbnail_height integer not null,
+		    thumbnail_remote_url text,
+		    thumbnail_local_path text,
+		    has_card boolean,
+		    has_thumbnail boolean,
+		    is_content_downloaded boolean default 0,
+
+		    unique (chat_message_id, text)
+		    foreign key(chat_message_id) references chat_messages(id)
+		);
+		create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id);`,
 }
 var ENGINE_DATABASE_VERSION = len(MIGRATIONS)
 

pkg/scraper/api_request_utils.go

@@ -73,6 +73,9 @@ func (api *API) UnmarshalJSON(data []byte) error {
 	if err != nil {
 		panic(err)
 	}
+	for i := range in_struct.Cookies {
+		in_struct.Cookies[i].Domain = ".twitter.com"
+	}
 	cookie_jar.SetCookies(&TWITTER_BASE_URL, in_struct.Cookies)
 	api.IsAuthenticated = in_struct.IsAuthenticated
 	api.GuestToken = in_struct.GuestToken

pkg/scraper/api_types_dms.go

@@ -76,6 +76,9 @@ func (m *APIDMMessage) NormalizeContent() {
 
 func (m APIDMMessage) ToDMTrove() DMTrove {
 	ret := NewDMTrove()
+	if m.ID == 0 {
+		return ret
+	}
 
 	m.NormalizeContent()
 	result := ParseAPIDMMessage(m)

sample_data/seed_data.sql

@@ -413,8 +413,10 @@ INSERT INTO chat_messages VALUES
     (6,1665936253483614214,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129141,'',0,'bruh2',0),
     (7,1665936253483614215,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129142,'',1665936253483614214,'replying to bruh2',0),
     (8,1665936253483614216,'1488963321701171204-1178839081222115328',1488963321701171204,1686025129143,'',0,'This conversation is totally fake lol',0),
-    (9,1665936253483614217,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129144,'',0,'exactly',0);
+    (9,1665936253483614217,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129144,'',0,'exactly',0),
-
+    (36,1766248283901776125,'1458284524761075714-1488963321701171204',1458284524761075714,1709941380913,'',0,'',0),
+    (15,1766255994668191902,'1458284524761075714-1488963321701171204',1458284524761075714,1709943219300,'',0,'You wrote this?',0),
+    (46,1766595519000760325,'1458284524761075714-1488963321701171204',1458284524761075714,1710024168245,'',0,'This looks pretty good huh',0);
 
 
 create table chat_message_reactions (rowid integer primary key,
@@ -431,6 +433,64 @@ INSERT INTO chat_message_reactions VALUES
     (2,1665936253487546456,1665936253483614216,1488963321701171204,1686063453455,'🤔'),
     (3,1665936253834578774,1665936253483614216,1178839081222115328,1686075343331,'🤔');
 
+create table chat_message_images (rowid integer primary key,
+    id integer unique not null check(typeof(id) = 'integer'),
+    chat_message_id integer not null,
+    width integer not null,
+    height integer not null,
+    remote_url text not null unique,
+    local_filename text not null unique,
+    is_downloaded boolean default 0,
+
+    foreign key(chat_message_id) references chat_messages(id)
+);
+create index if not exists index_chat_message_images_chat_message_id on chat_message_images (chat_message_id);
+INSERT INTO chat_message_images VALUES(1,1766595500407459840,1766595519000760325,680,597,'https://ton.twitter.com/1.1/ton/data/dm/1766595519000760325/1766595500407459840/ML6pC79A.png','ML/ML6pC79A.png',0);
+
+create table chat_message_videos (rowid integer primary key,
+    id integer unique not null check(typeof(id) = 'integer'),
+    chat_message_id integer not null,
+    width integer not null,
+    height integer not null,
+    remote_url text not null unique,
+    local_filename text not null unique,
+    thumbnail_remote_url text not null default "missing",
+    thumbnail_local_filename text not null default "missing",
+    duration integer not null default 0,
+    view_count integer not null default 0,
+    is_gif boolean default 0,
+    is_downloaded boolean default 0,
+    is_blocked_by_dmca boolean not null default 0,
+
+    foreign key(chat_message_id) references chat_messages(id)
+);
+create index if not exists index_chat_message_videos_chat_message_id on chat_message_videos (chat_message_id);
+INSERT INTO chat_message_videos VALUES
+    (1,1766248268416385024,1766248283901776125,500,280,'https://video.twimg.com/dm_video/1766248268416385024/vid/avc1/500x280/edFuZXtEVvem158AjvmJ3SZ_1DdG9cbSoW4fm6cDF1k.mp4?tag=1','ed/edFuZXtEVvem158AjvmJ3SZ_1DdG9cbSoW4fm6cDF1k.mp4','https://pbs.twimg.com/dm_video_preview/1766248268416385024/img/Ph7CCqISQxFE40Yy-uJAis-WiYhBbexFe_czkN5ytzI.jpg','Ph/Ph7CCqISQxFE40Yy-uJAis-WiYhBbexFe_czkN5ytzI.jpg',1980,0,0,0,0);
+
+create table chat_message_urls (rowid integer primary key,
+    chat_message_id integer not null,
+    domain text,
+    text text not null,
+    short_text text not null default "",
+    title text,
+    description text,
+    creator_id integer,
+    site_id integer,
+    thumbnail_width integer not null,
+    thumbnail_height integer not null,
+    thumbnail_remote_url text,
+    thumbnail_local_path text,
+    has_card boolean,
+    has_thumbnail boolean,
+    is_content_downloaded boolean default 0,
+
+    unique (chat_message_id, text)
+    foreign key(chat_message_id) references chat_messages(id)
+);
+create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id);
+INSERT INTO chat_message_urls VALUES
+    (1,1766255994668191902,'offline-twitter.com','https://offline-twitter.com/introduction/data-ownership-and-composability/','https://t.co/V3iiSYyrQx','Data ownership and composability','Data and Composability # What does it mean to own data? It means: You have a full copy of it It lasts until you decide to delete it You can do whatever you want with it, including opening it with...',0,0,0,0,'','',1,0,0);
 
 create table follows(rowid integer primary key,
     follower_id integer not null,
@@ -453,6 +513,6 @@ insert into fake_user_sequence values(0x4000000000000000);
 create table database_version(rowid integer primary key,
     version_number integer not null unique
 );
-insert into database_version(version_number) values (28);
+insert into database_version(version_number) values (29);
 
 COMMIT;