diff --git a/internal/webserver/tpl/tweet_page_includes/chat_view.tpl b/internal/webserver/tpl/tweet_page_includes/chat_view.tpl index a737da5..de1b6b8 100644 --- a/internal/webserver/tpl/tweet_page_includes/chat_view.tpl +++ b/internal/webserver/tpl/tweet_page_includes/chat_view.tpl @@ -35,9 +35,61 @@ }} {{end}} -
- {{template "text-with-entities" $message.Text}} -
+ {{range $message.Images}} + + {{end}} + {{range $message.Videos}} + + {{end}} + {{range $message.Urls}} + + +

{{.Title}}

+

{{.Description}}

+ + + {{(.GetDomain)}} + + + {{end}} + {{if $message.Text}} +
+ {{template "text-with-entities" $message.Text}} +
+ {{end}}
diff --git a/pkg/persistence/dm_queries.go b/pkg/persistence/dm_queries.go index 1a26f02..2ec1f1c 100644 --- a/pkg/persistence/dm_queries.go +++ b/pkg/persistence/dm_queries.go @@ -67,7 +67,6 @@ func (p Profile) SaveChatRoom(r DMChatRoom) error { if err != nil { return fmt.Errorf("Error saving chat participant: %#v\n %w", r, err) } - // } return nil } @@ -100,6 +99,7 @@ func (p Profile) GetChatRoom(id DMChatRoomID) (ret DMChatRoom, err error) { } func (p Profile) SaveChatMessage(m DMMessage) error { + // The message itself _, err := p.DB.NamedExec(` insert into chat_messages (id, chat_room_id, sender_id, sent_at, request_id, in_reply_to_id, text, embedded_tweet_id) values (:id, :chat_room_id, :sender_id, :sent_at, :request_id, :in_reply_to_id, :text, :embedded_tweet_id) @@ -110,6 +110,7 @@ func (p Profile) SaveChatMessage(m DMMessage) error { return fmt.Errorf("Error saving message: %#v\n %w", m, err) } + // Reactions for _, reacc := range m.Reactions { fmt.Println(reacc) _, err = p.DB.NamedExec(` @@ -122,6 +123,59 @@ func (p Profile) SaveChatMessage(m DMMessage) error { return fmt.Errorf("Error saving message reaction (message %d, reacc %d): %#v\n %w", m.ID, reacc.ID, reacc, err) } } + + // Images + for _, img := range m.Images { + _, err := p.DB.NamedExec(` + insert into chat_message_images (id, chat_message_id, width, height, remote_url, local_filename, is_downloaded) + values (:id, :chat_message_id, :width, :height, :remote_url, :local_filename, :is_downloaded) + on conflict do update + set is_downloaded=(is_downloaded or :is_downloaded) + `, + img, + ) + if err != nil { + return fmt.Errorf("Error saving image (message ID %d):\n %w", img.DMMessageID, err) + } + } + + // Videos + for _, vid := range m.Videos { + _, err := p.DB.NamedExec(` + insert into chat_message_videos + (id, chat_message_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, + duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif) + values (:id, :chat_message_id, :width, :height, :remote_url, :local_filename, :thumbnail_remote_url, + :thumbnail_local_filename, :duration, :view_count, :is_downloaded, :is_blocked_by_dmca, :is_gif) + on conflict do update + set is_downloaded=(is_downloaded or :is_downloaded), + view_count=max(view_count, :view_count), + is_blocked_by_dmca = :is_blocked_by_dmca + `, + vid, + ) + if err != nil { + return fmt.Errorf("Error saving video (message ID %d):\n %w", vid.DMMessageID, err) + } + } + + // Urls + for _, url := range m.Urls { + _, err := p.DB.NamedExec(` + insert into chat_message_urls (chat_message_id, domain, text, short_text, title, description, creator_id, site_id, + thumbnail_width, thumbnail_height, thumbnail_remote_url, thumbnail_local_path, has_card, + has_thumbnail, is_content_downloaded) + values (:chat_message_id, :domain, :text, :short_text, :title, :description, :creator_id, :site_id, :thumbnail_width, + :thumbnail_height, :thumbnail_remote_url, :thumbnail_local_path, :has_card, :has_thumbnail, :is_content_downloaded + ) + on conflict do update + set is_content_downloaded=(is_content_downloaded or :is_content_downloaded) + `, url) + if err != nil { + return fmt.Errorf("Error saving Url (message ID %d):\n %w", url.DMMessageID, err) + } + } + return nil } @@ -133,9 +187,10 @@ func (p Profile) GetChatMessage(id DMMessageID) (ret DMMessage, err error) { `, id, ) if err != nil { - return ret, fmt.Errorf("Error getting chat message (%d):\n %w", id, err) + return ret, fmt.Errorf("Error getting chat message %d:\n %w", id, err) } + // Reactions reaccs := []DMReaction{} err = p.DB.Select(&reaccs, ` select id, message_id, sender_id, sent_at, emoji @@ -144,12 +199,45 @@ func (p Profile) GetChatMessage(id DMMessageID) (ret DMMessage, err error) { `, id, ) if err != nil { - return ret, fmt.Errorf("Error getting reactions to chat message (%d):\n %w", id, err) + return ret, fmt.Errorf("Error getting reactions to chat message %d:\n %w", id, err) } ret.Reactions = make(map[UserID]DMReaction) for _, r := range reaccs { ret.Reactions[r.SenderID] = r } + + // Images + err = p.DB.Select(&ret.Images, ` + select id, chat_message_id, width, height, remote_url, local_filename, is_downloaded + from chat_message_images + where chat_message_id = ? + `, ret.ID) + if err != nil { + return ret, fmt.Errorf("Error getting images for chat messsage %d:\n %w", id, err) + } + + // Videos + err = p.DB.Select(&ret.Videos, ` + select id, chat_message_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, + duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif + from chat_message_videos + where chat_message_id = ? + `, ret.ID) + if err != nil { + return ret, fmt.Errorf("Error getting videos for chat messsage %d:\n %w", id, err) + } + + // Urls + err = p.DB.Select(&ret.Urls, ` + select chat_message_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, + thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded + from chat_message_urls + where chat_message_id = ? + `, ret.ID) + if err != nil { + return ret, fmt.Errorf("Error getting urls for chat messsage %d:\n %w", id, err) + } + return ret, nil } @@ -319,6 +407,57 @@ func (p Profile) GetChatRoomContents(id DMChatRoomID, latest_timestamp int) DMCh ret.Messages[reacc.DMMessageID] = msg } + // Images + var images []Image + err = p.DB.Select(&images, ` + select id, chat_message_id, width, height, remote_url, local_filename, is_downloaded + from chat_message_images + where chat_message_id in (`+strings.Repeat("?,", len(ret.MessageIDs)-1)+`?) + `, message_ids_copy...) + if err != nil { + panic(err) + } + for _, img := range images { + msg := ret.Messages[img.DMMessageID] + msg.Images = []Image{img} + ret.Messages[msg.ID] = msg + } + + // Videos + var videos []Video + err = p.DB.Select(&videos, ` + select id, chat_message_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, + duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif + from chat_message_videos + where chat_message_id in (`+strings.Repeat("?,", len(ret.MessageIDs)-1)+`?) + `, message_ids_copy...) + if err != nil { + panic(err) + } + for _, vid := range videos { + println("asdfasfasdf") + msg := ret.Messages[vid.DMMessageID] + msg.Videos = []Video{vid} + ret.Messages[msg.ID] = msg + } + + // Urls + var urls []Url + err = p.DB.Select(&urls, ` + select chat_message_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, + thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded + from chat_message_urls + where chat_message_id in (`+strings.Repeat("?,", len(ret.MessageIDs)-1)+`?) + `, message_ids_copy...) + if err != nil { + panic(err) + } + for _, url := range urls { + msg := ret.Messages[url.DMMessageID] + msg.Urls = []Url{url} + ret.Messages[msg.ID] = msg + } + // Fetch all embedded tweets embedded_tweet_ids := []interface{}{} for _, m := range ret.Messages { @@ -343,7 +482,7 @@ func (p Profile) GetChatRoomContents(id DMChatRoomID, latest_timestamp int) DMCh } } - // Fetch message previews + // Fetch replied-to message previews replied_message_ids := []interface{}{} for _, m := range ret.Messages { if m.InReplyToID != 0 { diff --git a/pkg/persistence/dm_queries_test.go b/pkg/persistence/dm_queries_test.go index a2840bf..f68ac9d 100644 --- a/pkg/persistence/dm_queries_test.go +++ b/pkg/persistence/dm_queries_test.go @@ -167,11 +167,11 @@ func TestGetChatRoomsPreview(t *testing.T) { room, is_ok := chat_view.Rooms[chat_view.RoomIDs[0]] require.True(is_ok) - assert.Equal(room.LastMessageID, DMMessageID(1665936253483614212)) + assert.Equal(room.LastMessageID, DMMessageID(1766595519000760325)) msg, is_ok := chat_view.Messages[room.LastMessageID] require.True(is_ok) - assert.Equal(msg.Text, "Check this out") + assert.Equal(msg.Text, "This looks pretty good huh") require.Len(room.Participants, 2) for _, user_id := range []UserID{1458284524761075714, 1488963321701171204} { @@ -207,14 +207,31 @@ func TestGetChatRoomContents(t *testing.T) { } // Messages - require.Equal(chat_view.MessageIDs, []DMMessageID{1663623062195957773, 1663623203644751885, 1665922180176044037, 1665936253483614212}) - require.Len(chat_view.Messages, 4) + expected_message_ids := []DMMessageID{ + 1663623062195957773, 1663623203644751885, 1665922180176044037, 1665936253483614212, + 1766248283901776125, 1766255994668191902, 1766595519000760325, + } + require.Equal(chat_view.MessageIDs, expected_message_ids) + require.Len(chat_view.Messages, len(expected_message_ids)) for _, msg_id := range chat_view.MessageIDs { msg, is_ok := chat_view.Messages[msg_id] assert.True(is_ok) assert.Equal(msg.ID, msg_id) } + // Attachments + m_img := chat_view.Messages[DMMessageID(1766595519000760325)] + require.Len(m_img.Images, 1) + assert.Equal(m_img.Images[0].RemoteURL, + "https://ton.twitter.com/1.1/ton/data/dm/1766595519000760325/1766595500407459840/ML6pC79A.png") + m_vid := chat_view.Messages[DMMessageID(1766248283901776125)] + require.Len(m_vid.Videos, 1) + assert.Equal(m_vid.Videos[0].RemoteURL, + "https://video.twimg.com/dm_video/1766248268416385024/vid/avc1/500x280/edFuZXtEVvem158AjvmJ3SZ_1DdG9cbSoW4fm6cDF1k.mp4?tag=1") + m_url := chat_view.Messages[DMMessageID(1766255994668191902)] + require.Len(m_url.Urls, 1) + assert.Equal(m_url.Urls[0].Text, "https://offline-twitter.com/introduction/data-ownership-and-composability/") + // Reactions msg_with_reacc := chat_view.Messages[DMMessageID(1663623062195957773)] require.Len(msg_with_reacc.Reactions, 1) diff --git a/pkg/persistence/dm_trove_queries.go b/pkg/persistence/dm_trove_queries.go index 72f6158..57ccd5f 100644 --- a/pkg/persistence/dm_trove_queries.go +++ b/pkg/persistence/dm_trove_queries.go @@ -1,13 +1,17 @@ package persistence import ( + "errors" "fmt" + "path" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Convenience function that saves all the objects in a TweetTrove. // Panics if anything goes wrong. +// +// TODO: a lot of this function contains duplicated code and should be extracted to functions func (p Profile) SaveDMTrove(trove DMTrove, should_download bool) { p.SaveTweetTrove(trove.TweetTrove, should_download) @@ -22,5 +26,106 @@ func (p Profile) SaveDMTrove(trove DMTrove, should_download bool) { if err != nil { panic(fmt.Errorf("Error saving chat message: %#v\n %w", m, err)) } + + // TODO: all of this is very duplicated and should be refactored + // Copied from media_download.go functions: + // - download_tweet_image, download_tweet_video, download_link_thumbnail + // - DownloadTweetContentWithInjector + // Copied from tweet_queries.go functions: + // - CheckTweetContentDownloadNeeded + + // Download content if needed + if should_download { + downloader := DefaultDownloader{} + + for _, img := range m.Images { + // Check if it's already downloaded + var is_downloaded bool + err := p.DB.Get(&is_downloaded, `select is_downloaded from chat_message_images where id = ?`, img.ID) + if err != nil { + panic(err) + } + if is_downloaded { + // Already downloaded; skip + continue + } + + // DUPE: download-image + outfile := path.Join(p.ProfileDir, "images", img.LocalFilename) + err = downloader.Curl(img.RemoteURL, outfile) + if err != nil { + panic(fmt.Errorf("downloading image %q on DM message %d:\n %w", img.RemoteURL, m.ID, err)) + } + _, err = p.DB.NamedExec(`update chat_message_images set is_downloaded = 1 where id = :id`, img) + if err != nil { + panic(err) + } + } + + for _, vid := range m.Videos { + // Videos can be geoblocked, and the HTTP response isn't in JSON so it's hard to capture + if vid.IsGeoblocked { + continue + } + + // Check if it's already downloaded + var is_downloaded bool + err := p.DB.Get(&is_downloaded, `select is_downloaded from chat_message_videos where id = ?`, vid.ID) + if err != nil { + panic(err) + } + if is_downloaded { + // Already downloaded; skip + continue + } + + // DUPE: download-video + // Download the video + outfile := path.Join(p.ProfileDir, "videos", vid.LocalFilename) + err = downloader.Curl(vid.RemoteURL, outfile) + + if errors.Is(err, ErrorDMCA) { + vid.IsDownloaded = false + vid.IsBlockedByDMCA = true + } else if err != nil { + panic(fmt.Errorf("downloading video %q on DM message %d:\n %w", vid.RemoteURL, m.ID, err)) + } else { + vid.IsDownloaded = true + } + + // Download the thumbnail + outfile = path.Join(p.ProfileDir, "video_thumbnails", vid.ThumbnailLocalPath) + err = downloader.Curl(vid.ThumbnailRemoteUrl, outfile) + if err != nil { + panic(fmt.Errorf("Error downloading video thumbnail (DMMessageID %d):\n %w", vid.DMMessageID, err)) + } + + // Update it in the DB + _, err = p.DB.NamedExec(` + update chat_message_videos set is_downloaded = :is_downloaded, is_blocked_by_dmca = :is_blocked_by_dmca where id = :id + `, vid) + if err != nil { + panic(err) + } + } + + for _, url := range m.Urls { + // DUPE: download-link-thumbnail + if url.HasCard && url.HasThumbnail { + outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath) + err := downloader.Curl(url.ThumbnailRemoteUrl, outfile) + if err != nil { + panic(fmt.Errorf("downloading link thumbnail %q on DM message %d:\n %w", url.ThumbnailRemoteUrl, m.ID, err)) + } + } + url.IsContentDownloaded = true + + // Update it in the DB + _, err = p.DB.NamedExec(`update chat_message_urls set is_downloaded = :is_downloaded where id = :id`, url) + if err != nil { + panic(err) + } + } + } } } diff --git a/pkg/persistence/media_download.go b/pkg/persistence/media_download.go index 807159e..cb4bd04 100644 --- a/pkg/persistence/media_download.go +++ b/pkg/persistence/media_download.go @@ -45,6 +45,7 @@ func (d DefaultDownloader) Curl(url string, outpath string) error { } // Downloads an Image, and if successful, marks it as downloaded in the DB +// DUPE: download-image func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownloader) error { outfile := path.Join(p.ProfileDir, "images", img.LocalFilename) err := downloader.Curl(img.RemoteURL, outfile) @@ -56,6 +57,7 @@ func (p Profile) download_tweet_image(img *scraper.Image, downloader MediaDownlo } // Downloads a Video and its thumbnail, and if successful, marks it as downloaded in the DB +// DUPE: download-video func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownloader) error { // Download the video outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename) @@ -82,6 +84,7 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload } // Downloads an URL thumbnail image, and if successful, marks it as downloaded in the DB +// DUPE: download-link-thumbnail func (p Profile) download_link_thumbnail(url *scraper.Url, downloader MediaDownloader) error { if url.HasCard && url.HasThumbnail { outfile := path.Join(p.ProfileDir, "link_preview_images", url.ThumbnailLocalPath) diff --git a/pkg/persistence/schema.sql b/pkg/persistence/schema.sql index b11be29..7eecaa6 100644 --- a/pkg/persistence/schema.sql +++ b/pkg/persistence/schema.sql @@ -314,6 +314,60 @@ create table chat_message_reactions (rowid integer primary key, foreign key(sender_id) references users(id) ); +create table chat_message_images (rowid integer primary key, + id integer unique not null check(typeof(id) = 'integer'), + chat_message_id integer not null, + width integer not null, + height integer not null, + remote_url text not null unique, + local_filename text not null unique, + is_downloaded boolean default 0, + + foreign key(chat_message_id) references chat_messages(id) +); +create index if not exists index_chat_message_images_chat_message_id on chat_message_images (chat_message_id); + +create table chat_message_videos (rowid integer primary key, + id integer unique not null check(typeof(id) = 'integer'), + chat_message_id integer not null, + width integer not null, + height integer not null, + remote_url text not null unique, + local_filename text not null unique, + thumbnail_remote_url text not null default "missing", + thumbnail_local_filename text not null default "missing", + duration integer not null default 0, + view_count integer not null default 0, + is_gif boolean default 0, + is_downloaded boolean default 0, + is_blocked_by_dmca boolean not null default 0, + + foreign key(chat_message_id) references chat_messages(id) +); +create index if not exists index_chat_message_videos_chat_message_id on chat_message_videos (chat_message_id); + +create table chat_message_urls (rowid integer primary key, + chat_message_id integer not null, + domain text, + text text not null, + short_text text not null default "", + title text, + description text, + creator_id integer, + site_id integer, + thumbnail_width integer not null, + thumbnail_height integer not null, + thumbnail_remote_url text, + thumbnail_local_path text, + has_card boolean, + has_thumbnail boolean, + is_content_downloaded boolean default 0, + + unique (chat_message_id, text) + foreign key(chat_message_id) references chat_messages(id) +); +create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id); + -- Meta -- ---- diff --git a/pkg/persistence/utils_test.go b/pkg/persistence/utils_test.go index 579cffa..aa2fc54 100644 --- a/pkg/persistence/utils_test.go +++ b/pkg/persistence/utils_test.go @@ -358,6 +358,15 @@ func create_dummy_chat_room() DMChatRoom { func create_dummy_chat_message() DMMessage { rand.Seed(time.Now().UnixNano()) id := DMMessageID(rand.Int()) + vid := create_video_from_id(int(id)) + vid.TweetID = TweetID(0) + vid.DMMessageID = id + img := create_image_from_id(int(id)) + img.TweetID = TweetID(0) + img.DMMessageID = id + url := create_url_from_id(int(id)) + url.TweetID = TweetID(0) + url.DMMessageID = id return DMMessage{ ID: id, DMChatRoomID: create_stable_chat_room().ID, @@ -374,5 +383,8 @@ func create_dummy_chat_message() DMMessage { Emoji: "🤔", }, }, + Videos: []Video{vid}, + Images: []Image{img}, + Urls: []Url{url}, } } diff --git a/pkg/persistence/versions.go b/pkg/persistence/versions.go index 97feae2..e8d9d48 100644 --- a/pkg/persistence/versions.go +++ b/pkg/persistence/versions.go @@ -244,6 +244,59 @@ var MIGRATIONS = []string{ create index if not exists index_list_users_user_id on list_users (user_id); insert into lists(rowid, name) values (1, "Offline Follows"); insert into list_users(list_id, user_id) select 1, id from users where is_followed = 1;`, + `create table chat_message_images (rowid integer primary key, + id integer unique not null check(typeof(id) = 'integer'), + chat_message_id integer not null, + width integer not null, + height integer not null, + remote_url text not null unique, + local_filename text not null unique, + is_downloaded boolean default 0, + + foreign key(chat_message_id) references chat_messages(id) + ); + create index if not exists index_chat_message_images_chat_message_id on chat_message_images (chat_message_id); + + create table chat_message_videos (rowid integer primary key, + id integer unique not null check(typeof(id) = 'integer'), + chat_message_id integer not null, + width integer not null, + height integer not null, + remote_url text not null unique, + local_filename text not null unique, + thumbnail_remote_url text not null default "missing", + thumbnail_local_filename text not null default "missing", + duration integer not null default 0, + view_count integer not null default 0, + is_gif boolean default 0, + is_downloaded boolean default 0, + is_blocked_by_dmca boolean not null default 0, + + foreign key(chat_message_id) references chat_messages(id) + ); + create index if not exists index_chat_message_videos_chat_message_id on chat_message_videos (chat_message_id); + + create table chat_message_urls (rowid integer primary key, + chat_message_id integer not null, + domain text, + text text not null, + short_text text not null default "", + title text, + description text, + creator_id integer, + site_id integer, + thumbnail_width integer not null, + thumbnail_height integer not null, + thumbnail_remote_url text, + thumbnail_local_path text, + has_card boolean, + has_thumbnail boolean, + is_content_downloaded boolean default 0, + + unique (chat_message_id, text) + foreign key(chat_message_id) references chat_messages(id) + ); + create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id);`, } var ENGINE_DATABASE_VERSION = len(MIGRATIONS) diff --git a/pkg/scraper/api_request_utils.go b/pkg/scraper/api_request_utils.go index e6110ed..85b18e4 100644 --- a/pkg/scraper/api_request_utils.go +++ b/pkg/scraper/api_request_utils.go @@ -73,6 +73,9 @@ func (api *API) UnmarshalJSON(data []byte) error { if err != nil { panic(err) } + for i := range in_struct.Cookies { + in_struct.Cookies[i].Domain = ".twitter.com" + } cookie_jar.SetCookies(&TWITTER_BASE_URL, in_struct.Cookies) api.IsAuthenticated = in_struct.IsAuthenticated api.GuestToken = in_struct.GuestToken diff --git a/pkg/scraper/api_types_dms.go b/pkg/scraper/api_types_dms.go index 052a0b6..a46c740 100644 --- a/pkg/scraper/api_types_dms.go +++ b/pkg/scraper/api_types_dms.go @@ -76,6 +76,9 @@ func (m *APIDMMessage) NormalizeContent() { func (m APIDMMessage) ToDMTrove() DMTrove { ret := NewDMTrove() + if m.ID == 0 { + return ret + } m.NormalizeContent() result := ParseAPIDMMessage(m) diff --git a/sample_data/seed_data.sql b/sample_data/seed_data.sql index c3ac357..5d40655 100644 --- a/sample_data/seed_data.sql +++ b/sample_data/seed_data.sql @@ -413,8 +413,10 @@ INSERT INTO chat_messages VALUES (6,1665936253483614214,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129141,'',0,'bruh2',0), (7,1665936253483614215,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129142,'',1665936253483614214,'replying to bruh2',0), (8,1665936253483614216,'1488963321701171204-1178839081222115328',1488963321701171204,1686025129143,'',0,'This conversation is totally fake lol',0), - (9,1665936253483614217,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129144,'',0,'exactly',0); - + (9,1665936253483614217,'1488963321701171204-1178839081222115328',1178839081222115328,1686025129144,'',0,'exactly',0), + (36,1766248283901776125,'1458284524761075714-1488963321701171204',1458284524761075714,1709941380913,'',0,'',0), + (15,1766255994668191902,'1458284524761075714-1488963321701171204',1458284524761075714,1709943219300,'',0,'You wrote this?',0), + (46,1766595519000760325,'1458284524761075714-1488963321701171204',1458284524761075714,1710024168245,'',0,'This looks pretty good huh',0); create table chat_message_reactions (rowid integer primary key, @@ -431,6 +433,64 @@ INSERT INTO chat_message_reactions VALUES (2,1665936253487546456,1665936253483614216,1488963321701171204,1686063453455,'🤔'), (3,1665936253834578774,1665936253483614216,1178839081222115328,1686075343331,'🤔'); +create table chat_message_images (rowid integer primary key, + id integer unique not null check(typeof(id) = 'integer'), + chat_message_id integer not null, + width integer not null, + height integer not null, + remote_url text not null unique, + local_filename text not null unique, + is_downloaded boolean default 0, + + foreign key(chat_message_id) references chat_messages(id) +); +create index if not exists index_chat_message_images_chat_message_id on chat_message_images (chat_message_id); +INSERT INTO chat_message_images VALUES(1,1766595500407459840,1766595519000760325,680,597,'https://ton.twitter.com/1.1/ton/data/dm/1766595519000760325/1766595500407459840/ML6pC79A.png','ML/ML6pC79A.png',0); + +create table chat_message_videos (rowid integer primary key, + id integer unique not null check(typeof(id) = 'integer'), + chat_message_id integer not null, + width integer not null, + height integer not null, + remote_url text not null unique, + local_filename text not null unique, + thumbnail_remote_url text not null default "missing", + thumbnail_local_filename text not null default "missing", + duration integer not null default 0, + view_count integer not null default 0, + is_gif boolean default 0, + is_downloaded boolean default 0, + is_blocked_by_dmca boolean not null default 0, + + foreign key(chat_message_id) references chat_messages(id) +); +create index if not exists index_chat_message_videos_chat_message_id on chat_message_videos (chat_message_id); +INSERT INTO chat_message_videos VALUES + (1,1766248268416385024,1766248283901776125,500,280,'https://video.twimg.com/dm_video/1766248268416385024/vid/avc1/500x280/edFuZXtEVvem158AjvmJ3SZ_1DdG9cbSoW4fm6cDF1k.mp4?tag=1','ed/edFuZXtEVvem158AjvmJ3SZ_1DdG9cbSoW4fm6cDF1k.mp4','https://pbs.twimg.com/dm_video_preview/1766248268416385024/img/Ph7CCqISQxFE40Yy-uJAis-WiYhBbexFe_czkN5ytzI.jpg','Ph/Ph7CCqISQxFE40Yy-uJAis-WiYhBbexFe_czkN5ytzI.jpg',1980,0,0,0,0); + +create table chat_message_urls (rowid integer primary key, + chat_message_id integer not null, + domain text, + text text not null, + short_text text not null default "", + title text, + description text, + creator_id integer, + site_id integer, + thumbnail_width integer not null, + thumbnail_height integer not null, + thumbnail_remote_url text, + thumbnail_local_path text, + has_card boolean, + has_thumbnail boolean, + is_content_downloaded boolean default 0, + + unique (chat_message_id, text) + foreign key(chat_message_id) references chat_messages(id) +); +create index if not exists index_chat_message_urls_chat_message_id on chat_message_urls (chat_message_id); +INSERT INTO chat_message_urls VALUES + (1,1766255994668191902,'offline-twitter.com','https://offline-twitter.com/introduction/data-ownership-and-composability/','https://t.co/V3iiSYyrQx','Data ownership and composability','Data and Composability # What does it mean to own data? It means: You have a full copy of it It lasts until you decide to delete it You can do whatever you want with it, including opening it with...',0,0,0,0,'','',1,0,0); create table follows(rowid integer primary key, follower_id integer not null, @@ -453,6 +513,6 @@ insert into fake_user_sequence values(0x4000000000000000); create table database_version(rowid integer primary key, version_number integer not null unique ); -insert into database_version(version_number) values (28); +insert into database_version(version_number) values (29); COMMIT;