diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 62dc584..54dbd23 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -603,6 +603,12 @@ func get_notifications(how_many int) { if err != nil && !errors.Is(err, scraper.END_OF_FEED) { panic(err) } + to_scrape := profile.CheckNotificationScrapesNeeded(trove) + trove, err = api.GetNotificationDetailForAll(trove, to_scrape) + if err != nil { + panic(err) + } + profile.SaveTweetTrove(trove, true, &api) happy_exit(fmt.Sprintf("Saved %d notifications, %d tweets and %d users", len(trove.Notifications), len(trove.Tweets), len(trove.Users), diff --git a/pkg/persistence/notification_queries.go b/pkg/persistence/notification_queries.go index cea6335..ea14c2f 100644 --- a/pkg/persistence/notification_queries.go +++ b/pkg/persistence/notification_queries.go @@ -1,6 +1,9 @@ package persistence import ( + "database/sql" + "errors" + "fmt" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) @@ -12,16 +15,20 @@ func (p Profile) SaveNotification(n Notification) { // Save the Notification _, err = tx.NamedExec(` - insert into notifications(id, type, sent_at, sort_index, user_id, action_user_id, action_tweet_id, action_retweet_id) + insert into notifications(id, type, sent_at, sort_index, user_id, action_user_id, action_tweet_id, action_retweet_id, + has_detail, last_scraped_at) values (:id, :type, :sent_at, :sort_index, :user_id, nullif(:action_user_id, 0), nullif(:action_tweet_id, 0), - nullif(:action_retweet_id, 0)) + nullif(:action_retweet_id, 0), :has_detail, :last_scraped_at) on conflict do update set sent_at = max(sent_at, :sent_at), sort_index = max(sort_index, :sort_index), action_user_id = nullif(:action_user_id, 0), - action_tweet_id = nullif(:action_tweet_id, 0) + action_tweet_id = nullif(:action_tweet_id, 0), + has_detail = has_detail or :has_detail, + last_scraped_at = max(last_scraped_at, :last_scraped_at) `, n) if err != nil { + fmt.Printf("failed to save notification %#v\n", n) panic(err) } @@ -62,7 +69,7 @@ func (p Profile) GetNotification(id NotificationID) Notification { var ret Notification err := p.DB.Get(&ret, `select id, type, sent_at, sort_index, user_id, ifnull(action_user_id, 0) action_user_id, - ifnull(action_tweet_id, 0) action_tweet_id, ifnull(action_retweet_id, 0) action_retweet_id + ifnull(action_tweet_id, 0) action_tweet_id, ifnull(action_retweet_id, 0) action_retweet_id, has_detail, last_scraped_at from notifications where id = ?`, id) if err != nil { @@ -82,3 +89,29 @@ func (p Profile) GetNotification(id NotificationID) Notification { } return ret } + +func (p Profile) CheckNotificationScrapesNeeded(trove TweetTrove) []NotificationID { + ret := []NotificationID{} + for n_id, notification := range trove.Notifications { + // If there's no detail page, skip + if !notification.HasDetail { + continue + } + + // Check its last-scraped + var last_scraped_at Timestamp + err := p.DB.Get(&last_scraped_at, `select last_scraped_at from notifications where id = ?`, n_id) + if errors.Is(err, sql.ErrNoRows) { + // It's not scraped at all yet + ret = append(ret, n_id) + continue + } else if err != nil { + panic(err) + } + // If the latest scrape is not fresh (older than the notification sent-at time), add it + if last_scraped_at.Time.Before(notification.SentAt.Time) { + ret = append(ret, n_id) + } + } + return ret +} diff --git a/pkg/persistence/schema.sql b/pkg/persistence/schema.sql index 3d1d278..cb21e19 100644 --- a/pkg/persistence/schema.sql +++ b/pkg/persistence/schema.sql @@ -414,6 +414,9 @@ create table notifications (rowid integer primary key, action_tweet_id integer references tweets(id), -- tweet associated with the notification action_retweet_id integer references retweets(retweet_id), + has_detail boolean not null default 0, + last_scraped_at not null default 0, + foreign key(type) references notification_types(rowid) foreign key(user_id) references users(id) ); diff --git a/pkg/persistence/utils_test.go b/pkg/persistence/utils_test.go index 884af90..cd8fc7a 100644 --- a/pkg/persistence/utils_test.go +++ b/pkg/persistence/utils_test.go @@ -402,16 +402,18 @@ func create_dummy_notification() Notification { id := NotificationID(fmt.Sprintf("Notification #%d", rand.Int())) return Notification{ - ID: id, - Type: NOTIFICATION_TYPE_REPLY, - SentAt: TimestampFromUnix(10000), - SortIndex: rand.Int63(), - UserID: create_stable_user().ID, - ActionUserID: create_stable_user().ID, - ActionTweetID: create_stable_tweet().ID, + ID: id, + Type: NOTIFICATION_TYPE_REPLY, + SentAt: TimestampFromUnix(10000), + SortIndex: rand.Int63(), + UserID: create_stable_user().ID, + ActionUserID: create_stable_user().ID, + ActionTweetID: create_stable_tweet().ID, ActionRetweetID: create_stable_retweet().RetweetID, - TweetIDs: []TweetID{create_stable_tweet().ID}, - UserIDs: []UserID{create_stable_user().ID}, - RetweetIDs: []TweetID{create_stable_retweet().RetweetID}, + HasDetail: true, + LastScrapedAt: TimestampFromUnix(57234728), + TweetIDs: []TweetID{create_stable_tweet().ID}, + UserIDs: []UserID{create_stable_user().ID}, + RetweetIDs: []TweetID{create_stable_retweet().RetweetID}, } } diff --git a/pkg/persistence/versions.go b/pkg/persistence/versions.go index 12353c9..1620f68 100644 --- a/pkg/persistence/versions.go +++ b/pkg/persistence/versions.go @@ -333,6 +333,9 @@ var MIGRATIONS = []string{ action_tweet_id integer references tweets(id), -- tweet associated with the notification action_retweet_id integer references retweets(retweet_id), + has_detail boolean not null default 0, + last_scraped_at not null default 0, + foreign key(type) references notification_types(rowid) foreign key(user_id) references users(id) ); diff --git a/pkg/scraper/api_types_notifications.go b/pkg/scraper/api_types_notifications.go index 15d6b89..57c573a 100644 --- a/pkg/scraper/api_types_notifications.go +++ b/pkg/scraper/api_types_notifications.go @@ -2,10 +2,12 @@ package scraper import ( "errors" + "fmt" "net/url" "regexp" "sort" "strings" + "time" log "github.com/sirupsen/logrus" ) @@ -56,6 +58,41 @@ func (api *API) GetNotifications(how_many int) (TweetTrove, error) { } trove.MergeWith(new_trove) } + + return trove, nil +} + +func (api *API) GetNotificationDetailForAll(trove TweetTrove, to_scrape []NotificationID) (TweetTrove, error) { + for _, n_id := range to_scrape { + notification := trove.Notifications[n_id] + resp, err := api.GetNotificationDetail(notification) + if errors.Is(err, ErrRateLimited) { + log.Warnf("Rate limited!") + break + } else if err != nil { + return TweetTrove{}, err + } + + // Fetch the notification detail + new_trove, ids, err := resp.ToTweetTroveAsNotificationDetail() + if err != nil { + panic(err) + } + trove.MergeWith(new_trove) + + // Add the fetched Tweet / Retweet IDs to the notification + for _, id := range ids { + _, is_retweet := trove.Retweets[id] + if is_retweet { + notification.RetweetIDs = append(notification.RetweetIDs, id) + } else { + notification.TweetIDs = append(notification.TweetIDs, id) + } + } + // Update the notification's last_scraped_at + notification.LastScrapedAt = Timestamp{time.Now()} + trove.Notifications[n_id] = notification + } return trove, nil } @@ -91,6 +128,17 @@ func (t *TweetResponse) ToTweetTroveAsNotifications(current_user_id UserID) (Twe notification.Type = NOTIFICATION_TYPE_QUOTE_TWEET } else if strings.Contains(entry.Content.Item.ClientEventInfo.Element, "mentioned") { notification.Type = NOTIFICATION_TYPE_MENTION + } else if strings.Contains(entry.Content.Item.ClientEventInfo.Element, "live_broadcast") { + // TODO: broadcast + notification.Type = NOTIFICATION_TYPE_USER_IS_LIVE + } else if strings.Contains(entry.Content.Item.ClientEventInfo.Element, "community_tweet_pinned") { + // TODO: communities + delete(ret.Notifications, notification.ID) + continue + } + + if strings.Contains(entry.Content.Item.ClientEventInfo.Element, "multiple") { + notification.HasDetail = true } if entry.Content.Item.Content.Tweet.ID != 0 { @@ -161,3 +209,39 @@ func ParseSingleNotification(n APINotification) Notification { return ret } + +func (api *API) GetNotificationDetail(n Notification) (TweetResponse, error) { + url, err := url.Parse(fmt.Sprintf("https://twitter.com/i/api/2/notifications/view/%s.json", n.ID)) + if err != nil { + panic(err) + } + + query := url.Query() + add_tweet_query_params(&query) + url.RawQuery = query.Encode() + + var result TweetResponse + err = api.do_http(url.String(), "", &result) + + return result, err +} + +func (t *TweetResponse) ToTweetTroveAsNotificationDetail() (TweetTrove, []TweetID, error) { + ids := []TweetID{} + ret, err := t.ToTweetTrove() + if err != nil { + return TweetTrove{}, ids, err + } + + // Find the "addEntries" instruction + for _, instr := range t.Timeline.Instructions { + sort.Sort(instr.AddEntries.Entries) + for _, entry := range instr.AddEntries.Entries { + if entry.Content.Item.Content.Tweet.ID != 0 { + ids = append(ids, TweetID(entry.Content.Item.Content.Tweet.ID)) + } + } + } + + return ret, ids, nil +} diff --git a/pkg/scraper/api_types_notifications_test.go b/pkg/scraper/api_types_notifications_test.go index f71f30d..33093e7 100644 --- a/pkg/scraper/api_types_notifications_test.go +++ b/pkg/scraper/api_types_notifications_test.go @@ -119,6 +119,21 @@ func TestParseNotificationsPage(t *testing.T) { assert.Len(notif10.RetweetIDs, 1) assert.Contains(notif10.RetweetIDs, TweetID(1827183097382654351)) + notif11, is_ok := tweet_trove.Notifications["FDzeDIfVUAIAAAABiJONco_yJRHyMqRjxDY"] + assert.True(is_ok) + assert.Equal(NOTIFICATION_TYPE_USER_IS_LIVE, notif11.Type) + assert.Equal(UserID(277536867), notif11.ActionUserID) + + // 1 user liked multiple posts + notif12, is_ok := tweet_trove.Notifications["FDzeDIfVUAIAAAABiJONco_yJRESfwtSqvg"] + assert.True(is_ok) + assert.True(notif12.HasDetail) + + // TODO: communities + // notif12, is_ok := tweet_trove.Notifications["FDzeDIfVUAIAAAABiJONco_yJRHPBNsDH88"] + // assert.True(is_ok) + // assert.Equal(NOTIFICATION_TYPE_COMMUNITY_PINNED_POST, notif12.Type) + // Check users for _, u_id := range []UserID{1458284524761075714, 28815778, 1633158398555353096} { _, is_ok := tweet_trove.Users[u_id] @@ -155,3 +170,25 @@ func TestParseNotificationsEndOfFeed(t *testing.T) { assert.True(resp.IsEndOfFeed()) } + +func TestParseNotificationDetail(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + data, err := os.ReadFile("test_responses/notifications/notification_detail.json") + require.NoError(err) + + var resp TweetResponse + err = json.Unmarshal(data, &resp) + require.NoError(err) + + trove, ids, err := resp.ToTweetTroveAsNotificationDetail() + require.NoError(err) + assert.Len(ids, 2) + assert.Contains(ids, TweetID(1827544032714633628)) + assert.Contains(ids, TweetID(1826743131108487390)) + + _, is_ok := trove.Tweets[1826743131108487390] + assert.True(is_ok) + _, is_ok = trove.Retweets[1827544032714633628] + assert.True(is_ok) +} diff --git a/pkg/scraper/notification.go b/pkg/scraper/notification.go index 526fef8..640f6b7 100644 --- a/pkg/scraper/notification.go +++ b/pkg/scraper/notification.go @@ -52,6 +52,10 @@ type Notification struct { ActionTweetID TweetID `db:"action_tweet_id"` ActionRetweetID TweetID `db:"action_retweet_id"` + // Used for "multiple" notifs, like "user liked multiple tweets" + HasDetail bool `db:"has_detail"` + LastScrapedAt Timestamp `db:"last_scraped_at"` + TweetIDs []TweetID UserIDs []UserID RetweetIDs []TweetID diff --git a/pkg/scraper/test_responses/notifications/notification_detail.json b/pkg/scraper/test_responses/notifications/notification_detail.json new file mode 100644 index 0000000..b2a0fb6 --- /dev/null +++ b/pkg/scraper/test_responses/notifications/notification_detail.json @@ -0,0 +1 @@ +{"globalObjects":{"users":{"1059028569655980032":{"id":1059028569655980032,"id_str":"1059028569655980032","name":"molinari","screen_name":"r_tZero19e","location":null,"description":"special forces operative trained to resist waterboarding","url":"https://t.co/E4ar1A2YZX","entities":{"url":{"urls":[{"url":"https://t.co/E4ar1A2YZX","expanded_url":"https://buttondown.email/tZero19e/","display_url":"buttondown.email/tZero19e/","indices":[0,23]}]},"description":{"urls":[]}},"protected":false,"followers_count":148,"friends_count":1970,"listed_count":2,"created_at":"Sun Nov 04 10:24:21 +0000 2018","favourites_count":4976,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":3888,"lang":null,"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http://pbs.twimg.com/profile_images/1816572161651261442/mleRm1H__normal.jpg","profile_image_url_https":"https://pbs.twimg.com/profile_images/1816572161651261442/mleRm1H__normal.jpg","profile_banner_url":"https://pbs.twimg.com/profile_banners/1059028569655980032/1624033304","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":false,"follow_request_sent":null,"notifications":null,"blocking":false,"blocked_by":false,"want_retweets":false,"profile_interstitial_type":"","translator_type":"none","withheld_in_countries":[],"followed_by":true,"ext_is_blue_verified":false,"ext_highlighted_label":{}},"1244082921683763201":{"id":1244082921683763201,"id_str":"1244082921683763201","name":"Seldon","screen_name":"seld_on","location":null,"description":"Quant. Interests: classical hist/lit, mad science, shitposting. Amphetamine nationalist. Pronouns are ๐’‚—/๐’ˆ—/๐“‡‹๐“˜๐“€›. Likes are mostly bookmarks or highlights.","url":null,"entities":{"description":{"urls":[]}},"protected":false,"followers_count":3276,"friends_count":472,"listed_count":14,"created_at":"Sun Mar 29 02:04:23 +0000 2020","favourites_count":91187,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":15107,"lang":null,"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http://pbs.twimg.com/profile_images/1376269223899594753/sqz0oXtt_normal.jpg","profile_image_url_https":"https://pbs.twimg.com/profile_images/1376269223899594753/sqz0oXtt_normal.jpg","profile_banner_url":"https://pbs.twimg.com/profile_banners/1244082921683763201/1647340344","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":false,"follow_request_sent":null,"notifications":null,"blocking":false,"blocked_by":false,"want_retweets":false,"profile_interstitial_type":"","translator_type":"none","withheld_in_countries":[],"followed_by":false,"ext_is_blue_verified":true,"ext_highlighted_label":{}},"1349373539858538497":{"id":1349373539858538497,"id_str":"1349373539858538497","name":"Ashwin @ KDD 2024","screen_name":"_ashwxn","location":"Barcelona, Spain","description":"MSc @UPFBarcelona ๐ŸŽ“ Research @justiciacat โš–๏ธ RecSys @headout โœˆ๏ธ DEI @QueerInAI ๐Ÿณ๏ธโ€๐ŸŒˆ๐Ÿณ๏ธโ€โšง๏ธ Algorithmic Fairness & Ethics | they/them","url":"https://t.co/wz2Qf2XkCM","entities":{"url":{"urls":[{"url":"https://t.co/wz2Qf2XkCM","expanded_url":"https://ashwin-19.github.io/","display_url":"ashwin-19.github.io","indices":[0,23]}]},"description":{"urls":[]}},"protected":false,"followers_count":684,"friends_count":496,"listed_count":5,"created_at":"Wed Jan 13 15:11:55 +0000 2021","favourites_count":5253,"utc_offset":null,"time_zone":null,"geo_enabled":true,"verified":false,"statuses_count":613,"lang":null,"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http://pbs.twimg.com/profile_images/1720134613983436800/NrtrH3-R_normal.jpg","profile_image_url_https":"https://pbs.twimg.com/profile_images/1720134613983436800/NrtrH3-R_normal.jpg","profile_banner_url":"https://pbs.twimg.com/profile_banners/1349373539858538497/1697909173","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":false,"follow_request_sent":null,"notifications":null,"blocking":false,"blocked_by":false,"want_retweets":false,"profile_interstitial_type":"","translator_type":"none","withheld_in_countries":[],"followed_by":false,"ext_is_blue_verified":false,"ext_highlighted_label":{}},"1458284524761075714":{"id":1458284524761075714,"id_str":"1458284524761075714","name":"wispem-wantex","screen_name":"wispem_wantex","location":"on my computer","description":"rightwing bodybuilder ยท composability guru ยท display: flex","url":"https://t.co/7nDTwkyzRJ","entities":{"url":{"urls":[{"url":"https://t.co/7nDTwkyzRJ","expanded_url":"https://offline-twitter.com/","display_url":"offline-twitter.com","indices":[0,23]}]},"description":{"urls":[]}},"protected":true,"followers_count":755,"friends_count":197,"listed_count":11,"created_at":"Wed Nov 10 04:05:16 +0000 2021","favourites_count":26445,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":9804,"lang":null,"contributors_enabled":false,"is_translator":false,"is_translation_enabled":false,"profile_background_color":"F5F8FA","profile_background_image_url":null,"profile_background_image_url_https":null,"profile_background_tile":false,"profile_image_url":"http://pbs.twimg.com/profile_images/1724933823144620032/sYTzWQy2_normal.jpg","profile_image_url_https":"https://pbs.twimg.com/profile_images/1724933823144620032/sYTzWQy2_normal.jpg","profile_link_color":"1DA1F2","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null,"blocking":null,"translator_type":"none","withheld_in_countries":[],"ext_is_blue_verified":false}},"tweets":{"1826743131108487390":{"created_at":"Thu Aug 22 22:07:56 +0000 2024","id":1826743131108487390,"id_str":"1826743131108487390","full_text":"All of \"decentralization tech\" sucks, it's an attempt to use technology to fix political problems.\n\n99% of \"crypto\" would be unnecessary if the government wasn't evil, that's literally the only reason any of it exists","truncated":false,"display_text_range":[0,217],"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[]},"source":"Twitter Web App","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user_id":1458284524761075714,"user_id_str":"1458284524761075714","geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":true,"quoted_status_id":1826733541239210361,"quoted_status_id_str":"1826733541239210361","retweet_count":2,"favorite_count":22,"reply_count":4,"quote_count":0,"conversation_id":1826743131108487390,"conversation_id_str":"1826743131108487390","conversation_muted":false,"favorited":false,"retweeted":false,"lang":"en","self_thread":{"id":1826743131108487390,"id_str":"1826743131108487390"},"ext":{"superFollowMetadata":{"r":{"ok":{}},"ttl":-1}}},"1827544032714633628":{"created_at":"Sun Aug 25 03:10:26 +0000 2024","id":1827544032714633628,"id_str":"1827544032714633628","full_text":"RT @seld_on: This is just a high-IQ analogue to open defecation. The general phenomenon, common to pretty much all of the 3rd world, is a cโ€ฆ","truncated":false,"display_text_range":[0,140],"entities":{"hashtags":[],"symbols":[],"user_mentions":[{"screen_name":"seld_on","name":"Seldon","id":1244082921683763201,"id_str":"1244082921683763201","indices":[3,11]}],"urls":[]},"source":"Twitter Web App","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user_id":1458284524761075714,"user_id_str":"1458284524761075714","geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status_id":1827480507187159332,"retweeted_status_id_str":"1827480507187159332","is_quote_status":true,"quoted_status_id":1827107496349032940,"quoted_status_id_str":"1827107496349032940","retweet_count":0,"favorite_count":0,"reply_count":0,"quote_count":0,"conversation_id":1827544032714633628,"conversation_id_str":"1827544032714633628","favorited":false,"retweeted":false,"lang":"en","ext":{"superFollowMetadata":{"r":{"ok":{}},"ttl":-1}}},"1827107496349032940":{"created_at":"Fri Aug 23 22:15:47 +0000 2024","id":1827107496349032940,"id_str":"1827107496349032940","full_text":"a certain indian-origin influencer popular in tech circles wrote an ultimate guide to US citizenship and straight up said \"commit scientific fraud\" lol you can't make this up https://t.co/XSZ97iI42I","truncated":false,"display_text_range":[0,174],"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[],"media":[{"id":1827104462273425408,"id_str":"1827104462273425408","indices":[175,198],"media_url":"http://pbs.twimg.com/media/GVst0_lWsAAFugk.jpg","media_url_https":"https://pbs.twimg.com/media/GVst0_lWsAAFugk.jpg","url":"https://t.co/XSZ97iI42I","display_url":"pic.x.com/XSZ97iI42I","expanded_url":"https://twitter.com/_ashwxn/status/1827107496349032940/photo/1","type":"photo","allow_download_status":true,"original_info":{"width":1590,"height":926,"focus_rects":[{"x":0,"y":36,"h":890,"w":1590},{"x":664,"y":0,"h":926,"w":926},{"x":778,"y":0,"h":926,"w":812},{"x":1127,"y":0,"h":926,"w":463},{"x":0,"y":0,"h":926,"w":1590}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":1200,"h":699,"resize":"fit"},"large":{"w":1590,"h":926,"resize":"fit"},"small":{"w":680,"h":396,"resize":"fit"}},"features":{"orig":{"faces":[]},"medium":{"faces":[]},"large":{"faces":[]},"small":{"faces":[]}}}]},"extended_entities":{"media":[{"id":1827104462273425408,"id_str":"1827104462273425408","indices":[175,198],"media_url":"http://pbs.twimg.com/media/GVst0_lWsAAFugk.jpg","media_url_https":"https://pbs.twimg.com/media/GVst0_lWsAAFugk.jpg","url":"https://t.co/XSZ97iI42I","display_url":"pic.x.com/XSZ97iI42I","expanded_url":"https://twitter.com/_ashwxn/status/1827107496349032940/photo/1","type":"photo","allow_download_status":true,"original_info":{"width":1590,"height":926,"focus_rects":[{"x":0,"y":36,"h":890,"w":1590},{"x":664,"y":0,"h":926,"w":926},{"x":778,"y":0,"h":926,"w":812},{"x":1127,"y":0,"h":926,"w":463},{"x":0,"y":0,"h":926,"w":1590}]},"sizes":{"thumb":{"w":150,"h":150,"resize":"crop"},"medium":{"w":1200,"h":699,"resize":"fit"},"large":{"w":1590,"h":926,"resize":"fit"},"small":{"w":680,"h":396,"resize":"fit"}},"features":{"orig":{"faces":[]},"medium":{"faces":[]},"large":{"faces":[]},"small":{"faces":[]}},"ext_sensitive_media_warning":null,"ext_media_availability":{"status":"available"},"ext_alt_text":null,"ext_media_color":{"palette":[{"rgb":{"red":255,"green":255,"blue":255},"percentage":99.82},{"rgb":{"red":134,"green":134,"blue":134},"percentage":0.18}]},"ext":{"mediaStats":{"r":"Missing","ttl":-1}}}]},"source":"Twitter Web App","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user_id":1349373539858538497,"user_id_str":"1349373539858538497","geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":581,"favorite_count":6678,"reply_count":122,"quote_count":171,"conversation_id":1827107496349032940,"conversation_id_str":"1827107496349032940","conversation_muted":false,"favorited":true,"retweeted":false,"possibly_sensitive":false,"lang":"en","self_thread":{"id":1827107496349032940,"id_str":"1827107496349032940"},"ext":{"superFollowMetadata":{"r":{"ok":{}},"ttl":-1}}},"1826733541239210361":{"created_at":"Thu Aug 22 21:29:49 +0000 2024","id":1826733541239210361,"id_str":"1826733541239210361","full_text":"Btw, all this \"blockchain\" and \"web3\" crap only exists because of DEI.\n\nWhite guys can't get jobs at normal companies doing useful stuff, so instead they pivoted to making this whole online universe that's scientifically impossible to exclude them from","truncated":false,"display_text_range":[0,252],"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[]},"source":"Twitter Web App","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user_id":1458284524761075714,"user_id_str":"1458284524761075714","geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":false,"retweet_count":3,"favorite_count":35,"reply_count":2,"quote_count":1,"conversation_id":1826733541239210361,"conversation_id_str":"1826733541239210361","conversation_muted":false,"favorited":false,"retweeted":false,"lang":"en","ext":{"superFollowMetadata":{"r":{"ok":{}},"ttl":-1}}},"1827480507187159332":{"created_at":"Sat Aug 24 22:58:00 +0000 2024","id":1827480507187159332,"id_str":"1827480507187159332","full_text":"This is just a high-IQ analogue to open defecation. The general phenomenon, common to pretty much all of the 3rd world, is a consistent pattern of abusing every single public commons until it's no longer usable","truncated":false,"display_text_range":[0,210],"entities":{"hashtags":[],"symbols":[],"user_mentions":[],"urls":[]},"source":"Twitter for Android","in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user_id":1244082921683763201,"user_id_str":"1244082921683763201","geo":null,"coordinates":null,"place":null,"contributors":null,"is_quote_status":true,"quoted_status_id":1827107496349032940,"quoted_status_id_str":"1827107496349032940","retweet_count":101,"favorite_count":647,"reply_count":9,"quote_count":1,"conversation_id":1827480507187159332,"conversation_id_str":"1827480507187159332","conversation_muted":false,"favorited":true,"retweeted":true,"current_user_retweet":{"id":1827544032714633628,"id_str":"1827544032714633628"},"lang":"en","ext":{"superFollowMetadata":{"r":{"ok":{}},"ttl":-1}}}}},"timeline":{"id":"FDzeDIfVUAIAAAABiJONco_yJREYxfWFH88","instructions":[{"addEntries":{"entries":[{"entryId":"tweet-1826743131108487390","sortIndex":"1724614504994","content":{"item":{"content":{"tweet":{"id":"1826743131108487390","displayType":"Tweet","displaySize":"Small"}}}}},{"entryId":"tweet-1827544032714633628","sortIndex":"1724614203114","content":{"item":{"content":{"tweet":{"id":"1827544032714633628","displayType":"Tweet","displaySize":"Small"}}}}},{"entryId":"main-user-1059028569655980032","sortIndex":"0","content":{"timelineModule":{"items":[{"entryId":"user-1059028569655980032","item":{"content":{"user":{"id":"1059028569655980032","displayType":"ProfileCard"}}}}],"displayType":"Vertical"}}}]}}]}}