diff --git a/cmd/twitter/helpers.go b/cmd/twitter/helpers.go index 00d7690..ff8557d 100644 --- a/cmd/twitter/helpers.go +++ b/cmd/twitter/helpers.go @@ -9,6 +9,7 @@ import ( "runtime" "strconv" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/terminal_utils" ) @@ -55,14 +56,14 @@ func happy_exit(text string, exit_err error) { * * returns: the id at the end of the tweet: e.g., 1395882872729477131 */ -func extract_id_from(url string) (scraper.TweetID, error) { +func extract_id_from(url string) (TweetID, error) { _, id, is_ok := scraper.TryParseTweetUrl(url) if is_ok { return id, nil } num, err := strconv.Atoi(url) - return scraper.TweetID(num), err + return TweetID(num), err } // Get a sensible default path to create a default profile. Uses `XDG_DATA_HOME` if available @@ -98,7 +99,7 @@ func is_scrape_failure(err error) bool { } // DUPE: full_save_tweet_trove -func full_save_tweet_trove(trove scraper.TweetTrove) { +func full_save_tweet_trove(trove TweetTrove) { conflicting_users := profile.SaveTweetTrove(trove, true, api.DownloadMedia) for _, u_id := range conflicting_users { fmt.Printf(terminal_utils.COLOR_YELLOW+ @@ -110,7 +111,7 @@ func full_save_tweet_trove(trove scraper.TweetTrove) { if errors.Is(err, scraper.ErrDoesntExist) { // Mark them as deleted. // Handle and display name won't be updated if the user exists. - updated_user = scraper.User{ID: u_id, DisplayName: "", Handle: "", IsDeleted: true} + updated_user = User{ID: u_id, DisplayName: "", Handle: "", IsDeleted: true} } else if err != nil { panic(fmt.Errorf("error scraping conflicting user (ID %d): %w", u_id, err)) } diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index feff547..275a5ba 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -15,12 +15,12 @@ import ( "time" "gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Global variable referencing the open data profile -var profile persistence.Profile +var profile Profile var version_string string @@ -119,7 +119,7 @@ func main() { } // Path exists and is a directory; safe to continue } - profile, err = persistence.LoadProfile(*profile_dir) + profile, err = LoadProfile(*profile_dir) if err != nil { if *use_default_profile { create_profile(*profile_dir) @@ -133,7 +133,7 @@ func main() { // Lop off the ".session" suffix (allows using `--session asdf.session` which lets you tab-autocomplete at command line) *session_name = (*session_name)[:len(*session_name)-8] } - profile.LoadSession(scraper.UserHandle(*session_name), &api) + profile.LoadSession(UserHandle(*session_name), &api) } else { var err error api, err = scraper.NewGuestSession() @@ -162,15 +162,15 @@ func main() { } login(target, password) case "fetch_user": - fetch_user(scraper.UserHandle(target)) + fetch_user(UserHandle(target)) case "fetch_user_by_id": id, err := strconv.Atoi(target) if err != nil { panic(err) } - fetch_user_by_id(scraper.UserID(id)) + fetch_user_by_id(UserID(id)) case "download_user_content": - download_user_content(scraper.UserHandle(target)) + download_user_content(UserHandle(target)) case "fetch_tweet_only": fetch_tweet_only(target) case "fetch_tweet": @@ -280,25 +280,25 @@ func login(username string, password string) { * - target_dir: the location of the new data dir. */ func create_profile(target_dir string) { - _, err := persistence.NewProfile(target_dir) + _, err := NewProfile(target_dir) if err != nil { panic(err) } } -func _fetch_user_by_id(id scraper.UserID) error { +func _fetch_user_by_id(id UserID) error { user, err := scraper.GetUserByID(id) if errors.Is(err, scraper.ErrDoesntExist) { // Mark them as deleted. // Handle and display name won't be updated if the user exists. - user = scraper.User{ID: id, DisplayName: "", Handle: "", IsDeleted: true} + user = User{ID: id, DisplayName: "", Handle: "", IsDeleted: true} } else if err != nil { return fmt.Errorf("scraping error on user ID %d: %w", id, err) } log.Debugf("%#v\n", user) err = profile.SaveUser(&user) - var conflict_err persistence.ErrConflictingUserHandle + var conflict_err ErrConflictingUserHandle if errors.As(err, &conflict_err) { log.Warnf( "Conflicting user handle found (ID %d); old user has been marked deleted. Rescraping them", @@ -319,7 +319,7 @@ func _fetch_user_by_id(id scraper.UserID) error { return nil } -func fetch_user(handle scraper.UserHandle) { +func fetch_user(handle UserHandle) { user, err := api.GetUser(handle) if errors.Is(err, scraper.ErrDoesntExist) { // There's several reasons we could get a ErrDoesntExist: @@ -335,7 +335,7 @@ func fetch_user(handle scraper.UserHandle) { log.Debugf("%#v\n", user) err = profile.SaveUser(&user) - var conflict_err persistence.ErrConflictingUserHandle + var conflict_err ErrConflictingUserHandle if errors.As(err, &conflict_err) { log.Warnf( "Conflicting user handle found (ID %d); old user has been marked deleted. Rescraping them", @@ -352,7 +352,7 @@ func fetch_user(handle scraper.UserHandle) { happy_exit("Saved the user", nil) } -func fetch_user_by_id(id scraper.UserID) { +func fetch_user_by_id(id UserID) { err := _fetch_user_by_id(id) if err != nil { die(err.Error(), false, -1) @@ -382,7 +382,7 @@ func fetch_tweet_only(tweet_identifier string) { if !ok { panic("Trove didn't contain its own tweet!") } - tweet.LastScrapedAt = scraper.Timestamp{time.Now()} + tweet.LastScrapedAt = Timestamp{time.Now()} tweet.IsConversationScraped = true log.Debug(tweet) @@ -422,7 +422,7 @@ func fetch_tweet_conversation(tweet_identifier string, how_many int) { * - handle: the user handle to get */ func fetch_user_feed(handle string, how_many int) { - user, err := profile.GetUserByHandle(scraper.UserHandle(handle)) + user, err := profile.GetUserByHandle(UserHandle(handle)) if is_scrape_failure(err) { die(fmt.Sprintf("Error getting user: %s\n %s", handle, err.Error()), false, -1) } @@ -440,7 +440,7 @@ func fetch_user_feed(handle string, how_many int) { } func get_user_likes(handle string, how_many int) { - user, err := profile.GetUserByHandle(scraper.UserHandle(handle)) + user, err := profile.GetUserByHandle(UserHandle(handle)) if err != nil { die(fmt.Sprintf("Error getting user: %s\n %s", handle, err.Error()), false, -1) } @@ -458,7 +458,7 @@ func get_user_likes(handle string, how_many int) { } func get_followees(handle string, how_many int) { - user, err := profile.GetUserByHandle(scraper.UserHandle(handle)) + user, err := profile.GetUserByHandle(UserHandle(handle)) if err != nil { die(fmt.Sprintf("Error getting user: %s\n %s", handle, err.Error()), false, -1) } @@ -473,7 +473,7 @@ func get_followees(handle string, how_many int) { happy_exit(fmt.Sprintf("Saved %d followees", len(trove.Users)), err) } func get_followers(handle string, how_many int) { - user, err := profile.GetUserByHandle(scraper.UserHandle(handle)) + user, err := profile.GetUserByHandle(UserHandle(handle)) if err != nil { die(fmt.Sprintf("Error getting user: %s\n %s", handle, err.Error()), false, -1) } @@ -528,7 +528,7 @@ func download_tweet_content(tweet_identifier string) { } } -func download_user_content(handle scraper.UserHandle) { +func download_user_content(handle UserHandle) { user, err := profile.GetUserByHandle(handle) if err != nil { panic("Couldn't get the user from database: " + err.Error()) @@ -550,7 +550,7 @@ func search(query string, how_many int) { } func follow_user(handle string, is_followed bool) { - user, err := profile.GetUserByHandle(scraper.UserHandle(handle)) + user, err := profile.GetUserByHandle(UserHandle(handle)) if err != nil { panic("Couldn't get the user from database: " + err.Error()) } @@ -612,11 +612,11 @@ func fetch_inbox(how_many int) { } func fetch_dm(id string, how_many int) { - room, err := profile.GetChatRoom(scraper.DMChatRoomID(id)) + room, err := profile.GetChatRoom(DMChatRoomID(id)) if is_scrape_failure(err) { panic(err) } - max_id := scraper.DMMessageID(^uint(0) >> 1) + max_id := DMMessageID(^uint(0) >> 1) trove, err := api.GetConversation(room.ID, max_id, how_many) if err != nil { die(fmt.Sprintf("Failed to fetch dm:\n %s", err.Error()), false, 1) @@ -629,12 +629,12 @@ func fetch_dm(id string, how_many int) { } func send_dm(room_id string, text string, in_reply_to_id int) { - room, err := profile.GetChatRoom(scraper.DMChatRoomID(room_id)) + room, err := profile.GetChatRoom(DMChatRoomID(room_id)) if err != nil { die(fmt.Sprintf("No such chat room: %d", in_reply_to_id), false, 1) } - trove, err := api.SendDMMessage(room.ID, text, scraper.DMMessageID(in_reply_to_id)) + trove, err := api.SendDMMessage(room.ID, text, DMMessageID(in_reply_to_id)) if err != nil { die(fmt.Sprintf("Failed to send dm:\n %s", err.Error()), false, 1) } @@ -643,15 +643,15 @@ func send_dm(room_id string, text string, in_reply_to_id int) { } func send_dm_reacc(room_id string, in_reply_to_id int, reacc string) { - room, err := profile.GetChatRoom(scraper.DMChatRoomID(room_id)) + room, err := profile.GetChatRoom(DMChatRoomID(room_id)) if err != nil { die(fmt.Sprintf("No such chat room: %d", in_reply_to_id), false, 1) } - _, err = profile.GetChatMessage(scraper.DMMessageID(in_reply_to_id)) + _, err = profile.GetChatMessage(DMMessageID(in_reply_to_id)) if err != nil { die(fmt.Sprintf("No such message: %d", in_reply_to_id), false, 1) } - err = api.SendDMReaction(room.ID, scraper.DMMessageID(in_reply_to_id), reacc) + err = api.SendDMReaction(room.ID, DMMessageID(in_reply_to_id), reacc) if err != nil { die(fmt.Sprintf("Failed to react to message:\n %s", err.Error()), false, 1) } diff --git a/internal/webserver/handler_bookmarks.go b/internal/webserver/handler_bookmarks.go index a759ab8..18acb7c 100644 --- a/internal/webserver/handler_bookmarks.go +++ b/internal/webserver/handler_bookmarks.go @@ -4,7 +4,7 @@ import ( "errors" "net/http" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) @@ -29,7 +29,7 @@ func (app *Application) Bookmarks(w http.ResponseWriter, r *http.Request) { app.full_save_tweet_trove(trove) } - c := persistence.NewUserFeedBookmarksCursor(app.ActiveUser.Handle) + c := NewUserFeedBookmarksCursor(app.ActiveUser.Handle) err := parse_cursor_value(&c, r) if err != nil { app.error_400_with_message(w, r, "invalid cursor (must be a number)") @@ -37,11 +37,11 @@ func (app *Application) Bookmarks(w http.ResponseWriter, r *http.Request) { } feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) - if err != nil && !errors.Is(err, persistence.ErrEndOfFeed) { + if err != nil && !errors.Is(err, ErrEndOfFeed) { panic(err) } - if is_htmx(r) && c.CursorPosition == persistence.CURSOR_MIDDLE { + if is_htmx(r) && c.CursorPosition == CURSOR_MIDDLE { // It's a Show More request app.buffered_render_htmx(w, "timeline", PageGlobalData{TweetTrove: feed.TweetTrove}, feed) } else { diff --git a/internal/webserver/handler_follow_unfollow.go b/internal/webserver/handler_follow_unfollow.go index 83ac035..f633af0 100644 --- a/internal/webserver/handler_follow_unfollow.go +++ b/internal/webserver/handler_follow_unfollow.go @@ -4,7 +4,7 @@ import ( "net/http" "strings" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) // TODO: deprecated-offline-follows @@ -22,7 +22,7 @@ func (app *Application) UserFollow(w http.ResponseWriter, r *http.Request) { app.error_400_with_message(w, r, "Bad URL: "+r.URL.Path) return } - user, err := app.Profile.GetUserByHandle(scraper.UserHandle(parts[1])) + user, err := app.Profile.GetUserByHandle(UserHandle(parts[1])) if err != nil { app.error_404(w, r) return @@ -46,7 +46,7 @@ func (app *Application) UserUnfollow(w http.ResponseWriter, r *http.Request) { app.error_400_with_message(w, r, "Bad URL: "+r.URL.Path) return } - user, err := app.Profile.GetUserByHandle(scraper.UserHandle(parts[1])) + user, err := app.Profile.GetUserByHandle(UserHandle(parts[1])) if err != nil { app.error_404(w, r) return diff --git a/internal/webserver/handler_lists.go b/internal/webserver/handler_lists.go index 329ae6e..4d27306 100644 --- a/internal/webserver/handler_lists.go +++ b/internal/webserver/handler_lists.go @@ -10,13 +10,12 @@ import ( "strconv" "strings" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type ListData struct { List - Feed persistence.Feed + Feed Feed UserIDs []UserID ActiveTab string } @@ -36,17 +35,17 @@ func NewListData(users []User) (ListData, TweetTrove) { func (app *Application) ListDetailFeed(w http.ResponseWriter, r *http.Request) { list := get_list_from_context(r.Context()) - c := persistence.NewListCursor(list.ID) + c := NewListCursor(list.ID) err := parse_cursor_value(&c, r) if err != nil { app.error_400_with_message(w, r, "invalid cursor (must be a number)") return } feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) - if err != nil && !errors.Is(err, persistence.ErrEndOfFeed) { + if err != nil && !errors.Is(err, ErrEndOfFeed) { panic(err) } - if is_htmx(r) && c.CursorPosition == persistence.CURSOR_MIDDLE { + if is_htmx(r) && c.CursorPosition == CURSOR_MIDDLE { // It's a Show More request app.buffered_render_htmx(w, "timeline", PageGlobalData{TweetTrove: feed.TweetTrove}, feed) } else { diff --git a/internal/webserver/handler_login.go b/internal/webserver/handler_login.go index 3b73e23..6363a56 100644 --- a/internal/webserver/handler_login.go +++ b/internal/webserver/handler_login.go @@ -7,12 +7,13 @@ import ( "io" "net/http" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type LoginData struct { LoginForm - ExistingSessions []scraper.UserHandle + ExistingSessions []UserHandle } type LoginForm struct { @@ -112,7 +113,7 @@ func (app *Application) ChangeSession(w http.ResponseWriter, r *http.Request) { formdata, err := io.ReadAll(r.Body) panic_if(err) panic_if(json.Unmarshal(formdata, &form)) // TODO: HTTP 400 not 500 - err = app.SetActiveUser(scraper.UserHandle(form.AccountName)) + err = app.SetActiveUser(UserHandle(form.AccountName)) if err != nil { app.error_400_with_message(w, r, fmt.Sprintf("User not in database: %s", form.AccountName)) return diff --git a/internal/webserver/handler_messages.go b/internal/webserver/handler_messages.go index 5e29b85..d4f938d 100644 --- a/internal/webserver/handler_messages.go +++ b/internal/webserver/handler_messages.go @@ -11,15 +11,15 @@ import ( "strings" "time" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type MessageData struct { - persistence.DMChatView + DMChatView LatestPollingTimestamp int ScrollBottom bool - UnreadRoomIDs map[scraper.DMChatRoomID]bool + UnreadRoomIDs map[DMChatRoomID]bool } func (app *Application) messages_index(w http.ResponseWriter, r *http.Request) { @@ -30,7 +30,7 @@ func (app *Application) messages_index(w http.ResponseWriter, r *http.Request) { func (app *Application) message_mark_as_read(w http.ResponseWriter, r *http.Request) { room_id := get_room_id_from_context(r.Context()) - c := persistence.NewConversationCursor(room_id) + c := NewConversationCursor(room_id) c.PageSize = 1 chat_contents := app.Profile.GetChatRoomMessagesByCursor(c) last_message_id := chat_contents.MessageIDs[len(chat_contents.MessageIDs)-1] @@ -76,7 +76,7 @@ func (app *Application) message_send(w http.ResponseWriter, r *http.Request) { app.error_401(w, r) return } - trove, err := app.API.SendDMMessage(room_id, message_data.Text, scraper.DMMessageID(in_reply_to_id)) + trove, err := app.API.SendDMMessage(room_id, message_data.Text, DMMessageID(in_reply_to_id)) if err != nil { panic(err) } @@ -107,8 +107,8 @@ func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) { return } var data struct { - MessageID scraper.DMMessageID `json:"message_id,string"` - Reacc string `json:"reacc"` + MessageID DMMessageID `json:"message_id,string"` + Reacc string `json:"reacc"` } data_, err := io.ReadAll(r.Body) panic_if(err) @@ -129,11 +129,11 @@ func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) { panic(global_data) } } - dm_message.Reactions[app.ActiveUser.ID] = scraper.DMReaction{ + dm_message.Reactions[app.ActiveUser.ID] = DMReaction{ ID: 0, // Hopefully will be OK temporarily DMMessageID: dm_message.ID, SenderID: app.ActiveUser.ID, - SentAt: scraper.Timestamp{time.Now()}, + SentAt: Timestamp{time.Now()}, Emoji: data.Reacc, } global_data.Messages[dm_message.ID] = dm_message @@ -147,7 +147,7 @@ func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) { } if r.URL.Query().Has("scrape") && !app.IsScrapingDisabled { - max_id := scraper.DMMessageID(^uint(0) >> 1) + max_id := DMMessageID(^uint(0) >> 1) trove, err := app.API.GetConversation(room_id, max_id, 50) // TODO: parameterizable if err != nil { panic(err) @@ -168,12 +168,12 @@ func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) { chat_view_data.ScrollBottom = true } - c := persistence.NewConversationCursor(room_id) - c.SinceTimestamp = scraper.TimestampFromUnixMilli(int64(chat_view_data.LatestPollingTimestamp)) + c := NewConversationCursor(room_id) + c.SinceTimestamp = TimestampFromUnixMilli(int64(chat_view_data.LatestPollingTimestamp)) if cursor_value := r.URL.Query().Get("cursor"); cursor_value != "" { until_time, err := strconv.Atoi(cursor_value) panic_if(err) // TODO: 400 not 500 - c.UntilTimestamp = scraper.TimestampFromUnixMilli(int64(until_time)) + c.UntilTimestamp = TimestampFromUnixMilli(int64(until_time)) } chat_contents := app.Profile.GetChatRoomMessagesByCursor(c) chat_view_data.DMChatView.MergeWith(chat_contents.TweetTrove) @@ -210,7 +210,7 @@ func (app *Application) message_detail(w http.ResponseWriter, r *http.Request) { func (app *Application) get_message_global_data() (MessageData, PageGlobalData) { // Get message list previews chat_view_data := MessageData{DMChatView: app.Profile.GetChatRoomsPreview(app.ActiveUser.ID)} - chat_view_data.UnreadRoomIDs = make(map[scraper.DMChatRoomID]bool) + chat_view_data.UnreadRoomIDs = make(map[DMChatRoomID]bool) for _, id := range app.Profile.GetUnreadConversations(app.ActiveUser.ID) { chat_view_data.UnreadRoomIDs[id] = true } @@ -223,7 +223,7 @@ func (app *Application) get_message_global_data() (MessageData, PageGlobalData) func (app *Application) messages_refresh_list(w http.ResponseWriter, r *http.Request) { chat_view_data, global_data := app.get_message_global_data() - chat_view_data.ActiveRoomID = scraper.DMChatRoomID(r.URL.Query().Get("active-chat")) + chat_view_data.ActiveRoomID = DMChatRoomID(r.URL.Query().Get("active-chat")) app.buffered_render_htmx(w, "chat-list", global_data, chat_view_data) } @@ -250,7 +250,7 @@ func (app *Application) Messages(w http.ResponseWriter, r *http.Request) { app.messages_refresh_list(w, r) return } - room_id := scraper.DMChatRoomID(parts[0]) + room_id := DMChatRoomID(parts[0]) // Messages index if room_id == "" { @@ -267,12 +267,12 @@ func (app *Application) Messages(w http.ResponseWriter, r *http.Request) { const ROOM_ID_KEY = key("room_id") // type `key` is defined in "handler_tweet_detail" -func add_room_id_to_context(ctx context.Context, room_id scraper.DMChatRoomID) context.Context { +func add_room_id_to_context(ctx context.Context, room_id DMChatRoomID) context.Context { return context.WithValue(ctx, ROOM_ID_KEY, room_id) } -func get_room_id_from_context(ctx context.Context) scraper.DMChatRoomID { - room_id, is_ok := ctx.Value(ROOM_ID_KEY).(scraper.DMChatRoomID) +func get_room_id_from_context(ctx context.Context) DMChatRoomID { + room_id, is_ok := ctx.Value(ROOM_ID_KEY).(DMChatRoomID) if !is_ok { panic("room_id not found in context") } diff --git a/internal/webserver/handler_messages_test.go b/internal/webserver/handler_messages_test.go index 2922e6e..4f5adc9 100644 --- a/internal/webserver/handler_messages_test.go +++ b/internal/webserver/handler_messages_test.go @@ -12,7 +12,7 @@ import ( "golang.org/x/net/html" "gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) func TestMessagesIndexPageRequiresActiveUser(t *testing.T) { @@ -63,7 +63,7 @@ func TestMessagesRoomRequiresCorrectUser(t *testing.T) { recorder := httptest.NewRecorder() app := webserver.NewApp(profile) app.IsScrapingDisabled = true - app.ActiveUser = scraper.User{ID: 782982734, Handle: "Not a real user"} // Simulate a login + app.ActiveUser = User{ID: 782982734, Handle: "Not a real user"} // Simulate a login app.WithMiddlewares().ServeHTTP(recorder, httptest.NewRequest("GET", "/messages/1488963321701171204-1178839081222115328", nil)) resp2 := recorder.Result() require.Equal(404, resp2.StatusCode) diff --git a/internal/webserver/handler_search.go b/internal/webserver/handler_search.go index 372e8a3..2aa34f2 100644 --- a/internal/webserver/handler_search.go +++ b/internal/webserver/handler_search.go @@ -8,24 +8,24 @@ import ( "strconv" "strings" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type SearchPageData struct { - persistence.Feed + Feed SearchText string - SortOrder persistence.SortOrder + SortOrder SortOrder SortOrderOptions []string IsUsersSearch bool - UserIDs []scraper.UserID + UserIDs []UserID // TODO: fill out the search text in the search bar as well (needs modifying the base template) } func NewSearchPageData() SearchPageData { - ret := SearchPageData{SortOrderOptions: []string{}, Feed: persistence.NewFeed()} + ret := SearchPageData{SortOrderOptions: []string{}, Feed: NewFeed()} for i := 0; i < 4; i++ { // Don't include "Liked At" option which is #4 - ret.SortOrderOptions = append(ret.SortOrderOptions, persistence.SortOrder(i).String()) + ret.SortOrderOptions = append(ret.SortOrderOptions, SortOrder(i).String()) } return ret } @@ -34,7 +34,7 @@ func (app *Application) SearchUsers(w http.ResponseWriter, r *http.Request) { ret := NewSearchPageData() ret.IsUsersSearch = true ret.SearchText = strings.Trim(r.URL.Path, "/") - ret.UserIDs = []scraper.UserID{} + ret.UserIDs = []UserID{} for _, u := range app.Profile.SearchUsers(ret.SearchText) { ret.TweetTrove.Users[u.ID] = u ret.UserIDs = append(ret.UserIDs, u.ID) @@ -110,7 +110,7 @@ func (app *Application) Search(w http.ResponseWriter, r *http.Request) { app.full_save_tweet_trove(trove) } - c, err := persistence.NewCursorFromSearchQuery(search_text) + c, err := NewCursorFromSearchQuery(search_text) if err != nil { app.error_400_with_message(w, r, err.Error()) return @@ -121,13 +121,13 @@ func (app *Application) Search(w http.ResponseWriter, r *http.Request) { return } var is_ok bool - c.SortOrder, is_ok = persistence.SortOrderFromString(r.URL.Query().Get("sort-order")) + c.SortOrder, is_ok = SortOrderFromString(r.URL.Query().Get("sort-order")) if !is_ok && r.URL.Query().Get("sort-order") != "" { app.error_400_with_message(w, r, "Invalid sort order") } feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) - if err != nil && !errors.Is(err, persistence.ErrEndOfFeed) { + if err != nil && !errors.Is(err, ErrEndOfFeed) { panic(err) } @@ -136,7 +136,7 @@ func (app *Application) Search(w http.ResponseWriter, r *http.Request) { data.SearchText = search_text data.SortOrder = c.SortOrder - if is_htmx(r) && c.CursorPosition == persistence.CURSOR_MIDDLE { + if is_htmx(r) && c.CursorPosition == CURSOR_MIDDLE { // It's a Show More request app.buffered_render_htmx(w, "timeline", PageGlobalData{TweetTrove: data.Feed.TweetTrove, SearchText: search_text}, data) } else { diff --git a/internal/webserver/handler_timeline.go b/internal/webserver/handler_timeline.go index 4ceb4d1..b46d6cf 100644 --- a/internal/webserver/handler_timeline.go +++ b/internal/webserver/handler_timeline.go @@ -5,12 +5,11 @@ import ( "net/http" "strings" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type TimelineData struct { - persistence.Feed + Feed ActiveTab string } @@ -19,7 +18,7 @@ type TimelineData struct { func (app *Application) OfflineTimeline(w http.ResponseWriter, r *http.Request) { app.traceLog.Printf("'Timeline' handler (path: %q)", r.URL.Path) - c := persistence.NewTimelineCursor() + c := NewTimelineCursor() err := parse_cursor_value(&c, r) if err != nil { app.error_400_with_message(w, r, "invalid cursor (must be a number)") @@ -27,11 +26,11 @@ func (app *Application) OfflineTimeline(w http.ResponseWriter, r *http.Request) } feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) - if err != nil && !errors.Is(err, persistence.ErrEndOfFeed) { + if err != nil && !errors.Is(err, ErrEndOfFeed) { panic(err) } - if is_htmx(r) && c.CursorPosition == persistence.CURSOR_MIDDLE { + if is_htmx(r) && c.CursorPosition == CURSOR_MIDDLE { // It's a Show More request app.buffered_render_htmx(w, "timeline", PageGlobalData{TweetTrove: feed.TweetTrove}, feed) } else { @@ -53,14 +52,14 @@ func (app *Application) Timeline(w http.ResponseWriter, r *http.Request) { return } - c := persistence.Cursor{ + c := Cursor{ Keywords: []string{}, - ToUserHandles: []scraper.UserHandle{}, - SinceTimestamp: scraper.TimestampFromUnix(0), - UntilTimestamp: scraper.TimestampFromUnix(0), - CursorPosition: persistence.CURSOR_START, + ToUserHandles: []UserHandle{}, + SinceTimestamp: TimestampFromUnix(0), + UntilTimestamp: TimestampFromUnix(0), + CursorPosition: CURSOR_START, CursorValue: 0, - SortOrder: persistence.SORT_ORDER_NEWEST, + SortOrder: SORT_ORDER_NEWEST, PageSize: 50, FollowedByUserHandle: app.ActiveUser.Handle, @@ -72,11 +71,11 @@ func (app *Application) Timeline(w http.ResponseWriter, r *http.Request) { } feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) - if err != nil && !errors.Is(err, persistence.ErrEndOfFeed) { + if err != nil && !errors.Is(err, ErrEndOfFeed) { panic(err) } - if is_htmx(r) && c.CursorPosition == persistence.CURSOR_MIDDLE { + if is_htmx(r) && c.CursorPosition == CURSOR_MIDDLE { // It's a Show More request app.buffered_render_htmx(w, "timeline", PageGlobalData{TweetTrove: feed.TweetTrove}, feed) } else { diff --git a/internal/webserver/handler_tweet_detail.go b/internal/webserver/handler_tweet_detail.go index 816be88..988c019 100644 --- a/internal/webserver/handler_tweet_detail.go +++ b/internal/webserver/handler_tweet_detail.go @@ -8,31 +8,31 @@ import ( "strconv" "strings" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) var ErrNotFound = errors.New("not found") type TweetDetailData struct { - persistence.TweetDetailView - MainTweetID scraper.TweetID + TweetDetailView + MainTweetID TweetID } func NewTweetDetailData() TweetDetailData { return TweetDetailData{ - TweetDetailView: persistence.NewTweetDetailView(), + TweetDetailView: NewTweetDetailView(), } } -func (app *Application) ensure_tweet(id scraper.TweetID, is_forced bool, is_conversation_required bool) (scraper.Tweet, error) { +func (app *Application) ensure_tweet(id TweetID, is_forced bool, is_conversation_required bool) (Tweet, error) { is_available := false is_needing_scrape := is_forced // Check if tweet is already in DB tweet, err := app.Profile.GetTweetById(id) if err != nil { - if errors.Is(err, persistence.ErrNotInDatabase) { + if errors.Is(err, ErrNotInDatabase) { is_needing_scrape = true is_available = false } else { @@ -58,14 +58,14 @@ func (app *Application) ensure_tweet(id scraper.TweetID, is_forced bool, is_conv } if err != nil && !errors.Is(err, scraper.END_OF_FEED) { - return scraper.Tweet{}, fmt.Errorf("scraper error: %w", err) + return Tweet{}, fmt.Errorf("scraper error: %w", err) } } else if is_needing_scrape { app.InfoLog.Printf("Would have scraped Tweet: %d", id) } if !is_available { - return scraper.Tweet{}, ErrNotFound + return Tweet{}, ErrNotFound } return tweet, nil } @@ -92,7 +92,7 @@ func (app *Application) UnlikeTweet(w http.ResponseWriter, r *http.Request) { // It's a different error panic(err) } - err = app.Profile.DeleteLike(scraper.Like{UserID: app.ActiveUser.ID, TweetID: tweet.ID}) + err = app.Profile.DeleteLike(Like{UserID: app.ActiveUser.ID, TweetID: tweet.ID}) panic_if(err) tweet.IsLikedByCurrentUser = false @@ -108,7 +108,7 @@ func (app *Application) TweetDetail(w http.ResponseWriter, r *http.Request) { app.error_400_with_message(w, r, fmt.Sprintf("Invalid tweet ID: %q", parts[1])) return } - tweet_id := scraper.TweetID(val) + tweet_id := TweetID(val) data := NewTweetDetailData() data.MainTweetID = tweet_id @@ -169,12 +169,12 @@ type key string const TWEET_KEY = key("tweet") -func add_tweet_to_context(ctx context.Context, tweet scraper.Tweet) context.Context { +func add_tweet_to_context(ctx context.Context, tweet Tweet) context.Context { return context.WithValue(ctx, TWEET_KEY, tweet) } -func get_tweet_from_context(ctx context.Context) scraper.Tweet { - tweet, is_ok := ctx.Value(TWEET_KEY).(scraper.Tweet) +func get_tweet_from_context(ctx context.Context) Tweet { + tweet, is_ok := ctx.Value(TWEET_KEY).(Tweet) if !is_ok { panic("Tweet not found in context") } diff --git a/internal/webserver/handler_tweet_detail_test.go b/internal/webserver/handler_tweet_detail_test.go index 11e345e..ca1ac4d 100644 --- a/internal/webserver/handler_tweet_detail_test.go +++ b/internal/webserver/handler_tweet_detail_test.go @@ -11,7 +11,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/net/html" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) func TestTweetDetail(t *testing.T) { @@ -105,7 +105,7 @@ func TestLongTweet(t *testing.T) { paragraphs := cascadia.QueryAll(root, selector(".tweet .text")) assert.Len(paragraphs, 22) - twt, err := profile.GetTweetById(scraper.TweetID(1695110851324256692)) + twt, err := profile.GetTweetById(TweetID(1695110851324256692)) require.NoError(err) for i, s := range strings.Split(twt.Text, "\n") { assert.Equal(strings.TrimSpace(s), strings.TrimSpace(paragraphs[i].FirstChild.Data)) diff --git a/internal/webserver/handler_user_feed.go b/internal/webserver/handler_user_feed.go index 73cbdf8..2aeb32e 100644 --- a/internal/webserver/handler_user_feed.go +++ b/internal/webserver/handler_user_feed.go @@ -6,8 +6,7 @@ import ( "net/http" "strings" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { @@ -15,10 +14,10 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/") - user, err := app.Profile.GetUserByHandle(scraper.UserHandle(parts[0])) - if errors.Is(err, persistence.ErrNotInDatabase) { + user, err := app.Profile.GetUserByHandle(UserHandle(parts[0])) + if errors.Is(err, ErrNotInDatabase) { if !app.IsScrapingDisabled { - user, err = app.API.GetUser(scraper.UserHandle(parts[0])) + user, err = app.API.GetUser(UserHandle(parts[0])) } if err != nil { // ErrDoesntExist or otherwise app.error_404(w, r) @@ -47,7 +46,7 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { } // Update the user themself - user, err = app.API.GetUser(scraper.UserHandle(parts[0])) + user, err = app.API.GetUser(UserHandle(parts[0])) panic_if(err) panic_if(app.Profile.SaveUser(&user)) // TODO: handle conflicting users panic_if(app.Profile.DownloadUserContentFor(&user, app.API.DownloadMedia)) @@ -70,17 +69,17 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { } } - var c persistence.Cursor + var c Cursor if len(parts) > 1 && parts[1] == "likes" { - c = persistence.NewUserFeedLikesCursor(user.Handle) + c = NewUserFeedLikesCursor(user.Handle) } else { - c = persistence.NewUserFeedCursor(user.Handle) + c = NewUserFeedCursor(user.Handle) } if len(parts) > 1 && parts[1] == "without_replies" { - c.FilterReplies = persistence.EXCLUDE + c.FilterReplies = EXCLUDE } if len(parts) > 1 && parts[1] == "media" { - c.FilterMedia = persistence.REQUIRE + c.FilterMedia = REQUIRE } err = parse_cursor_value(&c, r) if err != nil { @@ -89,15 +88,15 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { } feed, err := app.Profile.NextPage(c, app.ActiveUser.ID) - if err != nil && !errors.Is(err, persistence.ErrEndOfFeed) { + if err != nil && !errors.Is(err, ErrEndOfFeed) { panic(err) } feed.Users[user.ID] = user data := struct { - persistence.Feed - scraper.UserID - PinnedTweet scraper.Tweet + Feed + UserID + PinnedTweet Tweet FeedType string }{Feed: feed, UserID: user.ID} @@ -109,17 +108,17 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { // Add a pinned tweet if there is one and it's in the DB; otherwise skip // Also, only show pinned tweets on default tab (tweets+replies) or "without_replies" tab - if user.PinnedTweetID != scraper.TweetID(0) && (len(parts) <= 1 || parts[1] == "without_replies") { + if user.PinnedTweetID != TweetID(0) && (len(parts) <= 1 || parts[1] == "without_replies") { data.PinnedTweet, err = app.Profile.GetTweetById(user.PinnedTweetID) - if err != nil && !errors.Is(err, persistence.ErrNotInDatabase) { + if err != nil && !errors.Is(err, ErrNotInDatabase) { panic(err) } feed.TweetTrove.Tweets[data.PinnedTweet.ID] = data.PinnedTweet // Fetch quoted tweet if necessary - if data.PinnedTweet.QuotedTweetID != scraper.TweetID(0) { + if data.PinnedTweet.QuotedTweetID != TweetID(0) { feed.TweetTrove.Tweets[data.PinnedTweet.QuotedTweetID], err = app.Profile.GetTweetById(data.PinnedTweet.QuotedTweetID) - if err != nil && !errors.Is(err, persistence.ErrNotInDatabase) { + if err != nil && !errors.Is(err, ErrNotInDatabase) { panic(err) } // And the user @@ -129,7 +128,7 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { } } - if is_htmx(r) && c.CursorPosition == persistence.CURSOR_MIDDLE { + if is_htmx(r) && c.CursorPosition == CURSOR_MIDDLE { // It's a Show More request app.buffered_render_htmx(w, "timeline", PageGlobalData{TweetTrove: feed.TweetTrove}, data) } else { @@ -139,14 +138,14 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) { type FollowsData struct { Title string - HeaderUserID scraper.UserID - UserIDs []scraper.UserID + HeaderUserID UserID + UserIDs []UserID } -func NewFollowsData(users []scraper.User) (FollowsData, scraper.TweetTrove) { - trove := scraper.NewTweetTrove() +func NewFollowsData(users []User) (FollowsData, TweetTrove) { + trove := NewTweetTrove() data := FollowsData{ - UserIDs: []scraper.UserID{}, + UserIDs: []UserID{}, } for _, u := range users { trove.Users[u.ID] = u @@ -155,7 +154,7 @@ func NewFollowsData(users []scraper.User) (FollowsData, scraper.TweetTrove) { return data, trove } -func (app *Application) UserFollowees(w http.ResponseWriter, r *http.Request, user scraper.User) { +func (app *Application) UserFollowees(w http.ResponseWriter, r *http.Request, user User) { if r.URL.Query().Has("scrape") { if app.IsScrapingDisabled { app.InfoLog.Printf("Would have scraped: %s", r.URL.Path) @@ -180,7 +179,7 @@ func (app *Application) UserFollowees(w http.ResponseWriter, r *http.Request, us app.buffered_render_page(w, "tpl/follows.tpl", PageGlobalData{TweetTrove: trove}, data) } -func (app *Application) UserFollowers(w http.ResponseWriter, r *http.Request, user scraper.User) { +func (app *Application) UserFollowers(w http.ResponseWriter, r *http.Request, user User) { if r.URL.Query().Has("scrape") { if app.IsScrapingDisabled { app.InfoLog.Printf("Would have scraped: %s", r.URL.Path) diff --git a/internal/webserver/renderer_helpers.go b/internal/webserver/renderer_helpers.go index 6e1bfd6..088fec1 100644 --- a/internal/webserver/renderer_helpers.go +++ b/internal/webserver/renderer_helpers.go @@ -11,8 +11,7 @@ import ( "github.com/Masterminds/sprig/v3" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type NotificationBubbles struct { @@ -22,35 +21,35 @@ type NotificationBubbles struct { // TODO: this name sucks type PageGlobalData struct { - scraper.TweetTrove + TweetTrove SearchText string - FocusedTweetID scraper.TweetID + FocusedTweetID TweetID Toasts []Toast NotificationBubbles } -func (d PageGlobalData) Tweet(id scraper.TweetID) scraper.Tweet { +func (d PageGlobalData) Tweet(id TweetID) Tweet { return d.Tweets[id] } -func (d PageGlobalData) User(id scraper.UserID) scraper.User { +func (d PageGlobalData) User(id UserID) User { return d.Users[id] } -func (d PageGlobalData) Retweet(id scraper.TweetID) scraper.Retweet { +func (d PageGlobalData) Retweet(id TweetID) Retweet { return d.Retweets[id] } -func (d PageGlobalData) Space(id scraper.SpaceID) scraper.Space { +func (d PageGlobalData) Space(id SpaceID) Space { return d.Spaces[id] } -func (d PageGlobalData) Notification(id scraper.NotificationID) scraper.Notification { +func (d PageGlobalData) Notification(id NotificationID) Notification { return d.Notifications[id] } -func (d PageGlobalData) Message(id scraper.DMMessageID) scraper.DMMessage { +func (d PageGlobalData) Message(id DMMessageID) DMMessage { return d.Messages[id] } -func (d PageGlobalData) ChatRoom(id scraper.DMChatRoomID) scraper.DMChatRoom { +func (d PageGlobalData) ChatRoom(id DMChatRoomID) DMChatRoom { return d.Rooms[id] } -func (d PageGlobalData) GetFocusedTweetID() scraper.TweetID { +func (d PageGlobalData) GetFocusedTweetID() TweetID { return d.FocusedTweetID } func (d PageGlobalData) GetSearchText() string { @@ -143,18 +142,18 @@ func (app *Application) make_funcmap(global_data PageGlobalData) template.FuncMa "focused_tweet_id": global_data.GetFocusedTweetID, "search_text": global_data.GetSearchText, "global_data": global_data.GlobalData, // This fucking sucks - "active_user": func() scraper.User { + "active_user": func() User { return app.ActiveUser }, // Utility functions - "get_tombstone_text": func(t scraper.Tweet) string { + "get_tombstone_text": func(t Tweet) string { if t.TombstoneText != "" { return t.TombstoneText } return t.TombstoneType }, - "cursor_to_query_params": func(c persistence.Cursor) string { + "cursor_to_query_params": func(c Cursor) string { result := url.Values{} result.Set("cursor", fmt.Sprint(c.CursorValue)) result.Set("sort-order", c.SortOrder.String()) diff --git a/internal/webserver/server.go b/internal/webserver/server.go index c3d5101..bd41cc8 100644 --- a/internal/webserver/server.go +++ b/internal/webserver/server.go @@ -14,7 +14,7 @@ import ( "strings" "time" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) @@ -28,14 +28,14 @@ type Application struct { Middlewares []Middleware - Profile persistence.Profile - ActiveUser scraper.User + Profile Profile + ActiveUser User IsScrapingDisabled bool API scraper.API LastReadNotificationSortIndex int64 } -func NewApp(profile persistence.Profile) Application { +func NewApp(profile Profile) Application { ret := Application{ accessLog: log.New(os.Stdout, "ACCESS\t", log.Ldate|log.Ltime), traceLog: log.New(os.Stdout, "TRACE\t", log.Ldate|log.Ltime), @@ -67,7 +67,7 @@ func (app *Application) WithMiddlewares() http.Handler { return ret } -func (app *Application) SetActiveUser(handle scraper.UserHandle) error { +func (app *Application) SetActiveUser(handle UserHandle) error { if handle == "no account" { app.ActiveUser = get_default_user() app.IsScrapingDisabled = true // API requests will fail b/c not logged in @@ -83,12 +83,12 @@ func (app *Application) SetActiveUser(handle scraper.UserHandle) error { return nil } -func get_default_user() scraper.User { - return scraper.User{ +func get_default_user() User { + return User{ ID: 0, Handle: "[nobody]", DisplayName: "[Not logged in]", - ProfileImageLocalPath: path.Base(scraper.DEFAULT_PROFILE_IMAGE_URL), + ProfileImageLocalPath: path.Base(DEFAULT_PROFILE_IMAGE_URL), IsContentDownloaded: true, } } @@ -189,7 +189,7 @@ func openWebPage(url string) { } } -func parse_cursor_value(c *persistence.Cursor, r *http.Request) error { +func parse_cursor_value(c *Cursor, r *http.Request) error { cursor_param := r.URL.Query().Get("cursor") if cursor_param != "" { var err error @@ -197,7 +197,7 @@ func parse_cursor_value(c *persistence.Cursor, r *http.Request) error { if err != nil { return fmt.Errorf("attempted to parse cursor value %q as int: %w", c.CursorValue, err) } - c.CursorPosition = persistence.CURSOR_MIDDLE + c.CursorPosition = CURSOR_MIDDLE } return nil } diff --git a/internal/webserver/server_test.go b/internal/webserver/server_test.go index bcce349..6219a2a 100644 --- a/internal/webserver/server_test.go +++ b/internal/webserver/server_test.go @@ -10,8 +10,7 @@ import ( "github.com/stretchr/testify/require" "gitlab.com/offline-twitter/twitter_offline_engine/internal/webserver" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type CapturingWriter struct { @@ -23,11 +22,11 @@ func (w *CapturingWriter) Write(p []byte) (int, error) { return len(p), nil } -var profile persistence.Profile +var profile Profile func init() { var err error - profile, err = persistence.LoadProfile("../../sample_data/profile") + profile, err = LoadProfile("../../sample_data/profile") if err != nil { panic(err) } @@ -55,7 +54,7 @@ func do_request_with_active_user(req *http.Request) *http.Response { recorder := httptest.NewRecorder() app := webserver.NewApp(profile) app.IsScrapingDisabled = true - app.ActiveUser = scraper.User{ID: 1488963321701171204, Handle: "Offline_Twatter"} // Simulate a login + app.ActiveUser = User{ID: 1488963321701171204, Handle: "Offline_Twatter"} // Simulate a login app.WithMiddlewares().ServeHTTP(recorder, req) return recorder.Result() } diff --git a/internal/webserver/stopwatch.go b/internal/webserver/stopwatch.go index bcd96ca..e3b7e10 100644 --- a/internal/webserver/stopwatch.go +++ b/internal/webserver/stopwatch.go @@ -8,12 +8,13 @@ import ( "runtime/debug" "time" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type BackgroundTask struct { Name string - GetTroveFunc func(*scraper.API) scraper.TweetTrove + GetTroveFunc func(*scraper.API) TweetTrove StartDelay time.Duration Period time.Duration @@ -81,7 +82,7 @@ func (app *Application) start_background() { timeline_task := BackgroundTask{ Name: "home timeline", - GetTroveFunc: func(api *scraper.API) scraper.TweetTrove { + GetTroveFunc: func(api *scraper.API) TweetTrove { should_do_following_only := is_following_only%is_following_only_frequency == 0 trove, err := api.GetHomeTimeline("", should_do_following_only) if err != nil && !errors.Is(err, scraper.END_OF_FEED) && !errors.Is(err, scraper.ErrRateLimited) { @@ -97,7 +98,7 @@ func (app *Application) start_background() { likes_task := BackgroundTask{ Name: "user likes", - GetTroveFunc: func(api *scraper.API) scraper.TweetTrove { + GetTroveFunc: func(api *scraper.API) TweetTrove { trove, err := api.GetUserLikes(api.UserID, 50) // TODO: parameterizable if err != nil && !errors.Is(err, scraper.END_OF_FEED) && !errors.Is(err, scraper.ErrRateLimited) { panic(err) @@ -112,8 +113,8 @@ func (app *Application) start_background() { dms_task := BackgroundTask{ Name: "DM inbox", - GetTroveFunc: func(api *scraper.API) scraper.TweetTrove { - var trove scraper.TweetTrove + GetTroveFunc: func(api *scraper.API) TweetTrove { + var trove TweetTrove var err error if inbox_cursor == "" { trove, inbox_cursor, err = api.GetInbox(0) @@ -133,7 +134,7 @@ func (app *Application) start_background() { notifications_task := BackgroundTask{ Name: "DM inbox", - GetTroveFunc: func(api *scraper.API) scraper.TweetTrove { + GetTroveFunc: func(api *scraper.API) TweetTrove { trove, last_unread_notification_sort_index, err := api.GetNotifications(1) // Just 1 page if err != nil && !errors.Is(err, scraper.END_OF_FEED) && !errors.Is(err, scraper.ErrRateLimited) { panic(err) @@ -150,7 +151,7 @@ func (app *Application) start_background() { bookmarks_task := BackgroundTask{ Name: "bookmarks", - GetTroveFunc: func(api *scraper.API) scraper.TweetTrove { + GetTroveFunc: func(api *scraper.API) TweetTrove { trove, err := app.API.GetBookmarks(10) if err != nil && !errors.Is(err, scraper.END_OF_FEED) && !errors.Is(err, scraper.ErrRateLimited) { panic(err) @@ -165,7 +166,7 @@ func (app *Application) start_background() { own_profile_task := BackgroundTask{ Name: "user profile", - GetTroveFunc: func(api *scraper.API) scraper.TweetTrove { + GetTroveFunc: func(api *scraper.API) TweetTrove { trove, err := app.API.GetUserFeed(api.UserID, 1) if err != nil && !errors.Is(err, scraper.END_OF_FEED) && !errors.Is(err, scraper.ErrRateLimited) { panic(err) diff --git a/internal/webserver/temp_utils.go b/internal/webserver/temp_utils.go index 8666b2b..79017a9 100644 --- a/internal/webserver/temp_utils.go +++ b/internal/webserver/temp_utils.go @@ -4,7 +4,8 @@ import ( "errors" "fmt" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" + "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // DUPE: full_save_tweet_trove @@ -16,8 +17,8 @@ func (app *Application) full_save_tweet_trove(trove TweetTrove) { for _, u_id := range conflicting_users { app.InfoLog.Printf("Conflicting user handle found (ID %d); old user has been marked deleted. Rescraping manually", u_id) // Rescrape - updated_user, err := GetUserByID(u_id) - if errors.Is(err, ErrDoesntExist) { + updated_user, err := scraper.GetUserByID(u_id) + if errors.Is(err, scraper.ErrDoesntExist) { // Mark them as deleted. // Handle and display name won't be updated if the user exists. updated_user = User{ID: u_id, DisplayName: "", Handle: "", IsDeleted: true} diff --git a/pkg/scraper/bookmark.go b/pkg/persistence/bookmark.go similarity index 89% rename from pkg/scraper/bookmark.go rename to pkg/persistence/bookmark.go index 9dfbb08..b714450 100644 --- a/pkg/scraper/bookmark.go +++ b/pkg/persistence/bookmark.go @@ -1,4 +1,4 @@ -package scraper +package persistence type BookmarkSortID int64 diff --git a/pkg/persistence/bookmark_queries.go b/pkg/persistence/bookmark_queries.go index 429aef0..f354d37 100644 --- a/pkg/persistence/bookmark_queries.go +++ b/pkg/persistence/bookmark_queries.go @@ -2,8 +2,6 @@ package persistence import ( "fmt" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func (p Profile) SaveBookmark(l Bookmark) error { diff --git a/pkg/persistence/compound_queries.go b/pkg/persistence/compound_queries.go index fb1305e..64e0b89 100644 --- a/pkg/persistence/compound_queries.go +++ b/pkg/persistence/compound_queries.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/jmoiron/sqlx" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) var ( diff --git a/pkg/persistence/compound_queries_test.go b/pkg/persistence/compound_queries_test.go index db2f69a..a00588b 100644 --- a/pkg/persistence/compound_queries_test.go +++ b/pkg/persistence/compound_queries_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // A feed should load diff --git a/pkg/persistence/compound_ssf_queries.go b/pkg/persistence/compound_ssf_queries.go index 853cbb0..5750b75 100644 --- a/pkg/persistence/compound_ssf_queries.go +++ b/pkg/persistence/compound_ssf_queries.go @@ -6,8 +6,6 @@ import ( "strconv" "strings" "time" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type SortOrder int diff --git a/pkg/persistence/compound_ssf_queries_parse_test.go b/pkg/persistence/compound_ssf_queries_parse_test.go index f69152b..8c115ed 100644 --- a/pkg/persistence/compound_ssf_queries_parse_test.go +++ b/pkg/persistence/compound_ssf_queries_parse_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func TestTokenizeSearchString(t *testing.T) { diff --git a/pkg/persistence/compound_ssf_queries_test.go b/pkg/persistence/compound_ssf_queries_test.go index 791b357..c8ee02c 100644 --- a/pkg/persistence/compound_ssf_queries_test.go +++ b/pkg/persistence/compound_ssf_queries_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Use a cursor, sort by newest diff --git a/pkg/scraper/dm_chat_room.go b/pkg/persistence/dm_chat_room.go similarity index 98% rename from pkg/scraper/dm_chat_room.go rename to pkg/persistence/dm_chat_room.go index 5a6f552..deed649 100644 --- a/pkg/scraper/dm_chat_room.go +++ b/pkg/persistence/dm_chat_room.go @@ -1,4 +1,4 @@ -package scraper +package persistence type DMChatRoomID string diff --git a/pkg/scraper/dm_message.go b/pkg/persistence/dm_message.go similarity index 97% rename from pkg/scraper/dm_message.go rename to pkg/persistence/dm_message.go index cd566f8..8562d48 100644 --- a/pkg/scraper/dm_message.go +++ b/pkg/persistence/dm_message.go @@ -1,4 +1,4 @@ -package scraper +package persistence type DMMessageID int diff --git a/pkg/persistence/dm_queries.go b/pkg/persistence/dm_queries.go index 830500c..c587411 100644 --- a/pkg/persistence/dm_queries.go +++ b/pkg/persistence/dm_queries.go @@ -7,8 +7,6 @@ import ( "strings" "github.com/jmoiron/sqlx" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) const ( diff --git a/pkg/persistence/dm_queries_test.go b/pkg/persistence/dm_queries_test.go index 0612a79..d7be4e7 100644 --- a/pkg/persistence/dm_queries_test.go +++ b/pkg/persistence/dm_queries_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func TestSaveAndLoadChatRoom(t *testing.T) { diff --git a/pkg/persistence/errors.go b/pkg/persistence/errors.go new file mode 100644 index 0000000..a441d2d --- /dev/null +++ b/pkg/persistence/errors.go @@ -0,0 +1,14 @@ +package persistence + +import ( + "errors" +) + +// Downloader errors +var ( + ErrorDMCA = errors.New("video is DMCAed, unable to download (HTTP 403 Forbidden)") + ErrMediaDownload404 = errors.New("media download HTTP 404") + + // TODO: this DEFINITELY does not belong here + ErrRequestTimeout = errors.New("request timed out") +) diff --git a/pkg/persistence/follow_queries.go b/pkg/persistence/follow_queries.go index a0d4707..95d1a24 100644 --- a/pkg/persistence/follow_queries.go +++ b/pkg/persistence/follow_queries.go @@ -1,9 +1,5 @@ package persistence -import ( - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" -) - func (p Profile) SaveFollow(follower_id UserID, followee_id UserID) { _, err := p.DB.Exec(` insert into follows (follower_id, followee_id) diff --git a/pkg/persistence/follow_queries_test.go b/pkg/persistence/follow_queries_test.go index cbbc48c..b869952 100644 --- a/pkg/persistence/follow_queries_test.go +++ b/pkg/persistence/follow_queries_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func TestSaveAndLoadFollows(t *testing.T) { diff --git a/pkg/scraper/image.go b/pkg/persistence/image.go similarity index 95% rename from pkg/scraper/image.go rename to pkg/persistence/image.go index 2d14b0f..c7a89ee 100644 --- a/pkg/scraper/image.go +++ b/pkg/persistence/image.go @@ -1,4 +1,4 @@ -package scraper +package persistence type ImageID int64 diff --git a/pkg/scraper/like.go b/pkg/persistence/like.go similarity index 88% rename from pkg/scraper/like.go rename to pkg/persistence/like.go index 8b276f2..c162b7c 100644 --- a/pkg/scraper/like.go +++ b/pkg/persistence/like.go @@ -1,4 +1,4 @@ -package scraper +package persistence type LikeSortID int64 diff --git a/pkg/persistence/likes_queries.go b/pkg/persistence/likes_queries.go index e8bb781..9ebd8a6 100644 --- a/pkg/persistence/likes_queries.go +++ b/pkg/persistence/likes_queries.go @@ -2,8 +2,6 @@ package persistence import ( "fmt" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func (p Profile) SaveLike(l Like) error { diff --git a/pkg/scraper/list.go b/pkg/persistence/list.go similarity index 92% rename from pkg/scraper/list.go rename to pkg/persistence/list.go index 6443cce..5495a2c 100644 --- a/pkg/scraper/list.go +++ b/pkg/persistence/list.go @@ -1,4 +1,4 @@ -package scraper +package persistence type ListID int64 type OnlineListID int64 diff --git a/pkg/persistence/list_queries.go b/pkg/persistence/list_queries.go index 5de506a..014f6a9 100644 --- a/pkg/persistence/list_queries.go +++ b/pkg/persistence/list_queries.go @@ -4,8 +4,6 @@ import ( "database/sql" "errors" "fmt" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Create an empty list, or rename an existing list diff --git a/pkg/persistence/list_queries_test.go b/pkg/persistence/list_queries_test.go index e582071..4a441f2 100644 --- a/pkg/persistence/list_queries_test.go +++ b/pkg/persistence/list_queries_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func TestSaveAndLoadOfflineList(t *testing.T) { diff --git a/pkg/persistence/media_download.go b/pkg/persistence/media_download.go index dc14185..9d023a5 100644 --- a/pkg/persistence/media_download.go +++ b/pkg/persistence/media_download.go @@ -5,8 +5,6 @@ import ( "fmt" "os" "path/filepath" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type MediaDownloader interface { diff --git a/pkg/persistence/media_download_test.go b/pkg/persistence/media_download_test.go index 01e0123..37ca40a 100644 --- a/pkg/persistence/media_download_test.go +++ b/pkg/persistence/media_download_test.go @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) // Some types to spy on a MediaDownloader diff --git a/pkg/persistence/media_queries.go b/pkg/persistence/media_queries.go index 563624a..f43aa7c 100644 --- a/pkg/persistence/media_queries.go +++ b/pkg/persistence/media_queries.go @@ -2,8 +2,6 @@ package persistence import ( "fmt" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Save an Image diff --git a/pkg/persistence/media_queries_test.go b/pkg/persistence/media_queries_test.go index cd826e1..da36d58 100644 --- a/pkg/persistence/media_queries_test.go +++ b/pkg/persistence/media_queries_test.go @@ -8,7 +8,7 @@ import ( "github.com/go-test/deep" "github.com/stretchr/testify/require" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) // Create an Image, save it, reload it, and make sure it comes back the same diff --git a/pkg/scraper/notification.go b/pkg/persistence/notification.go similarity index 99% rename from pkg/scraper/notification.go rename to pkg/persistence/notification.go index 640f6b7..142382f 100644 --- a/pkg/scraper/notification.go +++ b/pkg/persistence/notification.go @@ -1,4 +1,4 @@ -package scraper +package persistence type NotificationID string diff --git a/pkg/persistence/notification_queries.go b/pkg/persistence/notification_queries.go index 08aaeae..6b61a18 100644 --- a/pkg/persistence/notification_queries.go +++ b/pkg/persistence/notification_queries.go @@ -4,8 +4,6 @@ import ( "database/sql" "errors" "fmt" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func (p Profile) SaveNotification(n Notification) { diff --git a/pkg/persistence/notification_queries_test.go b/pkg/persistence/notification_queries_test.go index c3f8796..6537d23 100644 --- a/pkg/persistence/notification_queries_test.go +++ b/pkg/persistence/notification_queries_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func TestSaveAndLoadNotification(t *testing.T) { diff --git a/pkg/scraper/poll.go b/pkg/persistence/poll.go similarity index 98% rename from pkg/scraper/poll.go rename to pkg/persistence/poll.go index 3ae0611..ce3584f 100644 --- a/pkg/scraper/poll.go +++ b/pkg/persistence/poll.go @@ -1,4 +1,4 @@ -package scraper +package persistence import ( "time" diff --git a/pkg/persistence/profile.go b/pkg/persistence/profile.go index dc0eb5e..dbe08e5 100644 --- a/pkg/persistence/profile.go +++ b/pkg/persistence/profile.go @@ -8,8 +8,6 @@ import ( sql "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) //go:embed schema.sql diff --git a/pkg/scraper/retweet.go b/pkg/persistence/retweet.go similarity index 92% rename from pkg/scraper/retweet.go rename to pkg/persistence/retweet.go index ce94c16..b2c5946 100644 --- a/pkg/scraper/retweet.go +++ b/pkg/persistence/retweet.go @@ -1,4 +1,4 @@ -package scraper +package persistence type Retweet struct { RetweetID TweetID `db:"retweet_id"` diff --git a/pkg/persistence/retweet_queries.go b/pkg/persistence/retweet_queries.go index a953be6..6c447cd 100644 --- a/pkg/persistence/retweet_queries.go +++ b/pkg/persistence/retweet_queries.go @@ -2,8 +2,6 @@ package persistence import ( "fmt" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Save a Retweet. Do nothing if it already exists, because none of its parameters are modifiable. diff --git a/pkg/persistence/session.go b/pkg/persistence/session.go index 7b3a102..a18b495 100644 --- a/pkg/persistence/session.go +++ b/pkg/persistence/session.go @@ -6,8 +6,6 @@ import ( "os" log "github.com/sirupsen/logrus" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func (p Profile) SaveSession(userhandle UserHandle, data []byte) { diff --git a/pkg/scraper/space.go b/pkg/persistence/space.go similarity index 98% rename from pkg/scraper/space.go rename to pkg/persistence/space.go index c84e8dd..157364d 100644 --- a/pkg/scraper/space.go +++ b/pkg/persistence/space.go @@ -1,4 +1,4 @@ -package scraper +package persistence import ( "fmt" diff --git a/pkg/persistence/space_queries.go b/pkg/persistence/space_queries.go index 672a2d1..4bb67a8 100644 --- a/pkg/persistence/space_queries.go +++ b/pkg/persistence/space_queries.go @@ -4,8 +4,6 @@ import ( "database/sql" "errors" "fmt" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type SpaceParticipant struct { diff --git a/pkg/persistence/space_queries_test.go b/pkg/persistence/space_queries_test.go index 06323c2..b7bad8d 100644 --- a/pkg/persistence/space_queries_test.go +++ b/pkg/persistence/space_queries_test.go @@ -9,7 +9,7 @@ import ( "github.com/go-test/deep" "math/rand" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) // Create a Space, save it, reload it, and make sure it comes back the same diff --git a/pkg/scraper/space_test.go b/pkg/persistence/space_test.go similarity index 81% rename from pkg/scraper/space_test.go rename to pkg/persistence/space_test.go index 7d21a81..a928c81 100644 --- a/pkg/scraper/space_test.go +++ b/pkg/persistence/space_test.go @@ -1,11 +1,11 @@ -package scraper_test +package persistence_test import ( "testing" "github.com/stretchr/testify/assert" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) func TestFormatSpaceDuration(t *testing.T) { diff --git a/pkg/scraper/timestamp_type.go b/pkg/persistence/timestamp_type.go similarity index 97% rename from pkg/scraper/timestamp_type.go rename to pkg/persistence/timestamp_type.go index 6e3c05b..adf9e46 100644 --- a/pkg/scraper/timestamp_type.go +++ b/pkg/persistence/timestamp_type.go @@ -1,4 +1,4 @@ -package scraper +package persistence import ( "database/sql/driver" diff --git a/pkg/scraper/tweet.go b/pkg/persistence/tweet.go similarity index 94% rename from pkg/scraper/tweet.go rename to pkg/persistence/tweet.go index 9dfec10..8c25d29 100644 --- a/pkg/scraper/tweet.go +++ b/pkg/persistence/tweet.go @@ -1,4 +1,4 @@ -package scraper +package persistence import ( "database/sql/driver" @@ -46,9 +46,9 @@ type Tweet struct { User *User `db:"user"` // For processing tombstones - UserHandle UserHandle - in_reply_to_user_handle UserHandle - in_reply_to_user_id UserID + UserHandle UserHandle + InReplyToUserHandle UserHandle + InReplyToUserID UserID Images []Image Videos []Video diff --git a/pkg/persistence/tweet_queries.go b/pkg/persistence/tweet_queries.go index 8997198..112ed69 100644 --- a/pkg/persistence/tweet_queries.go +++ b/pkg/persistence/tweet_queries.go @@ -4,8 +4,6 @@ import ( "database/sql" "errors" "fmt" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func (p Profile) SaveTweet(t Tweet) error { diff --git a/pkg/persistence/tweet_queries_test.go b/pkg/persistence/tweet_queries_test.go index 144045d..0e79ea2 100644 --- a/pkg/persistence/tweet_queries_test.go +++ b/pkg/persistence/tweet_queries_test.go @@ -9,7 +9,6 @@ import ( "github.com/go-test/deep" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Create a Tweet, save it, reload it, and make sure it comes back the same diff --git a/pkg/scraper/tweet_trove.go b/pkg/persistence/tweet_trove.go similarity index 99% rename from pkg/scraper/tweet_trove.go rename to pkg/persistence/tweet_trove.go index 5a75248..01f3a05 100644 --- a/pkg/scraper/tweet_trove.go +++ b/pkg/persistence/tweet_trove.go @@ -1,4 +1,4 @@ -package scraper +package persistence import ( "fmt" diff --git a/pkg/persistence/tweet_trove_queries.go b/pkg/persistence/tweet_trove_queries.go index 3eb5493..cdc76e0 100644 --- a/pkg/persistence/tweet_trove_queries.go +++ b/pkg/persistence/tweet_trove_queries.go @@ -4,8 +4,6 @@ import ( "errors" "fmt" "path" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Convenience function that saves all the objects in a TweetTrove. diff --git a/pkg/scraper/tweet_trove_test.go b/pkg/persistence/tweet_trove_test.go similarity index 94% rename from pkg/scraper/tweet_trove_test.go rename to pkg/persistence/tweet_trove_test.go index 2e8bff9..0f98dd1 100644 --- a/pkg/scraper/tweet_trove_test.go +++ b/pkg/persistence/tweet_trove_test.go @@ -1,4 +1,4 @@ -package scraper_test +package persistence_test import ( "testing" @@ -6,7 +6,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) func TestMergeTweetTroves(t *testing.T) { diff --git a/pkg/scraper/url.go b/pkg/persistence/url.go similarity index 98% rename from pkg/scraper/url.go rename to pkg/persistence/url.go index f4f1ae2..e4348f5 100644 --- a/pkg/scraper/url.go +++ b/pkg/persistence/url.go @@ -1,4 +1,4 @@ -package scraper +package persistence import ( "net/url" diff --git a/pkg/scraper/user.go b/pkg/persistence/user.go similarity index 83% rename from pkg/scraper/user.go rename to pkg/persistence/user.go index af2edad..9d46dbb 100644 --- a/pkg/scraper/user.go +++ b/pkg/persistence/user.go @@ -1,4 +1,4 @@ -package scraper +package persistence import ( "fmt" @@ -80,32 +80,6 @@ func GetUnknownUserWithHandle(handle UserHandle) User { } } -/** - * Make a filename for the profile image, that hopefully won't clobber other ones - */ -func (u User) compute_profile_image_local_path() string { - return string(u.Handle) + "_profile_" + path.Base(u.ProfileImageUrl) -} - -/** - * Make a filename for the banner image, that hopefully won't clobber other ones. - * Add a file extension if necessary (seems to be necessary). - * If there is no banner image, just return nothing. - */ -func (u User) compute_banner_image_local_path() string { - if u.BannerImageUrl == "" { - return "" - } - base_name := path.Base(u.BannerImageUrl) - - // Check if it has an extension (e.g., ".png" or ".jpeg") - if !regexp.MustCompile(`\.\w{2,4}$`).MatchString(base_name) { - // If it doesn't have an extension, add one - base_name += ".jpg" - } - return string(u.Handle) + "_banner_" + base_name -} - /** * Get the URL where we would expect to find a User's tiny profile image */ diff --git a/pkg/persistence/user_queries.go b/pkg/persistence/user_queries.go index 3485bef..abf1d71 100644 --- a/pkg/persistence/user_queries.go +++ b/pkg/persistence/user_queries.go @@ -8,8 +8,6 @@ import ( "github.com/jmoiron/sqlx" "github.com/mattn/go-sqlite3" - - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) type ErrConflictingUserHandle struct { diff --git a/pkg/persistence/user_queries_test.go b/pkg/persistence/user_queries_test.go index adb6d31..6eee2e8 100644 --- a/pkg/persistence/user_queries_test.go +++ b/pkg/persistence/user_queries_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Create a user, save it, reload it, and make sure it comes back the same diff --git a/pkg/persistence/utils_test.go b/pkg/persistence/utils_test.go index 3fc0cdb..720f9bf 100644 --- a/pkg/persistence/utils_test.go +++ b/pkg/persistence/utils_test.go @@ -6,7 +6,6 @@ import ( "time" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) // Load a test profile, or create it if it doesn't exist. diff --git a/pkg/persistence/versions_test.go b/pkg/persistence/versions_test.go index 7bc54e7..2346df4 100644 --- a/pkg/persistence/versions_test.go +++ b/pkg/persistence/versions_test.go @@ -8,7 +8,6 @@ import ( "github.com/stretchr/testify/require" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" - . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) func TestVersionUpgrade(t *testing.T) { diff --git a/pkg/scraper/video.go b/pkg/persistence/video.go similarity index 97% rename from pkg/scraper/video.go rename to pkg/persistence/video.go index cbc9e68..08910c4 100644 --- a/pkg/scraper/video.go +++ b/pkg/persistence/video.go @@ -1,4 +1,4 @@ -package scraper +package persistence type VideoID int64 diff --git a/pkg/scraper/api_errors.go b/pkg/scraper/api_errors.go index 0c07e05..fb05e72 100644 --- a/pkg/scraper/api_errors.go +++ b/pkg/scraper/api_errors.go @@ -11,12 +11,9 @@ var ( EXTERNAL_API_ERROR = errors.New("Unexpected result from external API") ErrorIsTombstone = errors.New("tweet is a tombstone") ErrRateLimited = errors.New("rate limited") - ErrorDMCA = errors.New("video is DMCAed, unable to download (HTTP 403 Forbidden)") - ErrMediaDownload404 = errors.New("media download HTTP 404") ErrLoginRequired = errors.New("login required; please provide `--session ` flag") ErrSessionInvalidated = errors.New("session invalidated by Twitter") // These are not API errors, but network errors generally - ErrNoInternet = errors.New("no internet connection") - ErrRequestTimeout = errors.New("request timed out") + ErrNoInternet = errors.New("no internet connection") ) diff --git a/pkg/scraper/api_graphql_utils.go b/pkg/scraper/api_graphql_utils.go index 9ea3fac..ab63976 100644 --- a/pkg/scraper/api_graphql_utils.go +++ b/pkg/scraper/api_graphql_utils.go @@ -3,6 +3,8 @@ package scraper import ( "encoding/json" "net/url" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type GraphqlVariables struct { diff --git a/pkg/scraper/api_login.go b/pkg/scraper/api_login.go index f57fa2b..f637a79 100644 --- a/pkg/scraper/api_login.go +++ b/pkg/scraper/api_login.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "strings" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) const LOGIN_URL = "https://twitter.com/i/api/1.1/onboarding/task.json" diff --git a/pkg/scraper/api_request_utils.go b/pkg/scraper/api_request_utils.go index 594b02b..aca0a61 100644 --- a/pkg/scraper/api_request_utils.go +++ b/pkg/scraper/api_request_utils.go @@ -14,6 +14,8 @@ import ( "time" log "github.com/sirupsen/logrus" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type API struct { diff --git a/pkg/scraper/tweet_test.go b/pkg/scraper/api_tweet_test.go similarity index 99% rename from pkg/scraper/tweet_test.go rename to pkg/scraper/api_tweet_test.go index ecf30a8..d011eab 100644 --- a/pkg/scraper/tweet_test.go +++ b/pkg/scraper/api_tweet_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/scraper/api_tweet_trove_utils.go b/pkg/scraper/api_tweet_trove_utils.go index a91c1b0..a9ba606 100644 --- a/pkg/scraper/api_tweet_trove_utils.go +++ b/pkg/scraper/api_tweet_trove_utils.go @@ -5,6 +5,8 @@ import ( "fmt" log "github.com/sirupsen/logrus" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) func (api *API) FillSpaceDetails(trove *TweetTrove) error { diff --git a/pkg/scraper/api_types.go b/pkg/scraper/api_types.go index eb4f47b..429b621 100644 --- a/pkg/scraper/api_types.go +++ b/pkg/scraper/api_types.go @@ -12,6 +12,8 @@ import ( "strconv" "strings" "time" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) // ------------------------------------------------------------------------- @@ -533,8 +535,8 @@ func ParseSingleTweet(t APITweet) (ret Tweet, err error) { ret.IsConversationScraped = false // Safe due to the "No Worsening" principle // Extra data that can help piece together tombstoned tweet info - ret.in_reply_to_user_id = UserID(t.InReplyToUserID) - ret.in_reply_to_user_handle = UserHandle(t.InReplyToScreenName) + ret.InReplyToUserID = UserID(t.InReplyToUserID) + ret.InReplyToUserHandle = UserHandle(t.InReplyToScreenName) return } @@ -658,8 +660,8 @@ func ParseSingleUser(apiUser APIUser) (ret User, err error) { } ret.BannerImageUrl = apiUser.ProfileBannerURL - ret.ProfileImageLocalPath = ret.compute_profile_image_local_path() - ret.BannerImageLocalPath = ret.compute_banner_image_local_path() + ret.ProfileImageLocalPath = compute_profile_image_local_path(ret) + ret.BannerImageLocalPath = compute_banner_image_local_path(ret) if len(apiUser.PinnedTweetIdsStr) > 0 { ret.PinnedTweetID = TweetID(idstr_to_int(apiUser.PinnedTweetIdsStr[0])) diff --git a/pkg/scraper/api_types_dms.go b/pkg/scraper/api_types_dms.go index 0303de9..43dcf70 100644 --- a/pkg/scraper/api_types_dms.go +++ b/pkg/scraper/api_types_dms.go @@ -9,6 +9,8 @@ import ( "strings" "github.com/google/uuid" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type APIDMReaction struct { diff --git a/pkg/scraper/api_types_dms_test.go b/pkg/scraper/api_types_dms_test.go index 22dbb85..3437495 100644 --- a/pkg/scraper/api_types_dms_test.go +++ b/pkg/scraper/api_types_dms_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/scraper/api_types_lists.go b/pkg/scraper/api_types_lists.go index dc19fb5..d4de1d1 100644 --- a/pkg/scraper/api_types_lists.go +++ b/pkg/scraper/api_types_lists.go @@ -2,6 +2,8 @@ package scraper import ( "net/url" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) func (api *API) GetFolloweesPage(user_id UserID, cursor string) (APIV2Response, error) { diff --git a/pkg/scraper/api_types_notifications.go b/pkg/scraper/api_types_notifications.go index 50f5a62..1e19e21 100644 --- a/pkg/scraper/api_types_notifications.go +++ b/pkg/scraper/api_types_notifications.go @@ -10,6 +10,8 @@ import ( "time" log "github.com/sirupsen/logrus" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) // TODO: pagination diff --git a/pkg/scraper/api_types_notifications_test.go b/pkg/scraper/api_types_notifications_test.go index 2a68a00..15a43ab 100644 --- a/pkg/scraper/api_types_notifications_test.go +++ b/pkg/scraper/api_types_notifications_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/scraper/api_types_posting.go b/pkg/scraper/api_types_posting.go index ac2b1c0..788c4ca 100644 --- a/pkg/scraper/api_types_posting.go +++ b/pkg/scraper/api_types_posting.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "strings" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) var AlreadyLikedThisTweet error = errors.New("already liked this tweet") diff --git a/pkg/scraper/api_types_spaces.go b/pkg/scraper/api_types_spaces.go index 6d55a05..30eb31a 100644 --- a/pkg/scraper/api_types_spaces.go +++ b/pkg/scraper/api_types_spaces.go @@ -3,6 +3,8 @@ package scraper import ( "fmt" "net/url" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type SpaceResponse struct { diff --git a/pkg/scraper/api_types_spaces_test.go b/pkg/scraper/api_types_spaces_test.go index 49b85f7..6bda62b 100644 --- a/pkg/scraper/api_types_spaces_test.go +++ b/pkg/scraper/api_types_spaces_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/scraper/api_types_test.go b/pkg/scraper/api_types_test.go index b035b98..4529f9c 100644 --- a/pkg/scraper/api_types_test.go +++ b/pkg/scraper/api_types_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/scraper/api_types_user.go b/pkg/scraper/api_types_user.go index 9c96c27..b1bc396 100644 --- a/pkg/scraper/api_types_user.go +++ b/pkg/scraper/api_types_user.go @@ -4,6 +4,10 @@ import ( "errors" "fmt" "net/url" + "path/filepath" + "regexp" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type UserResponse struct { @@ -177,3 +181,25 @@ func (api API) GetUserByID(u_id UserID) (User, error) { } return ParseSingleUser(apiUser) } + +// Make a filename for the profile image, that hopefully won't clobber other ones +func compute_profile_image_local_path(u User) string { + return string(u.Handle) + "_profile_" + filepath.Base(u.ProfileImageUrl) +} + +// Make a filename for the banner image, that hopefully won't clobber other ones. +// Add a file extension if necessary (seems to be necessary). +// If there is no banner image, just return nothing. +func compute_banner_image_local_path(u User) string { + if u.BannerImageUrl == "" { + return "" + } + base_name := filepath.Base(u.BannerImageUrl) + + // Check if it has an extension (e.g., ".png" or ".jpeg") + if !regexp.MustCompile(`\.\w{2,4}$`).MatchString(base_name) { + // If it doesn't have an extension, add one + base_name += ".jpg" + } + return string(u.Handle) + "_banner_" + base_name +} diff --git a/pkg/scraper/api_types_user_test.go b/pkg/scraper/api_types_user_test.go index f766b7e..63b3afa 100644 --- a/pkg/scraper/api_types_user_test.go +++ b/pkg/scraper/api_types_user_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/scraper/api_types_v2.go b/pkg/scraper/api_types_v2.go index 3b1b001..0e9fd6b 100644 --- a/pkg/scraper/api_types_v2.go +++ b/pkg/scraper/api_types_v2.go @@ -10,6 +10,8 @@ import ( "time" log "github.com/sirupsen/logrus" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) type CardValue struct { @@ -777,10 +779,10 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) { panic(fmt.Sprintf("Tombstoned tweet has no ID (should be %d)", tweet.InReplyToID)) } - // Fill out the replied tweet's UserID using this tweet's "in_reply_to_user_id". + // Fill out the replied tweet's UserID using this tweet's "InReplyToUserID". // If this tweet doesn't have it (i.e., this tweet is also a tombstone), create a fake user instead, and add it to the tweet trove. if replied_tweet.UserID == 0 || replied_tweet.UserID == GetUnknownUser().ID { - replied_tweet.UserID = tweet.in_reply_to_user_id + replied_tweet.UserID = tweet.InReplyToUserID if replied_tweet.UserID == 0 || replied_tweet.UserID == GetUnknownUser().ID { fake_user := GetUnknownUser() ret.Users[fake_user.ID] = fake_user @@ -793,7 +795,7 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) { existing_user = User{ID: replied_tweet.UserID} } if existing_user.Handle == "" { - existing_user.Handle = tweet.in_reply_to_user_handle + existing_user.Handle = tweet.InReplyToUserHandle } ret.Users[replied_tweet.UserID] = existing_user ret.TombstoneUsers = append(ret.TombstoneUsers, existing_user.Handle) diff --git a/pkg/scraper/api_types_v2_test.go b/pkg/scraper/api_types_v2_test.go index 88bcb8a..7d752a0 100644 --- a/pkg/scraper/api_types_v2_test.go +++ b/pkg/scraper/api_types_v2_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/scraper/authentication_test.go b/pkg/scraper/authentication_test.go index 3216cf7..4f4b32b 100644 --- a/pkg/scraper/authentication_test.go +++ b/pkg/scraper/authentication_test.go @@ -12,6 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/scraper" ) diff --git a/pkg/scraper/link_expander.go b/pkg/scraper/link_expander.go index 11943d7..4908b81 100644 --- a/pkg/scraper/link_expander.go +++ b/pkg/scraper/link_expander.go @@ -6,6 +6,8 @@ import ( "net/url" "regexp" "time" + + . "gitlab.com/offline-twitter/twitter_offline_engine/pkg/persistence" ) /**