diff --git a/doc/login.py b/doc/login.py index c3bf917..1197ebc 100644 --- a/doc/login.py +++ b/doc/login.py @@ -22,8 +22,9 @@ login_curl = "https://twitter.com/i/api/1.1/onboarding/task.json" username = "offline_twatter" password = "S1pKIW#eRT016iA@OFcK" -response = requests.post(login_curl, headers=headers, params={"flow_name": "login"}) -assert response.status_code == 200, f"HTTP Response code {response.status_code}" +response = requests.post(login_curl, headers=headers, params={"flow_name": "login"}, json={"input_flow_data":{"flow_context":{"debug_overrides":{},"start_location":{"location":"unknown"}}},"subtask_versions":{"action_list":2,"alert_dialog":1,"app_download_cta":1,"check_logged_in_account":1,"choice_selection":3,"contacts_live_sync_permission_prompt":0,"cta":7,"email_verification":2,"end_flow":1,"enter_date":1,"enter_email":2,"enter_password":5,"enter_phone":2,"enter_recaptcha":1,"enter_text":5,"enter_username":2,"generic_urt":3,"in_app_notification":1,"interest_picker":3,"js_instrumentation":1,"menu_dialog":1,"notifications_permission_prompt":2,"open_account":2,"open_home_timeline":1,"open_link":1,"phone_verification":4,"privacy_options":1,"security_key":3,"select_avatar":4,"select_banner":2,"settings_list":7,"show_code":1,"sign_up":2,"sign_up_review":4,"tweet_selection_urt":1,"update_users":1,"upload_media":1,"user_recommendations_list":4,"user_recommendations_urt":1,"wait_spinner":3,"web_modal":1}}) + +assert response.status_code == 200, f"HTTP Response code {response.status_code}{response.json()}" flow_token = response.json()['flow_token'] second_request_data = { @@ -32,7 +33,7 @@ second_request_data = { { "subtask_id": "LoginJsInstrumentationSubtask", "js_instrumentation": { - "response": "{\"rf\":{\"a560cdc18ff70ce7662311eac0f2441dd3d3ed27c354f082f587e7a30d1a7d5f\":72,\"a8e890b5fec154e7af62d8f529fbec5942dfdd7ad41597245b71a3fbdc9a180d\":176,\"a3c24597ad4c862773c74b9194e675e96a7607708f6bbd0babcfdf8b109ed86d\":-161,\"af9847e2cd4e9a0ca23853da4b46bf00a2e801f98dc819ee0dd6ecc1032273fa\":-8},\"s\":\"hOai7h2KQi4RBGKSYLUhH0Y0fBm5KHIJgxD5AmNKtwP7N8gpVuAqP8o9n2FpCnNeR1d6XbB0QWkGAHiXkKao5PhaeXEZgPJU1neLcVgTnGuFzpjDnGutCUgYaxNiwUPfDX0eQkgr_q7GWmbB7yyYPt32dqSd5yt-KCpSt7MOG4aFmGf11xWE4MTpXfkefbnX4CwZeEFKQQYzJptOvmUWa7qI0A69BSOs7HZ_4Wry2TwB9k03Q_S-MDZAZ3yB_L7WoosVVb1e84YWgaLWWzqhz4C77jDy6isT8EKSWKWnVctsIcaqM_wMV8AiYa5lr0_WkN5TwK9h0vDOTS1obOZuhAAAAYTZan_3\"}", # pylint: disable=line-too-long + "response": "{\"rf\":{\"cbd6bdbb6add3e20bbda71c0cd9f43a2f00533d3d6549a7ccb89c4c06901dce2\":1,\"ae977661f2a886f4ffacdd57540338e2b7aef11c4113806eceae3d22055aa8e7\":-149,\"dd84b8c06765cf9bf1fb433be2f59155facb2873a28c957297de3a1e993494fb\":-143,\"ab1534f3890a2597699a4594ad9f54e8432fed1291abfd96c892620f49baa907\":-184},\"s\":\"XpY8TxYmi24ixccqjW68xUdUsWJhmMJgDdf0oEA99ufun62H3AJRHJT1f2-gsxrCa289ZjcvOrXC7C5miGlWlofiwpF-nK7bIH1jCW_Jp6_9NiXaGH151Kt3ChCfqwYNv7gROBkyXOVMXxV2I8WZ_WF1Da1r2DlMVK9DosRhZHGJUhYDtJRhn65gi5Xd73z8MjOWODXsDMxm_urFU5bY68Arf2D1oUJcZ70jhjMYV0yU09249xWiXIs81n0i_44dYqWV2tuYFkdU7kSazjYz4VGZ4P70l1k_MwUuI6dbueK9-R1RBRszGNle0kVZmpJNmVocc3j3TMxOxwso29_cEwAAAYUm0EPM\"}", "link": "next_link" } } @@ -40,7 +41,7 @@ second_request_data = { } response2 = requests.post(login_curl, headers=headers, json=second_request_data) -assert response2.status_code == 200, f"HTTP Response code {response2.status_code}" +assert response2.status_code == 200, f"HTTP Response code {response2.status_code}{response2.json()}" flow_token = response2.json()["flow_token"] @@ -67,7 +68,7 @@ third_request_data = { } response3 = requests.post(login_curl, headers=headers, json=third_request_data) -assert response3.status_code == 200, f"HTTP Response code {response3.status_code}" +assert response3.status_code == 200, f"HTTP Response code {response3.status_code}{response3.json()}" flow_token = response3.json()["flow_token"] fourth_request_data = { diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index 02d859f..9eb0a8e 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -19,6 +19,8 @@ type API struct { IsAuthenticated bool GuestToken string AuthenticationToken string + Client http.Client + CSRFToken string } func NewGuestSession() API { @@ -31,6 +33,8 @@ func NewGuestSession() API { IsAuthenticated: false, GuestToken: guestAPIString, AuthenticationToken: "", + Client: http.Client{Timeout: 10 * time.Second}, + CSRFToken: "", } } @@ -104,11 +108,26 @@ func (api *API) LogIn(username string, password string) { panic(err) } - panic("TODO") + dummyURL, err := url.Parse(loginURL) + if err != nil { + panic(err) + } + + for _, cookie := range api.Client.Jar.Cookies(dummyURL) { + if cookie.Name == "ct0" { + api.CSRFToken = cookie.Value + } + } + + if api.CSRFToken == "" { + panic("No CSRF Token Found") + } + + api.IsAuthenticated = true + } func (api *API) do_http_POST(url string, body string, result interface{}) error { - client := &http.Client{Timeout: 10 * time.Second} req, err := http.NewRequest("POST", url, strings.NewReader(body)) if err != nil { return fmt.Errorf("Error initializing HTTP POST request:\n %w", err) @@ -129,7 +148,7 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error req.Header.Set("X-Guest-Token", api.GuestToken) } - resp, err := client.Do(req) + resp, err := api.Client.Do(req) if err != nil { return fmt.Errorf("Error executing HTTP POST request:\n %w", err) } @@ -163,7 +182,6 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error } func (api API) do_http(url string, cursor string, result interface{}) error { - client := &http.Client{Timeout: 10 * time.Second} req, err := http.NewRequest("GET", url, nil) if err != nil { return fmt.Errorf("Error initializing HTTP GET request:\n %w", err) @@ -189,7 +207,7 @@ func (api API) do_http(url string, cursor string, result interface{}) error { req.Header.Set("X-Guest-Token", api.GuestToken) } - resp, err := client.Do(req) + resp, err := api.Client.Do(req) if err != nil { return fmt.Errorf("Error executing HTTP request:\n %w", err) } diff --git a/scraper/authentication_test.go b/scraper/authentication_test.go index 96f6dd6..93def4b 100644 --- a/scraper/authentication_test.go +++ b/scraper/authentication_test.go @@ -3,12 +3,20 @@ package scraper_test import ( "offline_twitter/scraper" "testing" + + "github.com/stretchr/testify/assert" ) func TestAuthentication(t *testing.T) { + assert := assert.New(t) + username := "offline_twatter" password := "S1pKIW#eRT016iA@OFcK" api := scraper.NewGuestSession() api.LogIn(username, password) + + assert.True(api.IsAuthenticated) + assert.NotEqual(api.CSRFToken, "") + }