some refactoring and tests

doc/login.py

@@ -22,8 +22,9 @@ login_curl = "https://twitter.com/i/api/1.1/onboarding/task.json"
 username = "offline_twatter"
 password = "S1pKIW#eRT016iA@OFcK"
 
-response = requests.post(login_curl, headers=headers, params={"flow_name": "login"})
+response = requests.post(login_curl, headers=headers, params={"flow_name": "login"}, json={"input_flow_data":{"flow_context":{"debug_overrides":{},"start_location":{"location":"unknown"}}},"subtask_versions":{"action_list":2,"alert_dialog":1,"app_download_cta":1,"check_logged_in_account":1,"choice_selection":3,"contacts_live_sync_permission_prompt":0,"cta":7,"email_verification":2,"end_flow":1,"enter_date":1,"enter_email":2,"enter_password":5,"enter_phone":2,"enter_recaptcha":1,"enter_text":5,"enter_username":2,"generic_urt":3,"in_app_notification":1,"interest_picker":3,"js_instrumentation":1,"menu_dialog":1,"notifications_permission_prompt":2,"open_account":2,"open_home_timeline":1,"open_link":1,"phone_verification":4,"privacy_options":1,"security_key":3,"select_avatar":4,"select_banner":2,"settings_list":7,"show_code":1,"sign_up":2,"sign_up_review":4,"tweet_selection_urt":1,"update_users":1,"upload_media":1,"user_recommendations_list":4,"user_recommendations_urt":1,"wait_spinner":3,"web_modal":1}})
-assert response.status_code == 200, f"HTTP Response code {response.status_code}"
+
+assert response.status_code == 200, f"HTTP Response code {response.status_code}{response.json()}"
 flow_token = response.json()['flow_token']
 
 second_request_data = {
@@ -32,7 +33,7 @@ second_request_data = {
         {
             "subtask_id": "LoginJsInstrumentationSubtask",
             "js_instrumentation": {
-                "response": "{\"rf\":{\"a560cdc18ff70ce7662311eac0f2441dd3d3ed27c354f082f587e7a30d1a7d5f\":72,\"a8e890b5fec154e7af62d8f529fbec5942dfdd7ad41597245b71a3fbdc9a180d\":176,\"a3c24597ad4c862773c74b9194e675e96a7607708f6bbd0babcfdf8b109ed86d\":-161,\"af9847e2cd4e9a0ca23853da4b46bf00a2e801f98dc819ee0dd6ecc1032273fa\":-8},\"s\":\"hOai7h2KQi4RBGKSYLUhH0Y0fBm5KHIJgxD5AmNKtwP7N8gpVuAqP8o9n2FpCnNeR1d6XbB0QWkGAHiXkKao5PhaeXEZgPJU1neLcVgTnGuFzpjDnGutCUgYaxNiwUPfDX0eQkgr_q7GWmbB7yyYPt32dqSd5yt-KCpSt7MOG4aFmGf11xWE4MTpXfkefbnX4CwZeEFKQQYzJptOvmUWa7qI0A69BSOs7HZ_4Wry2TwB9k03Q_S-MDZAZ3yB_L7WoosVVb1e84YWgaLWWzqhz4C77jDy6isT8EKSWKWnVctsIcaqM_wMV8AiYa5lr0_WkN5TwK9h0vDOTS1obOZuhAAAAYTZan_3\"}",  # pylint: disable=line-too-long
+                "response": "{\"rf\":{\"cbd6bdbb6add3e20bbda71c0cd9f43a2f00533d3d6549a7ccb89c4c06901dce2\":1,\"ae977661f2a886f4ffacdd57540338e2b7aef11c4113806eceae3d22055aa8e7\":-149,\"dd84b8c06765cf9bf1fb433be2f59155facb2873a28c957297de3a1e993494fb\":-143,\"ab1534f3890a2597699a4594ad9f54e8432fed1291abfd96c892620f49baa907\":-184},\"s\":\"XpY8TxYmi24ixccqjW68xUdUsWJhmMJgDdf0oEA99ufun62H3AJRHJT1f2-gsxrCa289ZjcvOrXC7C5miGlWlofiwpF-nK7bIH1jCW_Jp6_9NiXaGH151Kt3ChCfqwYNv7gROBkyXOVMXxV2I8WZ_WF1Da1r2DlMVK9DosRhZHGJUhYDtJRhn65gi5Xd73z8MjOWODXsDMxm_urFU5bY68Arf2D1oUJcZ70jhjMYV0yU09249xWiXIs81n0i_44dYqWV2tuYFkdU7kSazjYz4VGZ4P70l1k_MwUuI6dbueK9-R1RBRszGNle0kVZmpJNmVocc3j3TMxOxwso29_cEwAAAYUm0EPM\"}",
                 "link": "next_link"
             }
         }
@@ -40,7 +41,7 @@ second_request_data = {
 }
 
 response2 = requests.post(login_curl, headers=headers, json=second_request_data)
-assert response2.status_code == 200, f"HTTP Response code {response2.status_code}"
+assert response2.status_code == 200, f"HTTP Response code {response2.status_code}{response2.json()}"
 flow_token = response2.json()["flow_token"]
 
 
@@ -67,7 +68,7 @@ third_request_data = {
 }
 
 response3 = requests.post(login_curl, headers=headers, json=third_request_data)
-assert response3.status_code == 200, f"HTTP Response code {response3.status_code}"
+assert response3.status_code == 200, f"HTTP Response code {response3.status_code}{response3.json()}"
 flow_token = response3.json()["flow_token"]
 
 fourth_request_data = {

scraper/api_request_utils.go

@@ -19,6 +19,8 @@ type API struct {
 	IsAuthenticated     bool
 	GuestToken          string
 	AuthenticationToken string
+	Client              http.Client
+	CSRFToken           string
 }
 
 func NewGuestSession() API {
@@ -31,6 +33,8 @@ func NewGuestSession() API {
 		IsAuthenticated:     false,
 		GuestToken:          guestAPIString,
 		AuthenticationToken: "",
+		Client:              http.Client{Timeout: 10 * time.Second},
+		CSRFToken:           "",
 	}
 }
 
@@ -104,11 +108,26 @@ func (api *API) LogIn(username string, password string) {
 		panic(err)
 	}
 
-	panic("TODO")
+	dummyURL, err := url.Parse(loginURL)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, cookie := range api.Client.Jar.Cookies(dummyURL) {
+		if cookie.Name == "ct0" {
+			api.CSRFToken = cookie.Value
+		}
+	}
+
+	if api.CSRFToken == "" {
+		panic("No CSRF Token Found")
+	}
+
+	api.IsAuthenticated = true
+
 }
 
 func (api *API) do_http_POST(url string, body string, result interface{}) error {
-	client := &http.Client{Timeout: 10 * time.Second}
 	req, err := http.NewRequest("POST", url, strings.NewReader(body))
 	if err != nil {
 		return fmt.Errorf("Error initializing HTTP POST request:\n  %w", err)
@@ -129,7 +148,7 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error
 		req.Header.Set("X-Guest-Token", api.GuestToken)
 	}
 
-	resp, err := client.Do(req)
+	resp, err := api.Client.Do(req)
 	if err != nil {
 		return fmt.Errorf("Error executing HTTP POST request:\n  %w", err)
 	}
@@ -163,7 +182,6 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error
 }
 
 func (api API) do_http(url string, cursor string, result interface{}) error {
-	client := &http.Client{Timeout: 10 * time.Second}
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {
 		return fmt.Errorf("Error initializing HTTP GET request:\n  %w", err)
@@ -189,7 +207,7 @@ func (api API) do_http(url string, cursor string, result interface{}) error {
 		req.Header.Set("X-Guest-Token", api.GuestToken)
 	}
 
-	resp, err := client.Do(req)
+	resp, err := api.Client.Do(req)
 	if err != nil {
 		return fmt.Errorf("Error executing HTTP request:\n  %w", err)
 	}

scraper/authentication_test.go

@@ -3,12 +3,20 @@ package scraper_test
 import (
 	"offline_twitter/scraper"
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestAuthentication(t *testing.T) {
+	assert := assert.New(t)
+
 	username := "offline_twatter"
 	password := "S1pKIW#eRT016iA@OFcK"
 
 	api := scraper.NewGuestSession()
 	api.LogIn(username, password)
+
+	assert.True(api.IsAuthenticated)
+	assert.NotEqual(api.CSRFToken, "")
+
 }