me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

some refactoring and tests

Browse Source
This commit is contained in:
Jaeger Aquila 2022-12-20 19:05:08 -05:00 committed by Alessio
parent 140024e903
commit 2cf64db93e
3 changed files with 37 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
doc/login.py
View File

@ -22,8 +22,9 @@ login_curl = "https://twitter.com/i/api/1.1/onboarding/task.json"
username = "offline_twatter"
password = "S1pKIW#eRT016iA@OFcK"
response = requests.post(login_curl, headers=headers, params={"flow_name": "login"})
assert response.status_code == 200, f"HTTP Response code {response.status_code}"
response = requests.post(login_curl, headers=headers, params={"flow_name": "login"}, json={"input_flow_data":{"flow_context":{"debug_overrides":{},"start_location":{"location":"unknown"}}},"subtask_versions":{"action_list":2,"alert_dialog":1,"app_download_cta":1,"check_logged_in_account":1,"choice_selection":3,"contacts_live_sync_permission_prompt":0,"cta":7,"email_verification":2,"end_flow":1,"enter_date":1,"enter_email":2,"enter_password":5,"enter_phone":2,"enter_recaptcha":1,"enter_text":5,"enter_username":2,"generic_urt":3,"in_app_notification":1,"interest_picker":3,"js_instrumentation":1,"menu_dialog":1,"notifications_permission_prompt":2,"open_account":2,"open_home_timeline":1,"open_link":1,"phone_verification":4,"privacy_options":1,"security_key":3,"select_avatar":4,"select_banner":2,"settings_list":7,"show_code":1,"sign_up":2,"sign_up_review":4,"tweet_selection_urt":1,"update_users":1,"upload_media":1,"user_recommendations_list":4,"user_recommendations_urt":1,"wait_spinner":3,"web_modal":1}})
assert response.status_code == 200, f"HTTP Response code {response.status_code}{response.json()}"
flow_token = response.json()['flow_token']
second_request_data = {
@ -32,7 +33,7 @@ second_request_data = {
{
"subtask_id": "LoginJsInstrumentationSubtask",
"js_instrumentation": {
"response": "{\"rf\":{\"a560cdc18ff70ce7662311eac0f2441dd3d3ed27c354f082f587e7a30d1a7d5f\":72,\"a8e890b5fec154e7af62d8f529fbec5942dfdd7ad41597245b71a3fbdc9a180d\":176,\"a3c24597ad4c862773c74b9194e675e96a7607708f6bbd0babcfdf8b109ed86d\":-161,\"af9847e2cd4e9a0ca23853da4b46bf00a2e801f98dc819ee0dd6ecc1032273fa\":-8},\"s\":\"hOai7h2KQi4RBGKSYLUhH0Y0fBm5KHIJgxD5AmNKtwP7N8gpVuAqP8o9n2FpCnNeR1d6XbB0QWkGAHiXkKao5PhaeXEZgPJU1neLcVgTnGuFzpjDnGutCUgYaxNiwUPfDX0eQkgr_q7GWmbB7yyYPt32dqSd5yt-KCpSt7MOG4aFmGf11xWE4MTpXfkefbnX4CwZeEFKQQYzJptOvmUWa7qI0A69BSOs7HZ_4Wry2TwB9k03Q_S-MDZAZ3yB_L7WoosVVb1e84YWgaLWWzqhz4C77jDy6isT8EKSWKWnVctsIcaqM_wMV8AiYa5lr0_WkN5TwK9h0vDOTS1obOZuhAAAAYTZan_3\"}", # pylint: disable=line-too-long
"response": "{\"rf\":{\"cbd6bdbb6add3e20bbda71c0cd9f43a2f00533d3d6549a7ccb89c4c06901dce2\":1,\"ae977661f2a886f4ffacdd57540338e2b7aef11c4113806eceae3d22055aa8e7\":-149,\"dd84b8c06765cf9bf1fb433be2f59155facb2873a28c957297de3a1e993494fb\":-143,\"ab1534f3890a2597699a4594ad9f54e8432fed1291abfd96c892620f49baa907\":-184},\"s\":\"XpY8TxYmi24ixccqjW68xUdUsWJhmMJgDdf0oEA99ufun62H3AJRHJT1f2-gsxrCa289ZjcvOrXC7C5miGlWlofiwpF-nK7bIH1jCW_Jp6_9NiXaGH151Kt3ChCfqwYNv7gROBkyXOVMXxV2I8WZ_WF1Da1r2DlMVK9DosRhZHGJUhYDtJRhn65gi5Xd73z8MjOWODXsDMxm_urFU5bY68Arf2D1oUJcZ70jhjMYV0yU09249xWiXIs81n0i_44dYqWV2tuYFkdU7kSazjYz4VGZ4P70l1k_MwUuI6dbueK9-R1RBRszGNle0kVZmpJNmVocc3j3TMxOxwso29_cEwAAAYUm0EPM\"}",
"link": "next_link"
}
}
@ -40,7 +41,7 @@ second_request_data = {
}
response2 = requests.post(login_curl, headers=headers, json=second_request_data)
assert response2.status_code == 200, f"HTTP Response code {response2.status_code}"
assert response2.status_code == 200, f"HTTP Response code {response2.status_code}{response2.json()}"
flow_token = response2.json()["flow_token"]
@ -67,7 +68,7 @@ third_request_data = {
}
response3 = requests.post(login_curl, headers=headers, json=third_request_data)
assert response3.status_code == 200, f"HTTP Response code {response3.status_code}"
assert response3.status_code == 200, f"HTTP Response code {response3.status_code}{response3.json()}"
flow_token = response3.json()["flow_token"]
fourth_request_data = {

28
scraper/api_request_utils.go
View File

@ -19,6 +19,8 @@ type API struct {
IsAuthenticated bool
GuestToken string
AuthenticationToken string
Client http.Client
CSRFToken string
}
func NewGuestSession() API {
@ -31,6 +33,8 @@ func NewGuestSession() API {
IsAuthenticated: false,
GuestToken: guestAPIString,
AuthenticationToken: "",
Client: http.Client{Timeout: 10 * time.Second},
CSRFToken: "",
}
}
@ -104,11 +108,26 @@ func (api *API) LogIn(username string, password string) {
panic(err)
}
panic("TODO")
dummyURL, err := url.Parse(loginURL)
if err != nil {
panic(err)
}
for _, cookie := range api.Client.Jar.Cookies(dummyURL) {
if cookie.Name == "ct0" {
api.CSRFToken = cookie.Value
}
}
if api.CSRFToken == "" {
panic("No CSRF Token Found")
}
api.IsAuthenticated = true
}
func (api *API) do_http_POST(url string, body string, result interface{}) error {
client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("POST", url, strings.NewReader(body))
if err != nil {
return fmt.Errorf("Error initializing HTTP POST request:\n %w", err)
@ -129,7 +148,7 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error
req.Header.Set("X-Guest-Token", api.GuestToken)
}
resp, err := client.Do(req)
resp, err := api.Client.Do(req)
if err != nil {
return fmt.Errorf("Error executing HTTP POST request:\n %w", err)
}
@ -163,7 +182,6 @@ func (api *API) do_http_POST(url string, body string, result interface{}) error
}
func (api API) do_http(url string, cursor string, result interface{}) error {
client := &http.Client{Timeout: 10 * time.Second}
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return fmt.Errorf("Error initializing HTTP GET request:\n %w", err)
@ -189,7 +207,7 @@ func (api API) do_http(url string, cursor string, result interface{}) error {
req.Header.Set("X-Guest-Token", api.GuestToken)
}
resp, err := client.Do(req)
resp, err := api.Client.Do(req)
if err != nil {
return fmt.Errorf("Error executing HTTP request:\n %w", err)
}

8
scraper/authentication_test.go
View File

@ -3,12 +3,20 @@ package scraper_test
import (
"offline_twitter/scraper"
"testing"
"github.com/stretchr/testify/assert"
)
func TestAuthentication(t *testing.T) {
assert := assert.New(t)
username := "offline_twatter"
password := "S1pKIW#eRT016iA@OFcK"
api := scraper.NewGuestSession()
api.LogIn(username, password)
assert.True(api.IsAuthenticated)
assert.NotEqual(api.CSRFToken, "")
}