Initial commit

doc/curl requests

@@ -0,0 +1,15 @@
+curl -X POST \
+    -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \
+    https://api.twitter.com/1.1/guest/activate.json
+
+
+curl \
+    -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \
+    -H "X-Guest-Token: 1391174194361217029" \
+    https://api.twitter.com/graphql/4S2ihIKfF3xhp-ENxvUAfQ/UserByScreenName?variables=%7B%22screen_name%22%3A%22michaelmalice%22%2C%22withHighlightedLabel%22%3Atrue%7D
+
+
+curl \
+    -H "Authorization: Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA" \
+    -H "X-Guest-Token: 1391214296126967816" \
+    https://api.twitter.com/2/timeline/profile/44067298.json

scraper/constants.go

@@ -0,0 +1,6 @@
+package scraper
+
+// Tokens
+// ------
+
+const BEARER_TOKEN string = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"

scraper/guest_token.go

@@ -0,0 +1,53 @@
+package scraper
+
+import "fmt"
+import "time"
+
+import "io/ioutil"
+import "net/http"
+import "encoding/json"
+
+type GuestTokenResponse struct {
+	Token       string `json:"guest_token"`
+	RefreshedAt time.Time
+}
+
+var guestToken GuestTokenResponse
+
+func GetGuestToken() (string, error) {
+	if time.Since(guestToken.RefreshedAt).Hours() < 1 {
+		return guestToken.Token, nil
+	}
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	req, err := http.NewRequest("POST", "https://api.twitter.com/1.1/guest/activate.json", nil)
+	if err != nil {
+		return "", err
+	}
+	req.Header.Set("Authorization", "Bearer " + BEARER_TOKEN)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", err
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		content, _ := ioutil.ReadAll(resp.Body)
+		return "", fmt.Errorf("HTTP %s: %s", resp.Status, content)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	err = json.Unmarshal(body, &guestToken)
+	if err != nil {
+		return "", err
+	}
+
+	guestToken.RefreshedAt = time.Now()
+	return guestToken.Token, nil
+}

scraper/guest_token_test.go

@@ -0,0 +1,27 @@
+package scraper_test
+
+import "testing"
+import "fmt"
+import "offline_twitter/scraper"
+
+// Makes an HTTP request
+func TestGetGuestToken(t *testing.T) {
+	token, err := scraper.GetGuestToken()
+	if err != nil {
+		t.Errorf("%v", err)
+	}
+
+	if len(token) < 15 {
+		t.Errorf("I don't think this is a token: %q", token)
+	}
+	fmt.Println(token)
+}
+
+
+// Tests the caching.  Should run much much faster than an HTTP request, since all requests
+// other than the first use the cache.
+func BenchmarkGetGuestToken(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		scraper.GetGuestToken()
+	}
+}