Add link expander

scraper/link_expander.go

@@ -0,0 +1,35 @@
+package scraper
+
+import (
+	"fmt"
+	"time"
+	"net/http"
+)
+
+
+/**
+ * Return the expanded version of a short URL.  Input must be a real short URL.
+ */
+func ExpandShortUrl(short_url string) string {
+	// Create a client that doesn't follow redirects
+	client := &http.Client{
+		Timeout: 5 * time.Second,
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+
+	resp, err := client.Get(short_url)
+	if err != nil {
+		panic(err)  // TODO: handle timeouts
+	}
+	if resp.StatusCode != 301 {
+		panic(fmt.Sprintf("Unknown status code returned when expanding short url %q: %s", short_url, resp.Status))
+	}
+
+	long_url := resp.Header.Get("Location")
+	if long_url == "" {
+		panic(fmt.Sprintf("Header didn't have a Location field for short url %q", short_url))
+	}
+	return long_url
+}

scraper/link_expander_test.go

@@ -0,0 +1,25 @@
+package scraper_test
+
+import (
+	"testing"
+
+	"net/http"
+	"net/http/httptest"
+
+	"offline_twitter/scraper"
+)
+
+
+func TestExpandShortUrl(t *testing.T) {
+	redirecting_to := "redirect target"
+	srvr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		w.Header().Set("Location", redirecting_to)
+		w.WriteHeader(301)
+	}))
+	defer srvr.Close()
+
+	result := scraper.ExpandShortUrl(srvr.URL)
+	if result != redirecting_to {
+		t.Errorf("Expected %q, got %q", redirecting_to, result)
+	}
+}