From 1b3c5d0ed3c9f34b48b8f8ce97c5e52922b7cf20 Mon Sep 17 00:00:00 2001
From: Alessio <alex@playfulpachyderm.com>
Date: Sat, 13 Apr 2024 16:10:23 -0700
Subject: [PATCH] Add timeout error handling for scraper requests to the
 request body download as well (rather than just headers)

---
 pkg/scraper/api_request_utils.go | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/pkg/scraper/api_request_utils.go b/pkg/scraper/api_request_utils.go
index 334ca4d..b277a22 100644
--- a/pkg/scraper/api_request_utils.go
+++ b/pkg/scraper/api_request_utils.go
@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"net"
 	"net/http"
 	"net/http/cookiejar"
 	"net/url"
@@ -147,7 +148,14 @@ func (api *API) update_csrf_token() {
 
 func is_timeout(err error) bool {
 	var urlErr *url.Error
-	return errors.As(err, &urlErr) && urlErr.Timeout()
+	if errors.As(err, &urlErr) {
+		return urlErr.Timeout()
+	}
+	var netErr net.Error
+	if errors.As(err, &netErr) {
+		return netErr.Timeout()
+	}
+	return false
 }
 
 func (api *API) do_http_POST(remote_url string, body string, result interface{}) error {
@@ -177,7 +185,9 @@ func (api *API) do_http_POST(remote_url string, body string, result interface{})
 	defer resp.Body.Close()
 
 	respBody, err := io.ReadAll(resp.Body)
-	if err != nil {
+	if is_timeout(err) {
+		return fmt.Errorf("GET %q:\n  reading response body:\n  %w", remote_url, ErrRequestTimeout)
+	} else if err != nil {
 		panic(err)
 	}
 
@@ -237,7 +247,9 @@ func (api *API) do_http(remote_url string, cursor string, result interface{}) er
 	}
 
 	body, err := io.ReadAll(resp.Body)
-	if err != nil {
+	if is_timeout(err) {
+		return fmt.Errorf("GET %q:\n  reading response body:\n  %w", remote_url, ErrRequestTimeout)
+	} else if err != nil {
 		panic(err)
 	}
 
@@ -397,7 +409,7 @@ func (api *API) DownloadMedia(remote_url string) ([]byte, error) {
 
 	resp, err := api.Client.Do(req)
 	if is_timeout(err) {
-		return []byte{}, fmt.Errorf("GET %q:\n  %w", remote_url, ErrRequestTimeout)
+		return []byte{}, fmt.Errorf("GET %q:\n  waiting for headers:\n  %w", remote_url, ErrRequestTimeout)
 	} else if err != nil {
 		return []byte{}, fmt.Errorf("Error executing HTTP request:\n  %w", err)
 	}
@@ -409,7 +421,9 @@ func (api *API) DownloadMedia(remote_url string) ([]byte, error) {
 	}
 
 	body, err := io.ReadAll(resp.Body)
-	if err != nil {
+	if is_timeout(err) {
+		return []byte{}, fmt.Errorf("GET %q:\n  reading response body:\n  %w", remote_url, ErrRequestTimeout)
+	} else if err != nil {
 		panic(err)
 	}