diff --git a/go.mod b/go.mod index a2381b0..409063c 100644 --- a/go.mod +++ b/go.mod @@ -9,5 +9,6 @@ require ( github.com/mattn/go-sqlite3 v1.14.7 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 + golang.org/x/sys v0.2.0 // indirect gopkg.in/yaml.v2 v2.4.0 ) diff --git a/go.sum b/go.sum index 9579380..1c27f6b 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= diff --git a/persistence/media_download.go b/persistence/media_download.go index 78a5fe9..164e1f1 100644 --- a/persistence/media_download.go +++ b/persistence/media_download.go @@ -1,6 +1,8 @@ package persistence import ( + "encoding/json" + "errors" "fmt" "io" "net/http" @@ -17,6 +19,8 @@ type MediaDownloader interface { type DefaultDownloader struct{} +var ErrorDCMA error = errors.New("Error Video is DCMAed, unable to download (HTTP 403 Forbidden)") + /** * Download a file over HTTP and save it. * @@ -30,6 +34,27 @@ func (d DefaultDownloader) Curl(url string, outpath string) error { if err != nil { return fmt.Errorf("Error executing HTTP GET(%q):\n %w", url, err) } + + if resp.StatusCode == 403 { + var response struct { + Error_response string `json:"error_response"` + } + body, err := io.ReadAll(resp.Body) + fmt.Println("body = " + string(body)) + + if err != nil { + panic(err) + } + + json.Unmarshal(body, &response) + + if response.Error_response == "Dmcaed" { + return ErrorDCMA + } + + return fmt.Errorf("Error 403 Forbidden %s: %s", url, resp.Status) + } + if resp.StatusCode != 200 { return fmt.Errorf("Error %s: %s", url, resp.Status) } @@ -76,18 +101,23 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload // Download the video outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename) err := downloader.Curl(v.RemoteURL, outfile) - if err != nil { + + if err == ErrorDCMA { + v.IsDownloaded = false //Would need to change the database schema / or add a flag + } else if err != nil { return fmt.Errorf("Error downloading video (TweetID %d):\n %w", v.TweetID, err) + } else { + v.IsDownloaded = true } // Download the thumbnail outfile = path.Join(p.ProfileDir, "video_thumbnails", v.ThumbnailLocalPath) err = downloader.Curl(v.ThumbnailRemoteUrl, outfile) if err != nil { + v.IsDownloaded = false return fmt.Errorf("Error downloading video thumbnail (TweetID %d):\n %w", v.TweetID, err) } - v.IsDownloaded = true return p.SaveVideo(*v) }