fix for dmca video downloads

go.mod

@@ -9,5 +9,6 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.7
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
+	golang.org/x/sys v0.2.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0
 )

go.sum

@@ -24,6 +24,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
+golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

persistence/media_download.go

@@ -1,6 +1,8 @@
 package persistence
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -17,6 +19,8 @@ type MediaDownloader interface {
 
 type DefaultDownloader struct{}
 
+var ErrorDCMA error = errors.New("Error Video is DCMAed, unable to download (HTTP 403 Forbidden)")
+
 /**
  * Download a file over HTTP and save it.
  *
@@ -30,6 +34,27 @@ func (d DefaultDownloader) Curl(url string, outpath string) error {
 	if err != nil {
 		return fmt.Errorf("Error executing HTTP GET(%q):\n  %w", url, err)
 	}
+
+	if resp.StatusCode == 403 {
+		var response struct {
+			Error_response string `json:"error_response"`
+		}
+		body, err := io.ReadAll(resp.Body)
+		fmt.Println("body = " + string(body))
+
+		if err != nil {
+			panic(err)
+		}
+
+		json.Unmarshal(body, &response)
+
+		if response.Error_response == "Dmcaed" {
+			return ErrorDCMA
+		}
+
+		return fmt.Errorf("Error 403 Forbidden %s: %s", url, resp.Status)
+	}
+
 	if resp.StatusCode != 200 {
 		return fmt.Errorf("Error %s: %s", url, resp.Status)
 	}
@@ -76,18 +101,23 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload
 	// Download the video
 	outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename)
 	err := downloader.Curl(v.RemoteURL, outfile)
-	if err != nil {
+
+	if err == ErrorDCMA {
+		v.IsDownloaded = false //Would need to change the database schema / or add a flag
+	} else if err != nil {
 		return fmt.Errorf("Error downloading video (TweetID %d):\n  %w", v.TweetID, err)
+	} else {
+		v.IsDownloaded = true
 	}
 
 	// Download the thumbnail
 	outfile = path.Join(p.ProfileDir, "video_thumbnails", v.ThumbnailLocalPath)
 	err = downloader.Curl(v.ThumbnailRemoteUrl, outfile)
 	if err != nil {
+		v.IsDownloaded = false
 		return fmt.Errorf("Error downloading video thumbnail (TweetID %d):\n  %w", v.TweetID, err)
 	}
 
-	v.IsDownloaded = true
 	return p.SaveVideo(*v)
 }