me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix for dmca video downloads

Browse Source
This commit is contained in:
James Raleigh 2022-11-27 18:10:25 -05:00
parent 6830be3c87
commit bacfd519cc
3 changed files with 35 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@ -9,5 +9,6 @@ require (
github.com/mattn/go-sqlite3 v1.14.7
github.com/sirupsen/logrus v1.8.1
github.com/stretchr/testify v1.7.0
golang.org/x/sys v0.2.0 // indirect
gopkg.in/yaml.v2 v2.4.0
)

2
go.sum
View File

@ -24,6 +24,8 @@ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5Cc
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A=
golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

34
persistence/media_download.go
View File

@ -1,6 +1,8 @@
package persistence
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@ -17,6 +19,8 @@ type MediaDownloader interface {
type DefaultDownloader struct{}
var ErrorDCMA error = errors.New("Error Video is DCMAed, unable to download (HTTP 403 Forbidden)")
/**
* Download a file over HTTP and save it.
*
@ -30,6 +34,27 @@ func (d DefaultDownloader) Curl(url string, outpath string) error {
if err != nil {
return fmt.Errorf("Error executing HTTP GET(%q):\n %w", url, err)
}
if resp.StatusCode == 403 {
var response struct {
Error_response string `json:"error_response"`
}
body, err := io.ReadAll(resp.Body)
fmt.Println("body = " + string(body))
if err != nil {
panic(err)
}
json.Unmarshal(body, &response)
if response.Error_response == "Dmcaed" {
return ErrorDCMA
}
return fmt.Errorf("Error 403 Forbidden %s: %s", url, resp.Status)
}
if resp.StatusCode != 200 {
return fmt.Errorf("Error %s: %s", url, resp.Status)
}
@ -76,18 +101,23 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload
// Download the video
outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename)
err := downloader.Curl(v.RemoteURL, outfile)
if err != nil {
if err == ErrorDCMA {
v.IsDownloaded = false //Would need to change the database schema / or add a flag
} else if err != nil {
return fmt.Errorf("Error downloading video (TweetID %d):\n %w", v.TweetID, err)
} else {
v.IsDownloaded = true
}
// Download the thumbnail
outfile = path.Join(p.ProfileDir, "video_thumbnails", v.ThumbnailLocalPath)
err = downloader.Curl(v.ThumbnailRemoteUrl, outfile)
if err != nil {
v.IsDownloaded = false
return fmt.Errorf("Error downloading video thumbnail (TweetID %d):\n %w", v.TweetID, err)
}
v.IsDownloaded = true
return p.SaveVideo(*v)
}