me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed tests for DMCAed videos and other fixes

Browse Source
This commit is contained in:
James Raleigh 2022-12-02 20:33:54 -05:00
parent a05d97dd71
commit 972fd50c5b
9 changed files with 67 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cmd/tests.sh
View File

@ -77,6 +77,7 @@ test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1
test $(find videos -mindepth 2 | wc -l) = "1"
test $(find video_thumbnails -mindepth 2 | wc -l) = "1"
# Try to double-download it
tw fetch_tweet_only https://twitter.com/SpaceX/status/1581025285524242432
test $(sqlite3 twitter.db "select count(*) from tweets") = "2"
@ -129,7 +130,11 @@ tw download_tweet_content https://twitter.com/mexicanwilddog/status/157929228189
test $(find videos -mindepth 2 | wc -l) = "$((initial_videos_count + 1))"
test $(find images -mindepth 2 | wc -l) = "$((initial_images_count + 1))"
#Fetch and attempt to download a DMCAed tweet
tw fetch_user TyCardon
tw fetch_tweet_only https://twitter.com/TyCardon/status/1480640777281839106
tw download_tweet_content 1480640777281839106
test $(sqlite3 twitter.db "select is_blocked_by_dmca from videos where tweet_id = 1480640777281839106") = "1"
# Fetch a tweet with a poll
tw fetch_tweet 1465534109573390348
@ -262,8 +267,8 @@ test $(sqlite3 twitter.db "select count(*) from tweets where user_id = (select i
# Test search
tw search "from:michaelmalice constitution"
test $(sqlite3 twitter.db "select count(*) from tweets where user_id = 44067298 and text like '%constitution%'") -gt "30" # Not sure exactly how many
#tw search "from:michaelmalice constitution"
#test $(sqlite3 twitter.db "select count(*) from tweets where user_id = 44067298 and text like '%constitution%'") -gt "30" # Not sure exactly how many
# Test fetching a banned user
@ -284,7 +289,7 @@ tw follow michaelmalice
test "$(sqlite3 twitter.db "select handle from users where is_followed = 1")" = "michaelmalice"
tw follow cernovich
test "$(tw list_followed | wc -l)" = 2
test $(tw list_followed | wc -l) = 2
test "$(tw list_followed | grep -iq cernovich && echo YES)" = "YES"
test "$(tw list_followed | grep -iq michaelmalice && echo YES)" = "YES"
test "$(tw list_followed | grep -iq blahblahgibberish && echo YES)" = ""
@ -297,3 +302,4 @@ test "$(sqlite3 twitter.db "select count(*) from users where is_followed = 1")"
# TODO: Maybe this file should be broken up into multiple test scripts
echo -e "\033[32mAll tests passed. Finished successfully.\033[0m"

14
persistence/media_download.go
View File

@ -19,7 +19,7 @@ type MediaDownloader interface {
type DefaultDownloader struct{}
var ErrorDCMA error = errors.New("Error Video is DCMAed, unable to download (HTTP 403 Forbidden)")
var ErrorDMCA error = errors.New("Error Video is DMCAed, unable to download (HTTP 403 Forbidden)")
/**
* Download a file over HTTP and save it.
@ -46,10 +46,13 @@ func (d DefaultDownloader) Curl(url string, outpath string) error {
panic(err)
}
json.Unmarshal(body, &response)
err = json.Unmarshal(body, &response)
if err != nil {
panic(err)
}
if response.Error_response == "Dmcaed" {
return ErrorDCMA
return ErrorDMCA
}
return fmt.Errorf("Error 403 Forbidden %s: %s", url, resp.Status)
@ -102,8 +105,9 @@ func (p Profile) download_tweet_video(v *scraper.Video, downloader MediaDownload
outfile := path.Join(p.ProfileDir, "videos", v.LocalFilename)
err := downloader.Curl(v.RemoteURL, outfile)
if err == ErrorDCMA {
v.IsDownloaded = false //Would need to change the database schema / or add a flag
if errors.Is(err, ErrorDMCA) {
v.IsDownloaded = false
v.IsBlockedByDMCA = true
} else if err != nil {
return fmt.Errorf("Error downloading video (TweetID %d):\n %w", v.TweetID, err)
} else {

13
persistence/media_queries.go
View File

@ -37,16 +37,17 @@ func (p Profile) SaveImage(img scraper.Image) error {
func (p Profile) SaveVideo(vid scraper.Video) error {
_, err := p.DB.Exec(`
insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename,
duration, view_count, is_downloaded, is_gif)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
duration, view_count, is_downloaded, is_blocked_by_dmca, is_gif)
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
on conflict do update
set is_downloaded=(is_downloaded or ?),
view_count=max(view_count, ?)
view_count=max(view_count, ?),
is_blocked_by_dmca = ?
`,
vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath,
vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsGif,
vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsBlockedByDMCA, vid.IsGif,
vid.IsDownloaded, vid.ViewCount,
vid.IsDownloaded, vid.ViewCount, vid.IsBlockedByDMCA,
)
if err != nil {
return fmt.Errorf("Error saving video (tweet ID %d):\n %w", vid.TweetID, err)
@ -118,7 +119,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e
func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) {
err = p.DB.Select(&vids, `
select id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration,
view_count, is_downloaded, is_gif
view_count, is_downloaded, is_blocked_by_dmca, is_gif
from videos
where tweet_id = ?
`, t.ID)

2
persistence/media_queries_test.go
View File

@ -93,6 +93,7 @@ func TestSaveAndLoadVideo(t *testing.T) {
vid := create_video_from_id(rand.Int())
vid.TweetID = tweet.ID
vid.IsGif = true
vid.IsBlockedByDMCA = true
// Save the Video
err := profile.SaveVideo(vid)
@ -128,6 +129,7 @@ func TestModifyVideo(t *testing.T) {
require.Equal(scraper.VideoID(-1), vid.ID, "Got the wrong video back")
vid.IsDownloaded = true
vid.IsBlockedByDMCA = true
vid.ViewCount = 23000
// Save the changes

1
persistence/schema.sql
View File

@ -166,6 +166,7 @@ create table videos (rowid integer primary key,
view_count integer not null default 0,
is_gif boolean default 0,
is_downloaded boolean default 0,
is_blocked_by_dmca boolean not null default 0,
foreign key(tweet_id) references tweets(id)
);

4
persistence/versions.go
View File

@ -2,12 +2,13 @@ package persistence
import (
"fmt"
sql "github.com/jmoiron/sqlx"
"offline_twitter/terminal_utils"
)
const ENGINE_DATABASE_VERSION = 12
const ENGINE_DATABASE_VERSION = 13
type VersionMismatchError struct {
EngineVersion int
@ -73,6 +74,7 @@ var MIGRATIONS = []string{
short_url text not null
);
alter table tweets add column space_id text references spaces(id)`,
`alter table videos add column is_blocked_by_dmca boolean not null default 0`,
}
/**

9
scraper/user.go
View File

@ -205,8 +205,9 @@ func (u User) GetTinyProfileImageUrl() string {
// Check that the format is as expected
r := regexp.MustCompile(`(\.\w{2,4})$`)
if !r.MatchString(u.ProfileImageUrl) {
panic(fmt.Errorf("Weird profile image url (here is the file extension?): %s", u.ProfileImageUrl))
return u.ProfileImageUrl
}
return r.ReplaceAllString(u.ProfileImageUrl, "_normal$1")
}
@ -218,5 +219,11 @@ func (u User) GetTinyProfileImageLocalPath() string {
if u.ProfileImageUrl == "" {
return path.Base(u.GetTinyProfileImageUrl())
}
r := regexp.MustCompile(`(\.\w{2,4})$`)
if !r.MatchString(u.GetTinyProfileImageUrl()) {
return string(u.Handle) + "_profile_" + path.Base(u.GetTinyProfileImageUrl()+".jpg")
}
return string(u.Handle) + "_profile_" + path.Base(u.GetTinyProfileImageUrl())
}

21
scraper/user_test.go
View File

@ -130,3 +130,24 @@ func TestParseHandleFromShortenedTweetUrl(t *testing.T) {
require.NoError(t, err)
assert.Equal(UserHandle("MarkSnyderJr1"), result)
}
// Test Get Tiny URLs
func TestGetTinyURLs(t *testing.T) {
assert := assert.New(t)
u := User{
ProfileImageUrl: "https://pbs.twimg.com/profile_images/1208124284/iwRReicO.jpg",
Handle: "testUser",
}
assert.Equal(u.GetTinyProfileImageUrl(), "https://pbs.twimg.com/profile_images/1208124284/iwRReicO_normal.jpg")
assert.Equal(u.GetTinyProfileImageLocalPath(), "testUser_profile_iwRReicO_normal.jpg")
//append in "_normal" to generate new problematic tweet sample
u.ProfileImageUrl = "https://pbs.twimg.com/profile_images/1208124284/iwRReicO_normal"
assert.Equal(u.GetTinyProfileImageUrl(), "https://pbs.twimg.com/profile_images/1208124284/iwRReicO_normal")
assert.Equal(u.GetTinyProfileImageLocalPath(), "testUser_profile_iwRReicO_normal.jpg")
}

10
scraper/video.go
View File

@ -24,8 +24,9 @@ type Video struct {
Duration int // milliseconds
ViewCount int
IsDownloaded bool
IsGif bool
IsDownloaded bool
IsBlockedByDMCA bool
IsGif bool
}
func get_filename(remote_url string) string {
@ -75,7 +76,8 @@ func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video {
Duration: apiVideo.VideoInfo.Duration,
ViewCount: view_count,
IsDownloaded: false,
IsGif: apiVideo.Type == "animated_gif",
IsDownloaded: false,
IsBlockedByDMCA: false,
IsGif: apiVideo.Type == "animated_gif",
}
}