me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix gifs

Browse Source
This commit is contained in:
Alessio 2021-10-04 21:06:53 -07:00
parent 18722a6b1b
commit 100fe6c718
8 changed files with 45 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/tests.sh
View File

@ -75,6 +75,22 @@ test $(sqlite3 twitter.db "select count(*) from tweets") = "2"
test $(sqlite3 twitter.db "select count(*) from videos") = "1"
# Fetch a tweet with a GIF
tw fetch_user Cernovich
initial_videos_count=$(find videos | wc -l)
initial_videos_db_count=$(sqlite3 twitter.db "select count(*) from videos")
tw fetch_tweet_only https://twitter.com/Cernovich/status/1444429517020274693
test $(sqlite3 twitter.db "select count(*) from videos") = "$((initial_videos_db_count + 1))"
test $(sqlite3 twitter.db "select is_gif from videos where tweet_id = 1444429517020274693") = "1"
# Download the GIF
test $(find videos | wc -l) = "$((initial_videos_count))" # Shouldn't have changed yet
tw download_tweet_content https://twitter.com/Cernovich/status/1444429517020274693
test $(find videos | wc -l) = "$((initial_videos_count + 1))"
# Download a full thread
tw fetch_tweet https://twitter.com/RememberAfghan1/status/1429585423702052867
test $(sqlite3 twitter.db "select handle from tweets join users on tweets.user_id = users.id where tweets.id=1429585423702052867") = "RememberAfghan1"

10
persistence/media_queries.go
View File

@ -31,12 +31,12 @@ func (p Profile) SaveImage(img scraper.Image) error {
*/
func (p Profile) SaveVideo(vid scraper.Video) error {
_, err := p.DB.Exec(`
insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded)
values (?, ?, ?, ?, ?)
insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded, is_gif)
values (?, ?, ?, ?, ?, ?)
on conflict do update
set is_downloaded=?
`,
vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded,
vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded, vid.IsGif,
vid.IsDownloaded,
)
return err
@ -89,7 +89,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e
* Get the list of videos for a tweet
*/
func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) {
stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded from videos where tweet_id=?")
stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded, is_gif from videos where tweet_id=?")
if err != nil {
return
}
@ -100,7 +100,7 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e
}
var vid scraper.Video
for rows.Next() {
err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded)
err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded, &vid.IsGif)
if err != nil {
return
}

1
persistence/media_queries_test.go
View File

@ -102,6 +102,7 @@ func TestSaveAndLoadVideo(t *testing.T) {
rand.Seed(time.Now().UnixNano())
vid := create_video_from_id(rand.Int())
vid.TweetID = tweet.ID
vid.IsGif = true;
// Save the Video
err := profile.SaveVideo(vid)

1
persistence/schema.sql
View File

@ -88,6 +88,7 @@ create table videos (rowid integer primary key,
tweet_id integer not null,
remote_url text not null unique,
local_filename text not null unique,
is_gif boolean default 0,
is_downloaded boolean default 0,
foreign key(tweet_id) references tweets(id)

1
persistence/utils_test.go
View File

@ -89,6 +89,7 @@ func create_video_from_id(id int) scraper.Video {
RemoteURL: filename,
LocalFilename: filename,
IsDownloaded: false,
IsGif: false,
}
}

2
scraper/tweet.go
View File

@ -129,7 +129,7 @@ func ParseSingleTweet(apiTweet APITweet) (ret Tweet, err error) {
ret.QuotedTweet = TweetID(apiTweet.QuotedStatusID)
for _, entity := range apiTweet.ExtendedEntities.Media {
if entity.Type != "video" {
if entity.Type != "video" && entity.Type != "animated_gif" {
continue
}
if len(apiTweet.ExtendedEntities.Media) != 1 {

18
scraper/tweet_test.go
View File

@ -94,12 +94,30 @@ func TestParseTweetWithVideo(t *testing.T) {
if len(tweet.Videos) != 1 || tweet.Videos[0].RemoteURL != expected_video {
t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos)
}
if tweet.Videos[0].IsGif != false {
t.Errorf("Expected it to be a regular video, but it was a gif")
}
if len(tweet.Images) != 0 {
t.Errorf("Should not have any images, but has %d", len(tweet.Images))
}
}
func TestParseTweetWithGif(t *testing.T) {
tweet := load_tweet_from_file("test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json")
expected_video := "https://video.twimg.com/tweet_video/E189-VhVoAYcrDv.mp4"
if len(tweet.Videos) != 1 {
t.Errorf("Expected 1 video (a gif), but got %d instead", len(tweet.Videos))
}
if tweet.Videos[0].RemoteURL != expected_video {
t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos)
}
if tweet.Videos[0].IsGif != true {
t.Errorf("Expected video to be a gif, but it wasn't")
}
}
func TestParseTweetWithUrl(t *testing.T) {
tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_url_card.json")

2
scraper/video.go
View File

@ -16,6 +16,7 @@ type Video struct {
RemoteURL string
LocalFilename string
IsDownloaded bool
IsGif bool
}
func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video {
@ -30,5 +31,6 @@ func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video {
RemoteURL: variants[0].URL,
LocalFilename: local_filename,
IsDownloaded: false,
IsGif: apiVideo.Type == "animated_gif",
}
}