me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add RemoteURL and LocalFilename fields to Video, remove Filename

Browse Source
This commit is contained in:
Alessio 2021-08-05 14:10:46 -07:00
parent 8462365d70
commit 9c9100d874
8 changed files with 27 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
cmd/tests.sh
View File

@ -20,5 +20,21 @@ test $(sqlite3 data/twitter.db "select count(*) from users") = "1"
go run ./twitter fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700
test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1"
test "$(sqlite3 data/twitter.db "select text from tweets")" = "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do"
# Try to double-download it
go run ./twitter fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700
test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1"
# Fetch a tweet with a video
go run ./twitter fetch_user data DiamondChariots
test $(sqlite3 data/twitter.db "select handle from users" | wc -l) = "2"
go run ./twitter fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340
test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2"
# Try to double-download it
go run ./twitter fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340
test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2"
test $(sqlite3 data/twitter.db "select count(*) from videos") = "1"
echo -e "\033[32mAll tests passed. Finished successfully.\033[0m"

19
persistence/media_download_test.go
View File

@ -1,19 +0,0 @@
package persistence_test
import (
"testing"
"offline_twitter/scraper"
)
/**
* Should return an `.mp4`file matching its parent Tweet's ID
*/
func TestVideoFilenameWhenDownloaded(t *testing.T) {
v := scraper.Video{TweetID: scraper.TweetID(23), IsDownloaded: false, Filename: "https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/320x568/IXaQ5rPyf9mbD1aD.mp4?tag=12"}
outpath := v.FilenameWhenDownloaded()
expected := "23.mp4"
if outpath != expected {
t.Errorf("Expected output path to be %q, but got %q", expected, outpath)
}
}

10
persistence/media_queries.go
View File

@ -31,12 +31,12 @@ func (p Profile) SaveImage(img scraper.Image) error {
*/
func (p Profile) SaveVideo(vid scraper.Video) error {
_, err := p.DB.Exec(`
insert into videos (id, tweet_id, filename, is_downloaded)
values (?, ?, ?, ?)
insert into videos (id, tweet_id, remote_url, local_filename, is_downloaded)
values (?, ?, ?, ?, ?)
on conflict do update
set is_downloaded=?
`,
vid.ID, vid.TweetID, vid.Filename, vid.IsDownloaded,
vid.ID, vid.TweetID, vid.RemoteURL, vid.LocalFilename, vid.IsDownloaded,
vid.IsDownloaded,
)
return err
@ -73,7 +73,7 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e
* Get the list of videos for a tweet
*/
func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) {
stmt, err := p.DB.Prepare("select id, filename, is_downloaded from videos where tweet_id=?")
stmt, err := p.DB.Prepare("select id, remote_url, local_filename, is_downloaded from videos where tweet_id=?")
if err != nil {
return
}
@ -84,7 +84,7 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e
}
var vid scraper.Video
for rows.Next() {
err = rows.Scan(&vid.ID, &vid.Filename, &vid.IsDownloaded)
err = rows.Scan(&vid.ID, &vid.RemoteURL, &vid.LocalFilename, &vid.IsDownloaded)
if err != nil {
return
}

3
persistence/schema.sql
View File

@ -70,7 +70,8 @@ create table images (rowid integer primary key,
create table videos (rowid integer primary key,
id integer unique not null check(typeof(id) = 'integer'),
tweet_id integer not null,
filename text not null unique,
remote_url text not null unique,
local_filename text not null unique,
is_downloaded boolean default 0,
foreign key(tweet_id) references tweets(id)

3
persistence/utils_test.go
View File

@ -79,7 +79,8 @@ func create_video_from_id(id int) scraper.Video {
return scraper.Video{
ID: scraper.VideoID(id),
TweetID: -1,
Filename: filename,
RemoteURL: filename,
LocalFilename: filename,
IsDownloaded: false,
}
}

4
scraper/tweet_test.go
View File

@ -153,8 +153,8 @@ func TestParseTweetWithVideo(t *testing.T) {
t.Errorf(err.Error())
}
expected_video := "https://video.twimg.com/ext_tw_video/1418951950020845568/pu/vid/720x1280/sm4iL9_f8Lclh0aa.mp4?tag=12"
if len(tweet.Videos) != 1 || tweet.Videos[0].Filename != expected_video {
t.Errorf("Expected video %q, but got %+v", expected_video, tweet.Videos)
if len(tweet.Videos) != 1 || tweet.Videos[0].RemoteURL != expected_video {
t.Errorf("Expected video URL %q, but got %+v", expected_video, tweet.Videos)
}
if len(tweet.Images) != 0 {

6
scraper/video.go
View File

@ -13,7 +13,6 @@ type VideoID int64
type Video struct {
ID VideoID
TweetID TweetID
Filename string // TODO video-filename: delete when it all works
RemoteURL string
LocalFilename string
IsDownloaded bool
@ -28,13 +27,8 @@ func ParseAPIVideo(apiVideo APIExtendedMedia, tweet_id TweetID) Video {
return Video{
ID: VideoID(apiVideo.ID),
TweetID: tweet_id,
Filename: variants[0].URL,
RemoteURL: variants[0].URL,
LocalFilename: local_filename,
IsDownloaded: false,
}
}
func (v Video) FilenameWhenDownloaded() string { // TODO video-filename: delete whole method and associated test
return fmt.Sprintf("%d.mp4", v.TweetID)
}

3
scraper/video_test.go
View File

@ -32,9 +32,6 @@ func TestParseAPIVideo(t *testing.T) {
if video.RemoteURL != expected_remote_url {
t.Errorf("Expected %q, got %q", expected_remote_url, video.RemoteURL)
}
if video.Filename != expected_remote_url { // TODO video-filename: delete this check
t.Errorf("Expected %q, got %q", expected_remote_url, video.Filename)
}
expected_local_filename := "28.mp4"
if video.LocalFilename != expected_local_filename {