diff --git a/.build.yml b/.build.yml index 917f8e4..6fbcfc4 100644 --- a/.build.yml +++ b/.build.yml @@ -69,17 +69,3 @@ tasks: duration=$SECONDS echo "Task completed in $(($duration / 60))m$(($duration % 60))s." - - - compile: | - SECONDS=0 - - cd twitter_offline_engine/cmd - go build -o ../../twitter ./twitter - cd ../.. - chmod +x twitter - sudo mv twitter /usr/local/bin - - which twitter - - duration=$SECONDS - echo "Task completed in $(($duration / 60))m$(($duration % 60))s." diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2d83068 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +coverage.out diff --git a/cmd/.gitignore b/cmd/.gitignore new file mode 100644 index 0000000..1c30d63 --- /dev/null +++ b/cmd/.gitignore @@ -0,0 +1,2 @@ +data +tw diff --git a/cmd/compile.sh b/cmd/compile.sh new file mode 100755 index 0000000..6c7eb8e --- /dev/null +++ b/cmd/compile.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -x +set -e + +go build -o tw ./twitter +chmod +x tw diff --git a/cmd/tests.sh b/cmd/tests.sh index e57daef..4387a04 100755 --- a/cmd/tests.sh +++ b/cmd/tests.sh @@ -5,36 +5,64 @@ set -x PS4='+(${BASH_SOURCE}:${LINENO}): ' +./compile.sh + test -e data && rm -r data -go run ./twitter create_profile data +./tw create_profile data # Fetch a user -go run ./twitter fetch_user data Denlesks +./tw fetch_user data Denlesks test $(sqlite3 data/twitter.db "select handle from users") = "Denlesks" test $(sqlite3 data/twitter.db "select count(*) from users") = "1" -go run ./twitter fetch_user data Denlesks +./tw fetch_user data Denlesks test $(sqlite3 data/twitter.db "select count(*) from users") = "1" # Fetch a tweet with images -go run ./twitter fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700 +./tw fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700 test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1" test "$(sqlite3 data/twitter.db "select text from tweets")" = "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do" +test $(sqlite3 data/twitter.db "select count(*) from images") = "4" + +# Download its images +test $(sqlite3 data/twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 0") = "4" +test $(sqlite3 data/twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 1") = "0" +test $(sqlite3 data/twitter.db "select is_content_downloaded from tweets where id = 1261483383483293700") = "0" +test $(find data/images | wc -l) = "1" +./tw download_tweet_content data 1261483383483293700 +test $(sqlite3 data/twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 0") = "0" +test $(sqlite3 data/twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 1") = "4" +test $(sqlite3 data/twitter.db "select is_content_downloaded from tweets where id = 1261483383483293700") = "1" +test $(find data/images | wc -l) = "5" # Try to double-download it -go run ./twitter fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700 +./tw fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700 test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1" +test $(sqlite3 data/twitter.db "select count(*) from images") = "4" # Fetch a tweet with a video -go run ./twitter fetch_user data DiamondChariots +./tw fetch_user data DiamondChariots test $(sqlite3 data/twitter.db "select handle from users" | wc -l) = "2" -go run ./twitter fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340 -test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2" - -# Try to double-download it -go run ./twitter fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340 +./tw fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340 test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2" test $(sqlite3 data/twitter.db "select count(*) from videos") = "1" +# Download the video +test $(sqlite3 data/twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 0") = "1" +test $(sqlite3 data/twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 1") = "0" +test $(sqlite3 data/twitter.db "select is_content_downloaded from tweets where id = 1418971605674467340") = "0" +test $(find data/videos| wc -l) = "1" +./tw download_tweet_content data 1418971605674467340 +test $(sqlite3 data/twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 0") = "0" +test $(sqlite3 data/twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 1") = "1" +test $(sqlite3 data/twitter.db "select is_content_downloaded from tweets where id = 1418971605674467340") = "1" +test $(find data/videos | wc -l) = "2" + +# Try to double-download it +./tw fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340 +test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2" +test $(sqlite3 data/twitter.db "select count(*) from videos") = "1" + + echo -e "\033[32mAll tests passed. Finished successfully.\033[0m" diff --git a/cmd/twitter/main.go b/cmd/twitter/main.go index 58fb74e..0fc64f4 100644 --- a/cmd/twitter/main.go +++ b/cmd/twitter/main.go @@ -2,6 +2,7 @@ package main import ( "os" + "strconv" "fmt" "offline_twitter/scraper" "offline_twitter/persistence" @@ -48,6 +49,8 @@ func main() { fetch_user(scraper.UserHandle(target)) case "fetch_tweet_only": fetch_tweet_only(target) + case "download_tweet_content": + download_tweet_content(target) default: die("Invalid operation: " + operation, true, 3) } @@ -116,3 +119,19 @@ func fetch_tweet_only(tweet_url string) { } fmt.Println("Saved the tweet. Exiting successfully") } + + +func download_tweet_content(tweet_id string) { + id, err := strconv.Atoi(tweet_id) + if err != nil { + panic(err) + } + tweet, err := profile.GetTweetById(scraper.TweetID(id)) + if err != nil { + panic("Couldn't get tweet from database: " + err.Error()) + } + err = profile.DownloadTweetContentFor(&tweet) + if err != nil { + panic("Error getting content: " + err.Error()) + } +}