me/offline-twitter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

twitter command now uses current directory as the profile dir by default

Browse Source
This commit is contained in:
Alessio 2021-08-11 06:56:42 -07:00
parent 2698bd4fd7
commit 2b23fcbc53
3 changed files with 77 additions and 64 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

98
cmd/tests.sh
View File

@ -9,67 +9,77 @@ PS4='+(${BASH_SOURCE}:${LINENO}): '
test -e data && rm -r data test -e data && rm -r data
./tw create_profile data PATH=`pwd`:$PATH
tw create_profile data
cd data
# Fetch a user # Fetch a user
./tw fetch_user data Denlesks tw fetch_user Denlesks
test $(sqlite3 data/twitter.db "select handle from users") = "Denlesks" test "$(sqlite3 twitter.db "select handle from users")" = "Denlesks"
test $(sqlite3 data/twitter.db "select count(*) from users") = "1" test $(sqlite3 twitter.db "select count(*) from users") = "1"
./tw fetch_user data Denlesks tw fetch_user Denlesks
test $(sqlite3 data/twitter.db "select count(*) from users") = "1" test $(sqlite3 twitter.db "select count(*) from users") = "1"
# Fetch a tweet with images # Fetch a tweet with images
./tw fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700 tw fetch_tweet_only https://twitter.com/Denlesks/status/1261483383483293700
test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1" test $(sqlite3 twitter.db "select count(*) from tweets") = "1"
test "$(sqlite3 data/twitter.db "select text from tweets")" = "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do" test "$(sqlite3 twitter.db "select text from tweets")" = "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do"
test $(sqlite3 data/twitter.db "select count(*) from images") = "4" test $(sqlite3 twitter.db "select count(*) from images") = "4"
# Download its images # Download its images
test $(sqlite3 data/twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 0") = "4" test $(sqlite3 twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 0") = "4"
test $(sqlite3 data/twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 1") = "0" test $(sqlite3 twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 1") = "0"
test $(sqlite3 data/twitter.db "select is_content_downloaded from tweets where id = 1261483383483293700") = "0" test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1261483383483293700") = "0"
test $(find data/images | wc -l) = "1" test $(find images | wc -l) = "1"
./tw download_tweet_content data 1261483383483293700 tw download_tweet_content 1261483383483293700
test $(sqlite3 data/twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 0") = "0" test $(sqlite3 twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 0") = "0"
test $(sqlite3 data/twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 1") = "4" test $(sqlite3 twitter.db "select count(*) from images where tweet_id = 1261483383483293700 and is_downloaded = 1") = "4"
test $(sqlite3 data/twitter.db "select is_content_downloaded from tweets where id = 1261483383483293700") = "1" test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1261483383483293700") = "1"
test $(find data/images | wc -l) = "5" test $(find images | wc -l) = "5"
# Try to double-download it # Try to double-download it
./tw fetch_tweet_only data https://twitter.com/Denlesks/status/1261483383483293700 tw fetch_tweet_only https://twitter.com/Denlesks/status/1261483383483293700
test $(sqlite3 data/twitter.db "select count(*) from tweets") = "1" test $(sqlite3 twitter.db "select count(*) from tweets") = "1"
test $(sqlite3 data/twitter.db "select count(*) from images") = "4" test $(sqlite3 twitter.db "select count(*) from images") = "4"
# Fetch a tweet with a video # Fetch a tweet with a video
./tw fetch_user data DiamondChariots tw fetch_user DiamondChariots
test $(sqlite3 data/twitter.db "select handle from users" | wc -l) = "2" test $(sqlite3 twitter.db "select handle from users" | wc -l) = "2"
./tw fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340 tw fetch_tweet_only https://twitter.com/DiamondChariots/status/1418971605674467340
test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2" test $(sqlite3 twitter.db "select count(*) from tweets") = "2"
test $(sqlite3 data/twitter.db "select count(*) from videos") = "1" test $(sqlite3 twitter.db "select count(*) from videos") = "1"
# Download the video # Download the video
test $(sqlite3 data/twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 0") = "1" test $(sqlite3 twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 0") = "1"
test $(sqlite3 data/twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 1") = "0" test $(sqlite3 twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 1") = "0"
test $(sqlite3 data/twitter.db "select is_content_downloaded from tweets where id = 1418971605674467340") = "0" test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1418971605674467340") = "0"
test $(find data/videos| wc -l) = "1" test $(find videos| wc -l) = "1"
./tw download_tweet_content data 1418971605674467340 tw download_tweet_content 1418971605674467340
test $(sqlite3 data/twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 0") = "0" test $(sqlite3 twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 0") = "0"
test $(sqlite3 data/twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 1") = "1" test $(sqlite3 twitter.db "select count(*) from videos where tweet_id = 1418971605674467340 and is_downloaded = 1") = "1"
test $(sqlite3 data/twitter.db "select is_content_downloaded from tweets where id = 1418971605674467340") = "1" test $(sqlite3 twitter.db "select is_content_downloaded from tweets where id = 1418971605674467340") = "1"
test $(find data/videos | wc -l) = "2" test $(find videos | wc -l) = "2"
# Try to double-download it # Try to double-download it
./tw fetch_tweet_only data https://twitter.com/DiamondChariots/status/1418971605674467340 tw fetch_tweet_only https://twitter.com/DiamondChariots/status/1418971605674467340
test $(sqlite3 data/twitter.db "select count(*) from tweets") = "2" test $(sqlite3 twitter.db "select count(*) from tweets") = "2"
test $(sqlite3 data/twitter.db "select count(*) from videos") = "1" test $(sqlite3 twitter.db "select count(*) from videos") = "1"
# Download a user's profile image and banner image # Download a user's profile image and banner image
test $(sqlite3 data/twitter.db "select is_content_downloaded from users where handle = 'DiamondChariots'") = "0" test $(sqlite3 twitter.db "select is_content_downloaded from users where handle = 'DiamondChariots'") = "0"
./tw download_user_content data DiamondChariots tw download_user_content DiamondChariots
test $(sqlite3 data/twitter.db "select is_content_downloaded from users where handle = 'DiamondChariots'") = "1" test $(sqlite3 twitter.db "select is_content_downloaded from users where handle = 'DiamondChariots'") = "1"
test -f data/profile_images/DiamondChariots_profile_rE4OTedS.jpg test -f profile_images/DiamondChariots_profile_rE4OTedS.jpg
test -f data/profile_images/DiamondChariots_banner_1615811094.jpg test -f profile_images/DiamondChariots_banner_1615811094.jpg
# Test that the `--profile` flag works
cd ..
tw --profile data fetch_user michaelmalice
test $(sqlite3 data/twitter.db "select count(*) from users where handle = 'michaelmalice'") = "1"
echo -e "\033[32mAll tests passed. Finished successfully.\033[0m" echo -e "\033[32mAll tests passed. Finished successfully.\033[0m"

16
cmd/twitter/helpers.go
View File

@ -13,18 +13,18 @@ import (
/** /**
* Help message to print if command syntax is incorrect * Help message to print if command syntax is incorrect
*/ */
const help_message = `Usage: twitter <operation> <profile_dir> [TARGET] const help_message = `Usage: twitter [--profile <profile_dir>] <operation> <TARGET>
<operation>: <operation>:
- create_profile (no target needed) - create_profile (<TARGET> is the directory to create).
<TARGET> must not exist. <profile_dir> will be ignored if provided.
- fetch_user (TARGET is the user handle) - fetch_user (<TARGET> is the user handle)
- fetch_tweet (TARGET is the full URL of the tweet) - fetch_tweet_only (<TARGET> is the full URL of the tweet)
- fetch_tweet_and_replies (TARGET is the full URL of the tweet) - download_tweet_content (<TARGET> is the ID of the tweet whomst contents to download / back up)
- download_user_content (<TARGET> is the user handle of the user whomst banner image and profile to download / back up)
<profile_dir>: the path to the directory containing the data directories, database files, and settings files. <profile_dir>: the path to the directory containing the data directories, database files, and settings files. By default, refers to the current directory. Ignored if <operation> is "create_profile".
TARGET is optional depending on <operation>
` `

27
cmd/twitter/main.go
View File

@ -4,6 +4,7 @@ import (
"os" "os"
"strconv" "strconv"
"fmt" "fmt"
"flag"
"offline_twitter/scraper" "offline_twitter/scraper"
"offline_twitter/persistence" "offline_twitter/persistence"
) )
@ -14,8 +15,6 @@ import (
var profile persistence.Profile var profile persistence.Profile
// TODO: use the current directory by default, add flag to set data-dir
/** /**
* Main method * Main method
*/ */
@ -24,22 +23,26 @@ func main() {
die("", true, 1) die("", true, 1)
} }
operation := os.Args[1] profile_dir := flag.String("profile", ".", "TODO USAGE")
profile_dir := os.Args[2] flag.Parse()
args := flag.Args()
if operation == "create_profile" { if len(args) < 2 {
create_profile(profile_dir)
return
}
if len(os.Args) < 4 {
die("", true, 1) die("", true, 1)
} }
target := os.Args[3] operation := args[0]
target := args[1]
if operation == "create_profile" {
create_profile(target)
return
}
var err error var err error
profile, err = persistence.LoadProfile(profile_dir) profile, err = persistence.LoadProfile(*profile_dir)
if err != nil { if err != nil {
die("Could not load profile: " + err.Error(), true, 2) die("Could not load profile: " + err.Error(), true, 2)
} }