Add manual re-scrape for user feeds and quote-tweets stat on tweets
This commit is contained in:
parent
655a47ec21
commit
8aca7d4ebe
@ -79,7 +79,6 @@ TODO: mobile-requests
|
||||
- implement mobile versions of various requests
|
||||
- Check in `pkg/scraper/test_responses/mobile_requests`
|
||||
|
||||
|
||||
TODO: search-bottom-cursor
|
||||
- Entry type "TimelineReplaceEntries" that replaces the cursor in the timeline instead of the new timeline having a new one
|
||||
- As first step, need helper function that returns the []Instruction element in a APIV2Response (not just the MainInstruction which is TimelineAddEntries)
|
||||
@ -103,7 +102,6 @@ TODO: login-routes-tests
|
||||
|
||||
TODO: web-ui-downloading
|
||||
- web UI needs buttons to trigger a scrape / refresh manually
|
||||
- user feed
|
||||
- timeline
|
||||
|
||||
TODO: webserver-session-arg-active-user
|
||||
@ -121,5 +119,8 @@ TODO: image-width-and-height
|
||||
- Images should have explicit "width" and "height" attributes. This reduces Cumulative Layout Shift (CLS) while loading the page.
|
||||
- https://web.dev/optimize-cls/#images-without-dimensions
|
||||
|
||||
TODO: quote-tweet-icon
|
||||
- show quote-tweets on a tweet
|
||||
TODO: download-buttons-not-ugly
|
||||
- Make the options buttons for downloading less ugly (rn they are ugly)
|
||||
|
||||
TODO: show-errors-in-UI
|
||||
- if an HTTP request fails, show an error in the UI somehow
|
||||
|
||||
@ -34,9 +34,6 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) {
|
||||
app.traceLog.Printf("'UserFeed' handler (path: %q)", r.URL.Path)
|
||||
|
||||
parts := strings.Split(strings.Trim(r.URL.Path, "/"), "/")
|
||||
if len(parts) != 1 {
|
||||
app.error_404(w)
|
||||
}
|
||||
|
||||
user, err := app.Profile.GetUserByHandle(scraper.UserHandle(parts[0]))
|
||||
if err != nil {
|
||||
@ -44,6 +41,21 @@ func (app *Application) UserFeed(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
if len(parts) == 2 && parts[1] == "scrape" {
|
||||
if app.IsScrapingDisabled {
|
||||
http.Error(w, "Scraping is disabled (are you logged in?)", 401)
|
||||
return
|
||||
}
|
||||
|
||||
// Run scraper
|
||||
trove, err := scraper.GetUserFeedGraphqlFor(user.ID, 50) // TODO: parameterizable
|
||||
if err != nil {
|
||||
app.ErrorLog.Print(err)
|
||||
// TOOD: show error in UI
|
||||
}
|
||||
app.Profile.SaveTweetTrove(trove)
|
||||
}
|
||||
|
||||
c := persistence.NewUserFeedCursor(user.Handle)
|
||||
err = parse_cursor_value(&c, r)
|
||||
if err != nil {
|
||||
|
||||
7
internal/webserver/static/icons/quote.svg
Normal file
7
internal/webserver/static/icons/quote.svg
Normal file
@ -0,0 +1,7 @@
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Uploaded to: SVG Repo, www.svgrepo.com, Transformed by: SVG Repo Mixer Tools -->
|
||||
<svg fill="#000000" height="800px" width="800px" version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" viewBox="-3.2 -3.2 70.40 70.40" enable-background="new 0 0 64 64" xml:space="preserve" stroke="#000000" stroke-width="3.2">
|
||||
<g id="SVGRepo_bgCarrier" stroke-width="0"/>
|
||||
<g id="SVGRepo_tracerCarrier" stroke-linecap="round" stroke-linejoin="round"/>
|
||||
<g id="SVGRepo_iconCarrier"> <g id="Quotemarks-right"> <path d="M14.1933422,9.4116497c-7.8260994,0-14.1922989,6.3662004-14.1922989,14.1924 c0,7.5498009,5.9247999,13.7420998,13.3690996,14.169899c0.1288996,1.3916016,0.0321999,5.1797028-3.5977001,10.4491997 C9.4980431,48.6206512,9.547843,49.1567497,9.888648,49.497551c1.4853945,1.4853973,2.4033947,2.4208984,3.0458946,3.0751991 c0.8408995,0.8554993,1.2247,1.2461014,1.7861996,1.7559013c0.1904001,0.1727982,0.4306002,0.259697,0.6719055,0.259697 c0.2342949,0,0.4676943-0.0819969,0.6561956-0.2450981c6.3251991-5.5038986,13.3515987-16.8759995,12.3349991-30.8115005 C27.7881413,15.3501501,21.820343,9.4116497,14.1933422,9.4116497z M15.4023428,52.2221489 c-0.2723999-0.2684975-0.5830002-0.5848999-1.0410004-1.0508003c-0.5565996-0.5672989-1.3203001-1.3446999-2.4784994-2.5067978 c4.4053001-6.7881012,3.5731993-11.6230011,3.2089996-12.3164024c-0.1729002-0.3290977-0.5274-0.5507965-0.8985004-0.5507965 c-6.7225995,0-12.1922989-5.4697018-12.1922989-12.1933022c0-6.7227001,5.4696999-12.1924,12.1922989-12.1924 c6.5489006,0,11.6777992,5.1582012,12.1963062,12.2646008C27.5322418,39.3501511,18.2168427,49.5268517,15.4023428,52.2221489z"/> <path d="M63.9004402,23.5317497v-0.0009995C63.302742,15.3501501,57.3340416,9.4116497,49.7090416,9.4116497 c-7.8261986,0-14.1933937,6.3662004-14.1933937,14.1924c0,7.5498009,5.9257927,13.7420998,13.3710938,14.169899 c0.1289062,1.3906021,0.0312004,5.1767006-3.5996017,10.4491997c-0.2743988,0.3975029-0.2245979,0.9336014,0.1162033,1.2744026 c1.4794998,1.4794998,2.3955002,2.4130974,3.0380974,3.0663986c0.8446999,0.8613014,1.2304993,1.2538986,1.7949028,1.7656021 c0.1903992,0.1718979,0.4315987,0.2587967,0.6718979,0.2587967c0.2344055,0,0.4678001-0.0819969,0.6562004-0.2460976 C57.8896484,48.8383484,64.9160385,37.4663506,63.9004402,23.5317497z M50.917942,52.2221489 c-0.2743988-0.2705002-0.5877991-0.5887985-1.0498009-1.0594978c-0.5565987-0.5665016-1.3172989-1.3418007-2.4706993-2.4981003 c4.4053001-6.7891006,3.5742989-11.6230011,3.2109985-12.3164024c-0.1728973-0.3280983-0.5282974-0.5507965-0.8993988-0.5507965 c-6.7237015,0-12.1933937-5.4697018-12.1933937-12.1933022c0-6.7227001,5.4696922-12.1924,12.1933937-12.1924 c6.5477982,0,11.6777,5.1582012,12.1972008,12.2656002v-0.0009995 C63.0478401,39.3481483,53.7324409,49.5268517,50.917942,52.2221489z"/> </g> </g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 2.9 KiB |
@ -546,6 +546,7 @@ ul.dropdown-items {
|
||||
position: absolute;
|
||||
top: 2em;
|
||||
width: fit-content;
|
||||
margin: 0;
|
||||
padding: 0.3em;
|
||||
background-color: var(--color-twitter-off-white);
|
||||
outline: 1px solid var(--color-outline-gray);
|
||||
|
||||
@ -91,10 +91,10 @@
|
||||
</div>
|
||||
|
||||
<div class="interactions-bar row">
|
||||
<!-- <div class="interaction-stat">
|
||||
{template "quote-tweet-icon"}
|
||||
<div class="interaction-stat">
|
||||
<img class="svg-icon" src="/static/icons/quote.svg" />
|
||||
<span>{{$main_tweet.NumQuoteTweets}}</span>
|
||||
</div> -->
|
||||
</div>
|
||||
<div class="interaction-stat">
|
||||
<img class="svg-icon" src="/static/icons/reply.svg" />
|
||||
<span>{{$main_tweet.NumReplies}}</span>
|
||||
|
||||
@ -41,6 +41,26 @@
|
||||
<span class="following-label">is following</span>
|
||||
<span class="following-count">{{$user.FollowingCount}}</span>
|
||||
</div>
|
||||
|
||||
<div class="dropdown" hx-trigger="click consume">
|
||||
<button class="dropdown-button" title="Options">
|
||||
<img class="svg-icon" src="/static/icons/more.svg" />
|
||||
</button>
|
||||
<ul class="dropdown-items">
|
||||
<a class="unstyled-link" target="_blank" href="https://twitter.com/{{$user.Handle}}">
|
||||
<li class="quick-link">
|
||||
<img class="svg-icon" src="/static/icons/external-link.svg" />
|
||||
<span>Open on twitter.com</span>
|
||||
</li>
|
||||
</a>
|
||||
<a class="unstyled-link" target="_blank" hx-post="/{{$user.Handle}}/scrape" hx-target="body">
|
||||
<li class="quick-link">
|
||||
<img class="svg-icon" src="/static/icons/download.svg" />
|
||||
<span>Re-fetch user feed</span>
|
||||
</li>
|
||||
</a>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@ -287,6 +287,11 @@ func (api *API) do_http(url string, cursor string, result interface{}) error {
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if api.IsAuthenticated {
|
||||
// New request has been made, so the cookie will be changed; update the csrf to match
|
||||
api.update_csrf_token()
|
||||
}
|
||||
|
||||
if resp.StatusCode != 200 && resp.StatusCode != 403 {
|
||||
content, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
@ -311,10 +316,6 @@ func (api *API) do_http(url string, cursor string, result interface{}) error {
|
||||
return fmt.Errorf("Error parsing API response:\n %w", err)
|
||||
}
|
||||
|
||||
if api.IsAuthenticated {
|
||||
// New request has been made, so the cookie will be changed; update the csrf to match
|
||||
api.update_csrf_token()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user