diff --git a/.golangci.yaml b/.golangci.yaml new file mode 100644 index 0000000..b59dd58 --- /dev/null +++ b/.golangci.yaml @@ -0,0 +1,698 @@ +# This file contains all available configuration options +# with their default values. + +# output configuration options +output: + # colored-line-number|line-number|json|tab|checkstyle|code-climate|junit-xml|github-actions + format: colored-line-number + + # sorts results by: filepath, line and column + sort-results: true + +linters: + disable-all: true + enable: + - deadcode + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + - structcheck + - typecheck + - unused + - varcheck + - whitespace + # - wsl + # - wrapcheck + - lll + # - godox + # - errorlint + + +# # all available settings of specific linters +linters-settings: +# cyclop: +# # the maximal code complexity to report +# max-complexity: 10 +# # the maximal average package complexity. If it's higher than 0.0 (float) the check is enabled (default 0.0) +# package-average: 0.0 +# # should ignore tests (default false) +# skip-tests: false + +# dogsled: +# # checks assignments with too many blank identifiers; default is 2 +# max-blank-identifiers: 2 + +# dupl: +# # tokens count to trigger issue, 150 by default +# threshold: 100 + + errcheck: + # report about not checking of errors in type assertions: `a := b.(MyStruct)`; + # default is false: such cases aren't reported by default. + check-type-assertions: true + + # report about assignment of errors to blank identifier: `num, _ := strconv.Atoi(numStr)`; + # default is false: such cases aren't reported by default. + check-blank: true + + # # list of functions to exclude from checking, where each entry is a single function to exclude. + # # see https://github.com/kisielk/errcheck#excluding-functions for details + # exclude-functions: + # - io/ioutil.ReadFile + # - io.Copy(*bytes.Buffer) + # - io.Copy(os.Stdout) + +# errorlint: +# # Check whether fmt.Errorf uses the %w verb for formatting errors. See the readme for caveats +# errorf: true +# # Check for plain type assertions and type switches +# asserts: true +# # Check for plain error comparisons +# comparison: true + +# exhaustive: +# # check switch statements in generated files also +# check-generated: false +# # indicates that switch statements are to be considered exhaustive if a +# # 'default' case is present, even if all enum members aren't listed in the +# # switch +# default-signifies-exhaustive: false + +# exhaustivestruct: +# # Struct Patterns is list of expressions to match struct packages and names +# # The struct packages have the form example.com/package.ExampleStruct +# # The matching patterns can use matching syntax from https://pkg.go.dev/path#Match +# # If this list is empty, all structs are tested. +# struct-patterns: +# - '*.Test' +# - 'example.com/package.ExampleStruct' + +# forbidigo: +# # Forbid the following identifiers (identifiers are written using regexp): +# forbid: +# - ^print.*$ +# - 'fmt\.Print.*' +# # Exclude godoc examples from forbidigo checks. Default is true. +# exclude_godoc_examples: false + +# funlen: +# lines: 60 +# statements: 40 + +# gci: +# # put imports beginning with prefix after 3rd-party packages; +# # only support one prefix +# # if not set, use goimports.local-prefixes +# local-prefixes: github.com/org/project + +# gocognit: +# # minimal code complexity to report, 30 by default (but we recommend 10-20) +# min-complexity: 10 + +# nestif: +# # minimal complexity of if statements to report, 5 by default +# min-complexity: 4 + +# goconst: +# # minimal length of string constant, 3 by default +# min-len: 3 +# # minimum occurrences of constant string count to trigger issue, 3 by default +# min-occurrences: 3 +# # ignore test files, false by default +# ignore-tests: false +# # look for existing constants matching the values, true by default +# match-constant: true +# # search also for duplicated numbers, false by default +# numbers: false +# # minimum value, only works with goconst.numbers, 3 by default +# min: 3 +# # maximum value, only works with goconst.numbers, 3 by default +# max: 3 +# # ignore when constant is not used as function argument, true by default +# ignore-calls: true + +# gocritic: +# # Which checks should be enabled; can't be combined with 'disabled-checks'; +# # See https://go-critic.github.io/overview#checks-overview +# # To check which checks are enabled run `GL_DEBUG=gocritic golangci-lint run` +# # By default list of stable checks is used. +# enabled-checks: +# - rangeValCopy + +# # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty +# disabled-checks: +# - regexpMust + +# # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint run` to see all tags and checks. +# # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". +# enabled-tags: +# - performance +# disabled-tags: +# - experimental + +# # Settings passed to gocritic. +# # The settings key is the name of a supported gocritic checker. +# # The list of supported checkers can be find in https://go-critic.github.io/overview. +# settings: +# captLocal: # must be valid enabled check name +# # whether to restrict checker to params only (default true) +# paramsOnly: true +# elseif: +# # whether to skip balanced if-else pairs (default true) +# skipBalanced: true +# hugeParam: +# # size in bytes that makes the warning trigger (default 80) +# sizeThreshold: 80 +# nestingReduce: +# # min number of statements inside a branch to trigger a warning (default 5) +# bodyWidth: 5 +# rangeExprCopy: +# # size in bytes that makes the warning trigger (default 512) +# sizeThreshold: 512 +# # whether to check test functions (default true) +# skipTestFuncs: true +# rangeValCopy: +# # size in bytes that makes the warning trigger (default 128) +# sizeThreshold: 32 +# # whether to check test functions (default true) +# skipTestFuncs: true +# ruleguard: +# # path to a gorules file for the ruleguard checker +# rules: '' +# truncateCmp: +# # whether to skip int/uint/uintptr types (default true) +# skipArchDependent: true +# underef: +# # whether to skip (*x).method() calls where x is a pointer receiver (default true) +# skipRecvDeref: true +# unnamedResult: +# # whether to check exported functions +# checkExported: true + +# gocyclo: +# # minimal code complexity to report, 30 by default (but we recommend 10-20) +# min-complexity: 10 + +# godot: +# # comments to be checked: `declarations`, `toplevel`, or `all` +# scope: declarations +# # list of regexps for excluding particular comment lines from check +# exclude: +# # example: exclude comments which contain numbers +# # - '[0-9]+' +# # check that each sentence starts with a capital letter +# capital: false + + # godox: + # # report any comments starting with keywords, this is useful for TODO or FIXME comments that + # # might be left in the code accidentally and should be resolved before merging + # keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting + # - XXX + +# gofmt: +# # simplify code: gofmt with `-s` option, true by default +# simplify: true + +# gofumpt: +# # Select the Go version to target. The default is `1.15`. +# lang-version: "1.15" + +# # Choose whether or not to use the extra rules that are disabled +# # by default +# extra-rules: false + +# goheader: +# values: +# const: +# # define here const type values in format k:v, for example: +# # COMPANY: MY COMPANY +# regexp: +# # define here regexp type values, for example +# # AUTHOR: .*@mycompany\.com +# template: # |- +# # put here copyright header template for source code files, for example: +# # Note: {{ YEAR }} is a builtin value that returns the year relative to the current machine time. +# # +# # {{ AUTHOR }} {{ COMPANY }} {{ YEAR }} +# # SPDX-License-Identifier: Apache-2.0 + +# # Licensed under the Apache License, Version 2.0 (the "License"); +# # you may not use this file except in compliance with the License. +# # You may obtain a copy of the License at: + +# # http://www.apache.org/licenses/LICENSE-2.0 + +# # Unless required by applicable law or agreed to in writing, software +# # distributed under the License is distributed on an "AS IS" BASIS, +# # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# # See the License for the specific language governing permissions and +# # limitations under the License. +# template-path: +# # also as alternative of directive 'template' you may put the path to file with the template source + +# goimports: +# # put imports beginning with prefix after 3rd-party packages; +# # it's a comma-separated list of prefixes +# local-prefixes: github.com/org/project + +# golint: +# # minimal confidence for issues, default is 0.8 +# min-confidence: 0.8 + +# gomnd: +# settings: +# mnd: +# # the list of enabled checks, see https://github.com/tommy-muehle/go-mnd/#checks for description. +# checks: argument,case,condition,operation,return,assign +# # ignored-numbers: 1000 +# # ignored-files: magic_.*.go +# # ignored-functions: math.* + +# gomoddirectives: +# # Allow local `replace` directives. Default is false. +# replace-local: false +# # List of allowed `replace` directives. Default is empty. +# replace-allow-list: +# - launchpad.net/gocheck +# # Allow to not explain why the version has been retracted in the `retract` directives. Default is false. +# retract-allow-no-explanation: false +# # Forbid the use of the `exclude` directives. Default is false. +# exclude-forbidden: false + +# gomodguard: +# allowed: +# modules: # List of allowed modules +# # - gopkg.in/yaml.v2 +# domains: # List of allowed module domains +# # - golang.org +# blocked: +# modules: # List of blocked modules +# # - github.com/uudashr/go-module: # Blocked module +# # recommendations: # Recommended modules that should be used instead (Optional) +# # - golang.org/x/mod +# # reason: "`mod` is the official go.mod parser library." # Reason why the recommended module should be used (Optional) +# versions: # List of blocked module version constraints +# # - github.com/mitchellh/go-homedir: # Blocked module with version constraint +# # version: "< 1.1.0" # Version constraint, see https://github.com/Masterminds/semver#basic-comparisons +# # reason: "testing if blocked version constraint works." # Reason why the version constraint exists. (Optional) +# local_replace_directives: false # Set to true to raise lint issues for packages that are loaded from a local path via replace directive + +# gosec: +# # To select a subset of rules to run. +# # Available rules: https://github.com/securego/gosec#available-rules +# includes: +# - G401 +# - G306 +# - G101 +# # To specify a set of rules to explicitly exclude. +# # Available rules: https://github.com/securego/gosec#available-rules +# excludes: +# - G204 +# # To specify the configuration of rules. +# # The configuration of rules is not fully documented by gosec: +# # https://github.com/securego/gosec#configuration +# # https://github.com/securego/gosec/blob/569328eade2ccbad4ce2d0f21ee158ab5356a5cf/rules/rulelist.go#L60-L102 +# config: +# G306: "0600" +# G101: +# pattern: "(?i)example" +# ignore_entropy: false +# entropy_threshold: "80.0" +# per_char_threshold: "3.0" +# truncate: "32" + +# gosimple: +# # Select the Go version to target. The default is '1.13'. +# go: "1.15" +# # https://staticcheck.io/docs/options#checks +# checks: [ "all" ] + +# govet: +# # report about shadowed variables +# check-shadowing: true + +# # settings per analyzer +# settings: +# printf: # analyzer name, run `go tool vet help` to see all analyzers +# funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer +# - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof +# - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf +# - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf +# - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + +# # enable or disable analyzers by name +# # run `go tool vet help` to see all analyzers +# enable: +# - atomicalign +# enable-all: false +# disable: +# - shadow +# disable-all: false + +# depguard: +# list-type: blacklist +# include-go-root: false +# packages: +# - github.com/sirupsen/logrus +# packages-with-error-message: +# # specify an error message to output when a blacklisted package is used +# - github.com/sirupsen/logrus: "logging is allowed only by logutils.Log" + +# ifshort: +# # Maximum length of variable declaration measured in number of lines, after which linter won't suggest using short syntax. +# # Has higher priority than max-decl-chars. +# max-decl-lines: 1 +# # Maximum length of variable declaration measured in number of characters, after which linter won't suggest using short syntax. +# max-decl-chars: 30 + +# importas: +# # if set to `true`, force to use alias. +# no-unaliased: true +# # List of aliases +# alias: +# # using `servingv1` alias for `knative.dev/serving/pkg/apis/serving/v1` package +# - pkg: knative.dev/serving/pkg/apis/serving/v1 +# alias: servingv1 +# # using `autoscalingv1alpha1` alias for `knative.dev/serving/pkg/apis/autoscaling/v1alpha1` package +# - pkg: knative.dev/serving/pkg/apis/autoscaling/v1alpha1 +# alias: autoscalingv1alpha1 +# # You can specify the package path by regular expression, +# # and alias by regular expression expansion syntax like below. +# # see https://github.com/julz/importas#use-regular-expression for details +# - pkg: knative.dev/serving/pkg/apis/(\w+)/(v[\w\d]+) +# alias: $1$2 + + lll: + # max line length, lines longer will be reported. Default is 120. + # '\t' is counted as 1 character by default, and can be changed with the tab-width option + line-length: 140 + # tab width in spaces. Default to 1. + tab-width: 4 + +# makezero: +# # Allow only slices initialized with a length of zero. Default is false. +# always: false + +# maligned: +# # print struct with more effective memory layout or not, false by default +# suggest-new: true + +# misspell: +# # Correct spellings using locale preferences for US or UK. +# # Default is to use a neutral variety of English. +# # Setting locale to US will correct the British spelling of 'colour' to 'color'. +# locale: US +# ignore-words: +# - someword + +# nakedret: +# # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 +# max-func-lines: 30 + +# prealloc: +# # XXX: we don't recommend using this linter before doing performance profiling. +# # For most programs usage of prealloc will be a premature optimization. + +# # Report preallocation suggestions only on simple loops that have no returns/breaks/continues/gotos in them. +# # True by default. +# simple: true +# range-loops: true # Report preallocation suggestions on range loops, true by default +# for-loops: false # Report preallocation suggestions on for loops, false by default + +# promlinter: +# # Promlinter cannot infer all metrics name in static analysis. +# # Enable strict mode will also include the errors caused by failing to parse the args. +# strict: false +# # Please refer to https://github.com/yeya24/promlinter#usage for detailed usage. +# disabled-linters: +# # - "Help" +# # - "MetricUnits" +# # - "Counter" +# # - "HistogramSummaryReserved" +# # - "MetricTypeInName" +# # - "ReservedChars" +# # - "CamelCase" +# # - "lintUnitAbbreviations" + +# predeclared: +# # comma-separated list of predeclared identifiers to not report on +# ignore: "" +# # include method names and field names (i.e., qualified names) in checks +# q: false + +# nolintlint: +# # Enable to ensure that nolint directives are all used. Default is true. +# allow-unused: false +# # Disable to ensure that nolint directives don't have a leading space. Default is true. +# allow-leading-space: true +# # Exclude following linters from requiring an explanation. Default is []. +# allow-no-explanation: [] +# # Enable to require an explanation of nonzero length after each nolint directive. Default is false. +# require-explanation: true +# # Enable to require nolint directives to mention the specific linter being suppressed. Default is false. +# require-specific: true + +# rowserrcheck: +# packages: +# - github.com/jmoiron/sqlx + +# revive: +# # see https://github.com/mgechev/revive#available-rules for details. +# ignore-generated-header: true +# severity: warning +# rules: +# - name: indent-error-flow +# severity: warning +# - name: add-constant +# severity: warning +# arguments: +# - maxLitCount: "3" +# allowStrs: '""' +# allowInts: "0,1,2" +# allowFloats: "0.0,0.,1.0,1.,2.0,2." + +# staticcheck: +# # Select the Go version to target. The default is '1.13'. +# go: "1.15" +# # https://staticcheck.io/docs/options#checks +# checks: [ "all" ] + +# stylecheck: +# # Select the Go version to target. The default is '1.13'. +# go: "1.15" +# # https://staticcheck.io/docs/options#checks +# checks: [ "all", "-ST1000", "-ST1003", "-ST1016", "-ST1020", "-ST1021", "-ST1022" ] +# # https://staticcheck.io/docs/options#dot_import_whitelist +# dot-import-whitelist: +# - fmt +# # https://staticcheck.io/docs/options#initialisms +# initialisms: [ "ACL", "API", "ASCII", "CPU", "CSS", "DNS", "EOF", "GUID", "HTML", "HTTP", "HTTPS", "ID", "IP", "JSON", "QPS", "RAM", "RPC", "SLA", "SMTP", "SQL", "SSH", "TCP", "TLS", "TTL", "UDP", "UI", "GID", "UID", "UUID", "URI", "URL", "UTF8", "VM", "XML", "XMPP", "XSRF", "XSS" ] +# # https://staticcheck.io/docs/options#http_status_code_whitelist +# http-status-code-whitelist: [ "200", "400", "404", "500" ] + +# tagliatelle: +# # check the struck tag name case +# case: +# # use the struct field name to check the name of the struct tag +# use-field-name: true +# rules: +# # any struct tag type can be used. +# # support string case: `camel`, `pascal`, `kebab`, `snake`, `goCamel`, `goPascal`, `goKebab`, `goSnake`, `upper`, `lower` +# json: camel +# yaml: camel +# xml: camel +# bson: camel +# avro: snake +# mapstructure: kebab + +# testpackage: +# # regexp pattern to skip files +# skip-regexp: (export|internal)_test\.go + +# thelper: +# # The following configurations enable all checks. It can be omitted because all checks are enabled by default. +# # You can enable only required checks deleting unnecessary checks. +# test: +# first: true +# name: true +# begin: true +# benchmark: +# first: true +# name: true +# begin: true +# tb: +# first: true +# name: true +# begin: true + +# unparam: +# # Inspect exported functions, default is false. Set to true if no external program/library imports your code. +# # XXX: if you enable this setting, unparam will report a lot of false-positives in text editors: +# # if it's called for subdir of a project it can't find external interfaces. All text editor integrations +# # with golangci-lint call it on a directory with the changed file. +# check-exported: false + +# unused: +# # Select the Go version to target. The default is '1.13'. +# go: "1.15" + +# whitespace: +# multi-if: false # Enforces newlines (or comments) after every multi-line if statement +# multi-func: false # Enforces newlines (or comments) after every multi-line function signature + +# wrapcheck: +# # An array of strings that specify substrings of signatures to ignore. +# # If this set, it will override the default set of ignored signatures. +# # See https://github.com/tomarrell/wrapcheck#configuration for more information. +# ignoreSigs: +# - .Errorf( +# - errors.New( +# - errors.Unwrap( +# - .Wrap( +# - .Wrapf( +# - .WithMessage( + + wsl: + # See https://github.com/bombsimon/wsl/blob/master/doc/configuration.md for documentation of available settings. + + # Controls if you're allowed to cuddle multiple variable declarations. + allow-cuddle-declarations: true + + # # Controls if you may cuddle assignments and anything without needing an empty line between them. + # # Default: false + # allow-assign-and-anything: false + # # Controls if you may cuddle assignments and calls without needing an empty line between them. + # # Default: true + # allow-assign-and-call: true + # # Controls if you may cuddle assignments even if they span over multiple lines. + # # Default: true + # allow-multiline-assign: true + # # This option allows whitespace after each comment group that begins a block. + # # Default: false + # allow-separated-leading-comment: false + # # Controls if blocks can end with comments. + # # This is not encouraged sine it's usually code smell but might be useful do improve understanding or learning purposes. + # # To be allowed there must be no whitespace between the comment and the last statement or the comment and the closing brace. + # # Default: false + # allow-trailing-comment: false + # # Can be set to force trailing newlines at the end of case blocks to improve readability. + # # If the number of lines (including comments) in a case block exceeds this number + # # a linter error will be yielded if the case does not end with a newline. + # # Default: 0 + # force-case-trailing-whitespace: 0 + # # Enforces that an `if` statement checking an error variable is cuddled + # # with the line that assigned that error variable. + # # Default: false + # force-err-cuddling: false + # # Enforces that an assignment which is actually a short declaration (using `:=`) + # # is only allowed to cuddle with other short declarations, and not plain assignments, blocks, etc. + # # This rule helps make declarations stand out by themselves, much the same as grouping var statement. + # # Default: false + # force-short-decl-cuddling: false + # # Controls if the checks for slice append should be "strict" + # # in the sense that it will only allow these assignments to be cuddled with variables being appended. + # # Default: true + # strict-append: true + +# # The custom section can be used to define linter plugins to be loaded at runtime. +# # See README doc for more info. +# custom: +# # Each custom linter should have a unique name. +# example: +# # The path to the plugin *.so. Can be absolute or local. Required for each custom linter +# path: /path/to/example.so +# # The description of the linter. Optional, just for documentation purposes. +# description: This is an example usage of a plugin linter. +# # Intended to point to the repo location of the linter. Optional, just for documentation purposes. +# original-url: github.com/golangci/example-linter + + +# issues: +# # List of regexps of issue texts to exclude, empty list by default. +# # But independently from this option we use default exclude patterns, +# # it can be disabled by `exclude-use-default: false`. To list all +# # excluded by default patterns execute `golangci-lint run --help` +# exclude: +# - abcdef + +# # Excluding configuration per-path, per-linter, per-text and per-source +# exclude-rules: +# # Exclude some linters from running on tests files. +# - path: _test\.go +# linters: +# - gocyclo +# - errcheck +# - dupl +# - gosec + +# # Exclude known linters from partially hard-vendored code, +# # which is impossible to exclude via "nolint" comments. +# - path: internal/hmac/ +# text: "weak cryptographic primitive" +# linters: +# - gosec + +# # Exclude some staticcheck messages +# - linters: +# - staticcheck +# text: "SA9003:" + +# # Exclude lll issues for long lines with go:generate +# - linters: +# - lll +# source: "^//go:generate " + +# # Independently from option `exclude` we use default exclude patterns, +# # it can be disabled by this option. To list all +# # excluded by default patterns execute `golangci-lint run --help`. +# # Default value for this option is true. +# exclude-use-default: false + +# # The default value is false. If set to true exclude and exclude-rules +# # regular expressions become case sensitive. +# exclude-case-sensitive: false + +# # The list of ids of default excludes to include or disable. By default it's empty. +# include: +# - EXC0002 # disable excluding of issues about comments from golint + +# # Maximum issues count per one linter. Set to 0 to disable. Default is 50. +# max-issues-per-linter: 0 + +# # Maximum count of issues with the same text. Set to 0 to disable. Default is 3. +# max-same-issues: 0 + +# # Show only new issues: if there are unstaged changes or untracked files, +# # only those changes are analyzed, else only changes in HEAD~ are analyzed. +# # It's a super-useful option for integration of golangci-lint into existing +# # large codebase. It's not practical to fix all existing issues at the moment +# # of integration: much better don't allow issues in new code. +# # Default is false. +# new: false + +# severity: +# # Default value is empty string. +# # Set the default severity for issues. If severity rules are defined and the issues +# # do not match or no severity is provided to the rule this will be the default +# # severity applied. Severities should match the supported severity names of the +# # selected out format. +# # - Code climate: https://docs.codeclimate.com/docs/issues#issue-severity +# # - Checkstyle: https://checkstyle.sourceforge.io/property_types.html#severity +# # - Github: https://help.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-error-message +# default-severity: error + +# # The default value is false. +# # If set to true severity-rules regular expressions become case sensitive. +# case-sensitive: false + +# # Default value is empty list. +# # When a list of severity rules are provided, severity information will be added to lint +# # issues. Severity rules have the same filtering capability as exclude rules except you +# # are allowed to specify one matcher per severity rule. +# # Only affects out formats that support setting severity information. +# rules: +# - linters: +# - dupl +# severity: info diff --git a/cmd/twitter/helpers.go b/cmd/twitter/helpers.go index d1b879d..c2eadf5 100644 --- a/cmd/twitter/helpers.go +++ b/cmd/twitter/helpers.go @@ -35,7 +35,8 @@ This application downloads tweets from twitter and saves them in a SQLite databa fetch_tweet fetch_tweet_only is either the full URL of the tweet, or its ID. - If using "fetch_tweet_only", then only that specific tweet will be saved. "fetch_tweet" will save the whole thread including replies. + If using "fetch_tweet_only", then only that specific tweet will be saved. "fetch_tweet" will save the whole + thread including replies. download_tweet_content is either the full URL of the tweet, or its ID. diff --git a/persistence/media_download.go b/persistence/media_download.go index 5c92b82..a69b6d6 100644 --- a/persistence/media_download.go +++ b/persistence/media_download.go @@ -193,6 +193,7 @@ func (p Profile) DownloadUserContentWithInjector(u *scraper.User, downloader Med /** * Download a User's tiny profile image, if it hasn't been downloaded yet. * If it has been downloaded, do nothing. + * If this user should have a big profile picture, defer to the regular `DownloadUserContentFor` method. */ func (p Profile) DownloadUserProfileImageTiny(u *scraper.User) error { if p.IsFollowing(u.Handle) { diff --git a/persistence/media_queries.go b/persistence/media_queries.go index 109a2db..7f15731 100644 --- a/persistence/media_queries.go +++ b/persistence/media_queries.go @@ -33,13 +33,16 @@ func (p Profile) SaveImage(img scraper.Image) error { */ func (p Profile) SaveVideo(vid scraper.Video) error { _, err := p.DB.Exec(` - insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration, view_count, is_downloaded, is_gif) + insert into videos (id, tweet_id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, + duration, view_count, is_downloaded, is_gif) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do update set is_downloaded=(is_downloaded or ?), view_count=max(view_count, ?) `, - vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath, vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsGif, + vid.ID, vid.TweetID, vid.Width, vid.Height, vid.RemoteURL, vid.LocalFilename, vid.ThumbnailRemoteUrl, vid.ThumbnailLocalPath, + vid.Duration, vid.ViewCount, vid.IsDownloaded, vid.IsGif, + vid.IsDownloaded, vid.ViewCount, ) return err @@ -50,12 +53,15 @@ func (p Profile) SaveVideo(vid scraper.Video) error { */ func (p Profile) SaveUrl(url scraper.Url) error { _, err := p.DB.Exec(` - insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded) + insert into urls (tweet_id, domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, + thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do update set is_content_downloaded=(is_content_downloaded or ?) `, - url.TweetID, url.Domain, url.Text, url.ShortText, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailWidth, url.ThumbnailHeight, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded, + url.TweetID, url.Domain, url.Text, url.ShortText, url.Title, url.Description, url.CreatorID, url.SiteID, url.ThumbnailWidth, + url.ThumbnailHeight, url.ThumbnailRemoteUrl, url.ThumbnailLocalPath, url.HasCard, url.HasThumbnail, url.IsContentDownloaded, + url.IsContentDownloaded, ) return err @@ -66,7 +72,8 @@ func (p Profile) SaveUrl(url scraper.Url) error { */ func (p Profile) SavePoll(poll scraper.Poll) error { _, err := p.DB.Exec(` - insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, choice4_votes, voting_duration, voting_ends_at, last_scraped_at) + insert into polls (id, tweet_id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, + choice4_votes, voting_duration, voting_ends_at, last_scraped_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do update set choice1_votes=?, @@ -75,7 +82,9 @@ func (p Profile) SavePoll(poll scraper.Poll) error { choice4_votes=?, last_scraped_at=? `, - poll.ID, poll.TweetID, poll.NumChoices, poll.Choice1, poll.Choice1_Votes, poll.Choice2, poll.Choice2_Votes, poll.Choice3, poll.Choice3_Votes, poll.Choice4, poll.Choice4_Votes, poll.VotingDuration, poll.VotingEndsAt.Unix(), poll.LastUpdatedAt.Unix(), + poll.ID, poll.TweetID, poll.NumChoices, poll.Choice1, poll.Choice1_Votes, poll.Choice2, poll.Choice2_Votes, poll.Choice3, + poll.Choice3_Votes, poll.Choice4, poll.Choice4_Votes, poll.VotingDuration, poll.VotingEndsAt.Unix(), poll.LastUpdatedAt.Unix(), + poll.Choice1_Votes, poll.Choice2_Votes, poll.Choice3_Votes, poll.Choice4_Votes, poll.LastUpdatedAt.Unix(), ) return err @@ -113,7 +122,12 @@ func (p Profile) GetImagesForTweet(t scraper.Tweet) (imgs []scraper.Image, err e * Get the list of videos for a tweet */ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err error) { - stmt, err := p.DB.Prepare("select id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration, view_count, is_downloaded, is_gif from videos where tweet_id=?") + stmt, err := p.DB.Prepare(` + select id, width, height, remote_url, local_filename, thumbnail_remote_url, thumbnail_local_filename, duration, view_count, + is_downloaded, is_gif + from videos + where tweet_id = ? + `) if err != nil { return } @@ -124,7 +138,8 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e } var vid scraper.Video for rows.Next() { - err = rows.Scan(&vid.ID, &vid.Width, &vid.Height, &vid.RemoteURL, &vid.LocalFilename, &vid.ThumbnailRemoteUrl, &vid.ThumbnailLocalPath, &vid.Duration, &vid.ViewCount, &vid.IsDownloaded, &vid.IsGif) + err = rows.Scan(&vid.ID, &vid.Width, &vid.Height, &vid.RemoteURL, &vid.LocalFilename, &vid.ThumbnailRemoteUrl, + &vid.ThumbnailLocalPath, &vid.Duration, &vid.ViewCount, &vid.IsDownloaded, &vid.IsGif) if err != nil { return } @@ -138,7 +153,13 @@ func (p Profile) GetVideosForTweet(t scraper.Tweet) (vids []scraper.Video, err e * Get the list of Urls for a Tweet */ func (p Profile) GetUrlsForTweet(t scraper.Tweet) (urls []scraper.Url, err error) { - stmt, err := p.DB.Prepare("select domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, thumbnail_remote_url, thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded from urls where tweet_id=? order by rowid") + stmt, err := p.DB.Prepare(` + select domain, text, short_text, title, description, creator_id, site_id, thumbnail_width, thumbnail_height, thumbnail_remote_url, + thumbnail_local_path, has_card, has_thumbnail, is_content_downloaded + from urls + where tweet_id = ? + order by rowid + `) if err != nil { return } @@ -149,7 +170,9 @@ func (p Profile) GetUrlsForTweet(t scraper.Tweet) (urls []scraper.Url, err error } var url scraper.Url for rows.Next() { - err = rows.Scan(&url.Domain, &url.Text, &url.ShortText, &url.Title, &url.Description, &url.CreatorID, &url.SiteID, &url.ThumbnailWidth, &url.ThumbnailHeight, &url.ThumbnailRemoteUrl, &url.ThumbnailLocalPath, &url.HasCard, &url.HasThumbnail, &url.IsContentDownloaded) + err = rows.Scan(&url.Domain, &url.Text, &url.ShortText, &url.Title, &url.Description, &url.CreatorID, &url.SiteID, + &url.ThumbnailWidth, &url.ThumbnailHeight, &url.ThumbnailRemoteUrl, &url.ThumbnailLocalPath, &url.HasCard, + &url.HasThumbnail, &url.IsContentDownloaded) if err != nil { return } @@ -163,7 +186,12 @@ func (p Profile) GetUrlsForTweet(t scraper.Tweet) (urls []scraper.Url, err error * Get the list of Polls for a Tweet */ func (p Profile) GetPollsForTweet(t scraper.Tweet) (polls []scraper.Poll, err error) { - stmt, err := p.DB.Prepare("select id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, choice4_votes, voting_duration, voting_ends_at, last_scraped_at from polls where tweet_id=?") + stmt, err := p.DB.Prepare(` + select id, num_choices, choice1, choice1_votes, choice2, choice2_votes, choice3, choice3_votes, choice4, choice4_votes, + voting_duration, voting_ends_at, last_scraped_at + from polls + where tweet_id = ? + `) if err != nil { return } @@ -176,7 +204,8 @@ func (p Profile) GetPollsForTweet(t scraper.Tweet) (polls []scraper.Poll, err er var voting_ends_at int var last_scraped_at int for rows.Next() { - err = rows.Scan(&poll.ID, &poll.NumChoices, &poll.Choice1, &poll.Choice1_Votes, &poll.Choice2, &poll.Choice2_Votes, &poll.Choice3, &poll.Choice3_Votes, &poll.Choice4, &poll.Choice4_Votes, &poll.VotingDuration, &voting_ends_at, &last_scraped_at) + err = rows.Scan(&poll.ID, &poll.NumChoices, &poll.Choice1, &poll.Choice1_Votes, &poll.Choice2, &poll.Choice2_Votes, &poll.Choice3, + &poll.Choice3_Votes, &poll.Choice4, &poll.Choice4_Votes, &poll.VotingDuration, &voting_ends_at, &last_scraped_at) if err != nil { return } diff --git a/persistence/profile_test.go b/persistence/profile_test.go index 0c45b4c..dbb57cb 100644 --- a/persistence/profile_test.go +++ b/persistence/profile_test.go @@ -24,13 +24,6 @@ func file_exists(path string) bool { } } -func isdir_map(is_dir bool) string { - if is_dir { - return "directory" - } - return "file" -} - /** * Should refuse to create a Profile if the target already exists (i.e., is a file or directory). diff --git a/persistence/tweet_queries.go b/persistence/tweet_queries.go index f6f33b0..d558a33 100644 --- a/persistence/tweet_queries.go +++ b/persistence/tweet_queries.go @@ -16,7 +16,9 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { return err } _, err = db.Exec(` - insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at) + insert into tweets (id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, + quoted_tweet_id, mentions, reply_mentions, hashtags, tombstone_type, is_stub, is_content_downloaded, + is_conversation_scraped, last_scraped_at) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, (select rowid from tombstone_types where short_name=?), ?, ?, ?, ?) on conflict do update set num_likes=?, @@ -28,8 +30,12 @@ func (p Profile) SaveTweet(t scraper.Tweet) error { is_conversation_scraped=(is_conversation_scraped or ?), last_scraped_at=max(last_scraped_at, ?) `, - t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID, t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt.Unix(), - t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt.Unix(), + t.ID, t.UserID, t.Text, t.PostedAt.Unix(), t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.InReplyToID, + t.QuotedTweetID, scraper.JoinArrayOfHandles(t.Mentions), scraper.JoinArrayOfHandles(t.ReplyMentions), + strings.Join(t.Hashtags, ","), t.TombstoneType, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, t.LastScrapedAt.Unix(), + + t.NumLikes, t.NumRetweets, t.NumReplies, t.NumQuoteTweets, t.IsStub, t.IsContentDownloaded, t.IsConversationScraped, + t.LastScrapedAt.Unix(), ) if err != nil { @@ -92,7 +98,9 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { db := p.DB stmt, err := db.Prepare(` - select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded, is_conversation_scraped, last_scraped_at + select id, user_id, text, posted_at, num_likes, num_retweets, num_replies, num_quote_tweets, in_reply_to_id, quoted_tweet_id, + mentions, reply_mentions, hashtags, ifnull(tombstone_types.short_name, ""), is_stub, is_content_downloaded, + is_conversation_scraped, last_scraped_at from tweets left join tombstone_types on tweets.tombstone_type = tombstone_types.rowid where id = ? `) @@ -110,7 +118,9 @@ func (p Profile) GetTweetById(id scraper.TweetID) (scraper.Tweet, error) { var hashtags string row := stmt.QueryRow(id) - err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID, &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded, &t.IsConversationScraped, &last_scraped_at) + err = row.Scan(&t.ID, &t.UserID, &t.Text, &postedAt, &t.NumLikes, &t.NumRetweets, &t.NumReplies, &t.NumQuoteTweets, &t.InReplyToID, + &t.QuotedTweetID, &mentions, &reply_mentions, &hashtags, &t.TombstoneType, &t.IsStub, &t.IsContentDownloaded, + &t.IsConversationScraped, &last_scraped_at) if err != nil { return t, err } diff --git a/persistence/user_queries.go b/persistence/user_queries.go index 257645c..d522f54 100644 --- a/persistence/user_queries.go +++ b/persistence/user_queries.go @@ -30,7 +30,9 @@ func (p Profile) SaveUser(u *scraper.User) error { } _, err := p.DB.Exec(` - insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, pinned_tweet_id, is_content_downloaded, is_id_fake) + insert into users (id, display_name, handle, bio, following_count, followers_count, location, website, join_date, is_private, + is_verified, is_banned, profile_image_url, profile_image_local_path, banner_image_url, banner_image_local_path, + pinned_tweet_id, is_content_downloaded, is_id_fake) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) on conflict do update set bio=?, @@ -49,8 +51,12 @@ func (p Profile) SaveUser(u *scraper.User) error { pinned_tweet_id=?, is_content_downloaded=(is_content_downloaded or ?) `, - u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate.Unix(), u.IsPrivate, u.IsVerified, u.IsBanned, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, u.IsIdFake, - u.Bio, u.DisplayName, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.IsBanned, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, + u.ID, u.DisplayName, u.Handle, u.Bio, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.JoinDate, u.IsPrivate, + u.IsVerified, u.IsBanned, u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, + u.IsContentDownloaded, u.IsIdFake, + + u.Bio, u.DisplayName, u.FollowingCount, u.FollowersCount, u.Location, u.Website, u.IsPrivate, u.IsVerified, u.IsBanned, + u.ProfileImageUrl, u.ProfileImageLocalPath, u.BannerImageUrl, u.BannerImageLocalPath, u.PinnedTweetID, u.IsContentDownloaded, ) if err != nil { return err diff --git a/persistence/versions.go b/persistence/versions.go index 1546ebf..cb5d6cf 100644 --- a/persistence/versions.go +++ b/persistence/versions.go @@ -53,19 +53,23 @@ var MIGRATIONS = []string{ `alter table tweets add column is_conversation_scraped boolean default 0; alter table tweets add column last_scraped_at integer not null default 0`, `update tombstone_types set tombstone_text = 'This Tweet is from a suspended account' where rowid = 2; - insert into tombstone_types (rowid, short_name, tombstone_text) values (5, 'violated', 'This Tweet violated the Twitter Rules'), (6, 'no longer exists', 'This Tweet is from an account that no longer exists')`, + insert into tombstone_types (rowid, short_name, tombstone_text) + values (5, 'violated', 'This Tweet violated the Twitter Rules'), + (6, 'no longer exists', 'This Tweet is from an account that no longer exists')`, `alter table videos add column thumbnail_remote_url text not null default "missing"; alter table videos add column thumbnail_local_filename text not null default "missing"`, `alter table videos add column duration integer not null default 0; alter table videos add column view_count integer not null default 0`, `alter table users add column is_banned boolean default 0`, `alter table urls add column short_text text not null default ""`, -`insert into tombstone_types (rowid, short_name, tombstone_text) values (7, 'age-restricted', 'Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter')`, +`insert into tombstone_types (rowid, short_name, tombstone_text) values (7, 'age-restricted', 'Age-restricted adult content. ' + || 'This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter')`, `alter table users add column is_followed boolean default 0`, `create table fake_user_sequence(latest_fake_id integer not null); insert into fake_user_sequence values(0x4000000000000000); alter table users add column is_id_fake boolean default 0;`, -`delete from urls where rowid in (select urls.rowid from tweets join urls on tweets.id = urls.tweet_id where urls.text like 'https://twitter.com/%/status/' || tweets.quoted_tweet_id || "%")`, +`delete from urls where rowid in (select urls.rowid from tweets join urls on tweets.id = urls.tweet_id where urls.text like + 'https://twitter.com/%/status/' || tweets.quoted_tweet_id || "%")`, } /** @@ -104,7 +108,8 @@ func (p Profile) check_and_update_version() error { if ENGINE_DATABASE_VERSION > version { fmt.Printf(terminal_utils.COLOR_YELLOW) fmt.Printf("================================================\n") - fmt.Printf("Database version is out of date. Upgrading database from version %d to version %d!\n", version, ENGINE_DATABASE_VERSION) + fmt.Printf("Database version is out of date. Upgrading database from version %d to version %d!\n", version, + ENGINE_DATABASE_VERSION) fmt.Printf(terminal_utils.COLOR_RESET) return p.UpgradeFromXToY(version, ENGINE_DATABASE_VERSION) } diff --git a/scraper/api_request_utils.go b/scraper/api_request_utils.go index b49b976..714cd5d 100644 --- a/scraper/api_request_utils.go +++ b/scraper/api_request_utils.go @@ -182,7 +182,11 @@ func UpdateQueryCursor(req *http.Request, new_cursor string, is_tweet bool) { func (api API) GetUser(handle UserHandle) (APIUser, error) { client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequest("GET", "https://api.twitter.com/graphql/4S2ihIKfF3xhp-ENxvUAfQ/UserByScreenName?variables=%7B%22screen_name%22%3A%22" + string(handle) + "%22%2C%22withHighlightedLabel%22%3Atrue%7D", nil) + req, err := http.NewRequest( + "GET", + "https://api.twitter.com/graphql/4S2ihIKfF3xhp-ENxvUAfQ/UserByScreenName?variables=%7B%22screen_name%22%3A%22" + string(handle) + + "%22%2C%22withHighlightedLabel%22%3Atrue%7D", + nil) if err != nil { return APIUser{}, err } @@ -232,7 +236,11 @@ func (api API) GetUser(handle UserHandle) (APIUser, error) { func (api API) Search(query string, cursor string) (TweetResponse, error) { client := &http.Client{Timeout: 10 * time.Second} - req, err := http.NewRequest("GET", "https://twitter.com/i/api/2/search/adaptive.json?count=50&spelling_corrections=1&query_source=typed_query&pc=1&q=" + url.QueryEscape(query), nil) + req, err := http.NewRequest( + "GET", + "https://twitter.com/i/api/2/search/adaptive.json?count=50&spelling_corrections=1&query_source=typed_query&pc=1&q=" + + url.QueryEscape(query), + nil) if err != nil { return TweetResponse{}, err } diff --git a/scraper/api_types.go b/scraper/api_types.go index f9a0334..414659b 100644 --- a/scraper/api_types.go +++ b/scraper/api_types.go @@ -193,7 +193,6 @@ func (t *APITweet) NormalizeContent() { index := strings.Index(t.FullText, url.ShortenedUrl) if index == (len(t.FullText) - len(url.ShortenedUrl)) { t.FullText = strings.TrimSpace(t.FullText[0:index]) - } } @@ -336,7 +335,8 @@ var tombstone_types = map[string]string{ "This Tweet is unavailable. Learn more": "unavailable", "This Tweet violated the Twitter Rules. Learn more": "violated", "This Tweet is from an account that no longer exists. Learn more": "no longer exists", - "Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, you’ll need to log in to Twitter. Learn more": "age-restricted", + "Age-restricted adult content. This content might not be appropriate for people under 18 years old. To view this media, " + + "you’ll need to log in to Twitter. Learn more": "age-restricted", } /** * Insert tweets into GlobalObjects for each tombstone. Returns a list of users that need to diff --git a/scraper/api_types_test.go b/scraper/api_types_test.go index afbcfd0..8415205 100644 --- a/scraper/api_types_test.go +++ b/scraper/api_types_test.go @@ -25,13 +25,20 @@ func TestNormalizeContent(t *testing.T) { {"test_responses/single_tweets/tweet_that_is_a_reply_with_gif.json", "", 0, 1395882872729477131, 0, "@michaelmalice"}, {"test_responses/single_tweets/tweet_with_image.json", "this saddens me every time", 0, 0, 0, ""}, {"test_responses/single_tweets/tweet_that_is_a_reply.json", "Noted", 0, 1396194494710788100, 0, "@RvaTeddy @michaelmalice"}, - {"test_responses/single_tweets/tweet_with_4_images.json", "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do", 0, 0, 0, ""}, - {"test_responses/single_tweets/tweet_with_at_mentions_in_front.json", "It always does, doesn't it?", 0, 1428907275532476416, 0, "@rob_mose @primalpoly @jmasseypoet @SpaceX"}, - {"test_responses/single_tweets/tweet_with_unicode_chars.json", "The fact that @michaelmalice new book ‘The Anarchist Handbook’ is just absolutely destroying on the charts is the largest white pill I’ve swallowed in years.", 0, 0, 0, ""}, + {"test_responses/single_tweets/tweet_with_4_images.json", "These are public health officials who are making decisions about " + + "your lifestyle because they know more about health, fitness and well-being than you do", 0, 0, 0, ""}, + {"test_responses/single_tweets/tweet_with_at_mentions_in_front.json", "It always does, doesn't it?", 0, 1428907275532476416, 0, + "@rob_mose @primalpoly @jmasseypoet @SpaceX"}, + {"test_responses/single_tweets/tweet_with_unicode_chars.json", "The fact that @michaelmalice new book ‘The Anarchist Handbook’ " + + "is just absolutely destroying on the charts is the largest white pill I’ve swallowed in years.", 0, 0, 0, ""}, {"test_responses/single_tweets/tweet_with_quoted_tweet_as_link.json", "", 1422680899670274048, 0, 0, ""}, - {"test_responses/single_tweets/tweet_with_quoted_tweet_as_link2.json", "sometimes they're too dimwitted to even get the wrong title right", 1396194494710788100, 1395882872729477131, 0, ""}, - {"test_responses/single_tweets/tweet_with_quoted_tweet_as_link3.json", "I was using an analogy about creating out-groups but the Germans sure love their literalism", 1442092399358930946, 1335678942020300802, 0, ""}, - {"test_responses/single_tweets/tweet_with_html_entities.json", "By the 1970s the elite consensus was that \"the hunt for atomic spies\" had been a grotesque over-reaction to minor leaks that cost the lives of the Rosenbergs & ruined many innocents. Only when the USSR fell was it discovered that they & other spies had given away ALL the secrets", 0, 0, 0, ""}, + {"test_responses/single_tweets/tweet_with_quoted_tweet_as_link2.json", "sometimes they're too dimwitted to even get the wrong " + + "title right", 1396194494710788100, 1395882872729477131, 0, ""}, + {"test_responses/single_tweets/tweet_with_quoted_tweet_as_link3.json", "I was using an analogy about creating out-groups but " + + "the Germans sure love their literalism", 1442092399358930946, 1335678942020300802, 0, ""}, + {"test_responses/single_tweets/tweet_with_html_entities.json", "By the 1970s the elite consensus was that \"the hunt for " + + "atomic spies\" had been a grotesque over-reaction to minor leaks that cost the lives of the Rosenbergs & ruined many " + + "innocents. Only when the USSR fell was it discovered that they & other spies had given away ALL the secrets", 0, 0, 0, ""}, } for _, v := range test_cases { @@ -80,7 +87,8 @@ func TestGetCursor(t *testing.T) { err = json.Unmarshal(data, &tweet_resp) assert.NoError(err) - assert.Equal("LBmGhsC+ibH1peAmgICjpbS0m98mgICj7a2lmd8mhsC4rbmsmN8mgMCqkbT1p+AmgsC4ucv4o+AmhoCyrf+nlt8mhMC9qfOwlt8mJQISAAA=", tweet_resp.GetCursor()) + assert.Equal("LBmGhsC+ibH1peAmgICjpbS0m98mgICj7a2lmd8mhsC4rbmsmN8mgMCqkbT1p+AmgsC4ucv4o+AmhoCyrf+nlt8mhMC9qfOwlt8mJQISAAA=", + tweet_resp.GetCursor()) } diff --git a/scraper/api_types_v2.go b/scraper/api_types_v2.go index c4b603a..8192fcb 100644 --- a/scraper/api_types_v2.go +++ b/scraper/api_types_v2.go @@ -237,7 +237,6 @@ func (api_result APIV2Result) ToTweetTrove(ignore_null_entries bool) TweetTrove main_tweet.Polls = []Poll{poll} ret.Tweets[main_tweet.ID] = main_tweet } - } return ret @@ -370,9 +369,9 @@ func (api_response APIV2Response) ToTweetTrove() (TweetTrove, error) { func get_graphql_user_timeline_url(user_id UserID, cursor string) string { if cursor != "" { - return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22cursor%22%3A%22" + url.QueryEscape(cursor) + "%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_responsive_web_uc_gql_enabled%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" + return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22cursor%22%3A%22" + url.QueryEscape(cursor) + "%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_responsive_web_uc_gql_enabled%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" // nolint:lll } - return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Afalse%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" + return "https://twitter.com/i/api/graphql/CwLU7qTfeu0doqhSr6tW4A/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22" + fmt.Sprint(user_id) + "%22%2C%22count%22%3A40%2C%22includePromotedContent%22%3Afalse%2C%22withCommunity%22%3Atrue%2C%22withSuperFollowsUserFields%22%3Atrue%2C%22withBirdwatchPivots%22%3Afalse%2C%22withDownvotePerspective%22%3Afalse%2C%22withReactionsMetadata%22%3Afalse%2C%22withReactionsPerspective%22%3Afalse%2C%22withSuperFollowsTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Afalse%2C%22__fs_interactive_text%22%3Afalse%2C%22__fs_dont_mention_me_view_api_enabled%22%3Afalse%7D" // nolint:lll } /** diff --git a/scraper/api_types_v2_test.go b/scraper/api_types_v2_test.go index ff3ba01..0f7d17e 100644 --- a/scraper/api_types_v2_test.go +++ b/scraper/api_types_v2_test.go @@ -34,7 +34,8 @@ func TestAPIV2ParseUser(t *testing.T) { assert.Equal(user.ID, UserID(44067298)) assert.Equal(user.DisplayName, "Michael Malice") assert.Equal(user.Handle, UserHandle("michaelmalice")) - assert.Equal(user.Bio, "Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model") + assert.Equal(user.Bio, "Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & " + + "Hubris by Harvey Pekar\nHe/Him ⚑\n@SheathUnderwear Model") assert.Equal(user.FollowingCount, 964) assert.Equal(user.FollowersCount, 334571) assert.Equal(user.Location, "Austin") @@ -69,7 +70,8 @@ func TestAPIV2ParseTweet(t *testing.T) { assert.True(ok) assert.Equal(tweet.ID, TweetID(1485708879174508550)) assert.Equal(tweet.UserID, UserID(44067298)) - assert.Equal(tweet.Text, "If Boris Johnson is driven out of office, it wouldn't mark the first time the Tories had four PMs in a row\nThey had previously governed the UK for 13 years with 4 PMs, from 1951-1964") + assert.Equal(tweet.Text, "If Boris Johnson is driven out of office, it wouldn't mark the first time the Tories had four PMs in a " + + "row\nThey had previously governed the UK for 13 years with 4 PMs, from 1951-1964") assert.Equal(tweet.PostedAt.Unix(), int64(1643055574)) assert.Equal(tweet.QuotedTweetID, TweetID(0)) assert.Equal(tweet.InReplyToID, TweetID(0)) @@ -131,7 +133,8 @@ func TestAPIV2ParseTweetWithQuotedTweet(t *testing.T) { assert.True(ok) assert.Equal(TweetID(1485690410899021826), quote_tweet.ID) assert.Equal(TweetID(1485690069079846915), quote_tweet.QuotedTweetID) - assert.Equal("Hatred is powerless in and of itself despite all the agitprop to the contrary\nHatred didnt stop Trump's election, for example", quote_tweet.Text) + assert.Equal("Hatred is powerless in and of itself despite all the agitprop to the contrary\nHatred didnt stop Trump's election, " + + "for example", quote_tweet.Text) // Should be 2 users: quoter and quoted assert.Equal(2, len(trove.Users)) @@ -323,7 +326,8 @@ func TestAPIV2ParseTweetWithURL(t *testing.T) { assert.Equal(1, len(trove.Tweets)) tweet, ok := trove.Tweets[1485695695025803264] assert.True(ok) - assert.Equal("This led to what I discussed as \"anguish signaling,\" where progs competed in proclaiming their distress both to show they were the Good Guys but also to get the pack to regroup, akin to wolves howling.", tweet.Text) + assert.Equal("This led to what I discussed as \"anguish signaling,\" where progs competed in proclaiming their distress both to " + + "show they were the Good Guys but also to get the pack to regroup, akin to wolves howling.", tweet.Text) assert.Equal(1, len(tweet.Urls)) url := tweet.Urls[0] @@ -331,7 +335,8 @@ func TestAPIV2ParseTweetWithURL(t *testing.T) { assert.Equal("observer.com", url.Domain) assert.Equal("Why Evangelical Progressives Need to Demonstrate Anguish Publicly", url.Title) assert.Equal("https://observer.com/2016/12/why-evangelical-progressives-need-to-demonstrate-anguish-publicly/", url.Text) - assert.Equal("The concept of “virtue signaling” gained a great deal of currency in this past year. It’s a way to demonstrate to others that one is a good person without having to do anything", url.Description) + assert.Equal("The concept of “virtue signaling” gained a great deal of currency in this past year. It’s a way to demonstrate to " + + "others that one is a good person without having to do anything", url.Description) assert.Equal("https://pbs.twimg.com/card_img/1485694664640507911/WsproWyP?format=jpg&name=600x600", url.ThumbnailRemoteUrl) assert.Equal(600, url.ThumbnailWidth) assert.Equal(300, url.ThumbnailHeight) diff --git a/scraper/guest_token_test.go b/scraper/guest_token_test.go index baeb76c..91abd33 100644 --- a/scraper/guest_token_test.go +++ b/scraper/guest_token_test.go @@ -24,6 +24,6 @@ func TestGetGuestToken(t *testing.T) { // other than the first use the cache. func BenchmarkGetGuestToken(b *testing.B) { for i := 0; i < b.N; i++ { - _, _ = GetGuestToken() + GetGuestToken() // nolint:errcheck } } diff --git a/scraper/tweet_test.go b/scraper/tweet_test.go index 13cb63c..2a86863 100644 --- a/scraper/tweet_test.go +++ b/scraper/tweet_test.go @@ -33,7 +33,8 @@ func TestParseSingleTweet(t *testing.T) { assert := assert.New(t) tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_unicode_chars.json") - assert.Equal("The fact that @michaelmalice new book ‘The Anarchist Handbook’ is just absolutely destroying on the charts is the largest white pill I’ve swallowed in years.", tweet.Text) + assert.Equal("The fact that @michaelmalice new book ‘The Anarchist Handbook’ is just absolutely destroying on the charts is the " + + "largest white pill I’ve swallowed in years.", tweet.Text) assert.Len(tweet.Mentions, 1) assert.Contains(tweet.Mentions, UserHandle("michaelmalice")) assert.Empty(tweet.Urls) @@ -72,7 +73,8 @@ func TestParseTweetWithQuotedTweetAndLink(t *testing.T) { assert := assert.New(t) tweet := load_tweet_from_file("test_responses/single_tweets/tweet_with_quoted_tweet_and_url.json") - assert.Equal("This is video he’s talking about. Please watch. Is there a single US politician capable of doing this with the weasels and rats running American industry today?", tweet.Text) + assert.Equal("This is video he’s talking about. Please watch. Is there a single US politician capable of doing this with the " + + "weasels and rats running American industry today?", tweet.Text) assert.Equal(TweetID(1497997890999898115), tweet.QuotedTweetID) assert.Len(tweet.Urls, 1) diff --git a/scraper/url_test.go b/scraper/url_test.go index f85e6c1..c4166d1 100644 --- a/scraper/url_test.go +++ b/scraper/url_test.go @@ -24,7 +24,8 @@ func TestParseAPIUrlCard(t *testing.T) { url := ParseAPIUrlCard(apiCard) assert.Equal("reason.com", url.Domain) assert.Equal("L.A. Teachers Union Leader: 'There's No Such Thing As Learning Loss'", url.Title) - assert.Equal("\"It’s OK that our babies may not have learned all their times tables,\" says Cecily Myart-Cruz. \"They learned resilience.\"", url.Description) + assert.Equal("\"It’s OK that our babies may not have learned all their times tables,\" says Cecily Myart-Cruz. \"They learned " + + "resilience.\"", url.Description) assert.Equal(600, url.ThumbnailWidth) assert.Equal(315, url.ThumbnailHeight) assert.Equal("https://pbs.twimg.com/card_img/1434998862305968129/odDi9EqO?format=jpg&name=600x600", url.ThumbnailRemoteUrl) @@ -48,7 +49,8 @@ func TestParseAPIUrlCardWithPlayer(t *testing.T) { url := ParseAPIUrlCard(apiCard) assert.Equal("www.youtube.com", url.Domain) assert.Equal("The Politically Incorrect Guide to the Constitution (Starring Tom...", url.Title) - assert.Equal("Watch this episode on LBRY/Odysee: https://odysee.com/@capitalresearch:5/the-politically-incorrect-guide-to-the:8Watch this episode on Rumble: https://rumble...", url.Description) + assert.Equal("Watch this episode on LBRY/Odysee: https://odysee.com/@capitalresearch:5/the-politically-incorrect-guide-to-the:8" + + "Watch this episode on Rumble: https://rumble...", url.Description) assert.Equal("https://pbs.twimg.com/card_img/1437849456423194639/_1t0btyt?format=jpg&name=800x320_1", url.ThumbnailRemoteUrl) assert.Equal("_1t0btyt_800x320_1.jpg", url.ThumbnailLocalPath) assert.Equal(UserID(10228272), url.SiteID) @@ -69,7 +71,8 @@ func TestParseAPIUrlCardWithPlayerAndPlaceholderThumbnail(t *testing.T) { url := ParseAPIUrlCard(apiCard) assert.Equal("www.youtube.com", url.Domain) assert.Equal("Did Michael Malice Turn Me into an Anarchist? | Ep 181", url.Title) - assert.Equal("SUBSCRIBE TO THE NEW SHOW W/ ELIJAH & SYDNEY: \"YOU ARE HERE\"YT: https://www.youtube.com/youareheredaily______________________________________________________...", url.Description) + assert.Equal("SUBSCRIBE TO THE NEW SHOW W/ ELIJAH & SYDNEY: \"YOU ARE HERE\"YT: https://www.youtube.com/youareheredaily____________" + + "__________________________________________...", url.Description) assert.Equal("https://pbs.twimg.com/cards/player-placeholder.png", url.ThumbnailRemoteUrl) assert.Equal("player-placeholder.png", url.ThumbnailLocalPath) assert.Equal(UserID(10228272), url.SiteID) diff --git a/scraper/user_test.go b/scraper/user_test.go index 166bd3d..bd8a923 100644 --- a/scraper/user_test.go +++ b/scraper/user_test.go @@ -31,7 +31,8 @@ func TestParseSingleUser(t *testing.T) { assert.Equal(UserID(44067298), user.ID) assert.Equal("Michael Malice", user.DisplayName) assert.Equal(UserHandle("michaelmalice"), user.Handle) - assert.Equal("Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by Harvey Pekar\nUnderwear Model\nHe/Him ⚑", user.Bio) + assert.Equal("Author of Dear Reader, The New Right & The Anarchist Handbook\nHost of \"YOUR WELCOME\" \nSubject of Ego & Hubris by " + + "Harvey Pekar\nUnderwear Model\nHe/Him ⚑", user.Bio) assert.Equal(941, user.FollowingCount) assert.Equal(208589, user.FollowersCount) assert.Equal("Brooklyn", user.Location) diff --git a/terminal_utils/formatting_test.go b/terminal_utils/formatting_test.go index cd4e659..6d2882c 100644 --- a/terminal_utils/formatting_test.go +++ b/terminal_utils/formatting_test.go @@ -14,7 +14,8 @@ func TestWrapParagraph(t *testing.T) { Expected []string } { { - "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do", + "These are public health officials who are making decisions about your lifestyle because they know more about health, " + + "fitness and well-being than you do", []string{ "These are public health officials who are making decisions", "about your lifestyle because they know more about health,", @@ -49,7 +50,8 @@ func TestWrapText(t *testing.T) { Expected string } { { - "These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do", + "These are public health officials who are making decisions about your lifestyle because they know more about health, " + + "fitness and well-being than you do", `These are public health officials who are making decisions about your lifestyle because they know more about health, fitness and well-being than you do`,