offline-labs/sqlite-lint
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Compare commits

base: offline-labs:81b801237ffbbacc1438b274071dfb48467fb863
Branches Tags
offline-labs:master
..
compare: offline-labs:5ff8e37c61224f581c3ccc2e0d7245c752a6946b
Branches Tags
offline-labs:master

No commits in common. "81b801237ffbbacc1438b274071dfb48467fb863" and "5ff8e37c61224f581c3ccc2e0d7245c752a6946b" have entirely different histories.
81b801237f ... 5ff8e37c61

15 changed files with 59 additions and 648 deletions
Show Stats Expand all files Collapse all files

69
.github/workflows/build.yaml vendored
View File

@ -1,69 +0,0 @@
name: Build
on:
push:
branches: ["*"] # Any branch
tags: ["v*.*.*"] # Release tags
jobs:
build:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Go
uses: actions/setup-go@v4
with:
go-version: "1.24"
- name: Install dependencies
run: |
go mod download
curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.0.1
- name: Lint
run: golangci-lint run
- name: Run tests
run: |
go test ./...
test-action:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Set up Docker
uses: docker/setup-buildx-action@v2
- name: Run SQLite schema lint action
uses: ./
with:
schema-file: 'test_schemas/success.sql'
# This step is expected to fail
- name: Run SQLite schema lint action with invalid schema
id: invalid_schema_test
uses: ./
with:
schema-file: 'test_schemas/failure-no-strict.sql'
continue-on-error: true
# This step should check that the previous step failed. If it was successful, this step should fail
- name: Check for expected failure
if: steps.invalid_schema_test.outcome != 'failure'
run: |
echo "Previous step result: ${{ steps.invalid_schema_test.outcome }}"
echo "Expected the invalid schema test to fail, but it succeeded"
exit 1
# This step should succeed, because the check that would fail is disabled
- name: Lint an invalid schema, but with the check disabled
uses: ./
with:
schema-file: 'test_schemas/failure-no-strict.sql'
require_strict: false

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
data
output.txt

87
.golangci.yaml
View File

@ -1,87 +0,0 @@
version: "2"
linters:
default: none
enable:
# Defaults
- errcheck
- govet
- ineffassign
- staticcheck
- unused
# Extras
- depguard
- errorlint
- godox
- lll
- nolintlint
- sqlclosecheck
- whitespace
- wrapcheck
settings:
depguard:
rules:
main:
deny:
- pkg: io/ioutil
desc: replace with the matching functions from `io` or `os` packages
- pkg: github.com/pkg/errors
desc: Should be replaced by standard lib errors package
errcheck:
# report about not checking of errors in type assertions: `a := b.(MyStruct)`;
# default is false: such cases aren't reported by default.
check-type-assertions: true
errorlint:
errorf: true # Ensure Errorf only uses %w (not %v or %s etc) for errors
asserts: true # Require errors.As instead of type-asserting
comparison: true # Require errors.Is instead of equality-checking
godox:
# report any comments starting with keywords, this is useful for TODO or FIXME comments that
# might be left in the code accidentally and should be resolved before merging
keywords: # default keywords are TODO, BUG, and FIXME, these can be overwritten by this setting
- XXX
govet:
enable-all: true
disable:
- fieldalignment
lll:
line-length: 140
tab-width: 4
nolintlint:
require-explanation: true
require-specific: true
allow-unused: false
staticcheck:
go: "1.24"
checks:
- all
- -ST1000 # Re-enable this once we have docstrings
- -ST1001 # Dot imports are good sometimes (e.g., in test packages)
- -ST1003 # snake_case is better for non-exported symbols
- -ST1013 # HTTP status codes are shorter and more readable than names
exclusions:
generated: lax # Don't lint generated files
paths:
formatters:
enable:
- gci
- gofmt
settings:
gci:
sections:
- standard
- default
- localmodule
gofmt:
simplify: true
exclusions:
generated: lax
paths:
issues:
max-same-issues: 0
max-issues-per-linter: 0
uniq-by-line: false

20
Dockerfile
View File

@ -1,17 +1,9 @@
from golang:alpine as builder
run apk add sqlite-dev build-base
copy . /code
workdir /code
env CGO_ENABLED=1
run go build -ldflags="-w -s -linkmode=external -extldflags=-static" -o sqlite_lint ./cmd/sqlite_lint/main.go
# ---
from alpine:3.20
COPY --from=builder /code/sqlite_lint /
run apk add sqlite
entrypoint ["/sqlite_lint"]
copy entrypoint.sh /
copy lints.sql /
run chmod +x /entrypoint.sh
entrypoint ["/entrypoint.sh"]

94
README.md
View File

@ -1,94 +0,0 @@
# SQLite Schema Lint
This GitHub Action lints SQLite schema files to enforce various constraints. It is designed to ensure that your SQLite schemas adhere to best practices.
You can also install it to validate schemas locally.
## Inputs
- **`schema-file`**: (Required) The SQL schema file to lint.
## Usage
To use this action in your workflow, include a step in your job that uses `playfulpachyderm/sqlite-lint`. Below is an example of how to set it up in a GitHub Actions workflow file:
```yaml
jobs:
lint:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Validate SQL schema
uses: playfulpachyderm/sqlite-lint@v1.0.0
with:
schema-file: pkg/db/schema.sql
```
## Installing
CGo is required. You will need a C compiler (and if it isn't `gcc` you might have to specify what it is as `CC=whatever` before installing).
```bash
CGO_ENABLED=1 go install github.com/playfulpachyderm/sqlite-lint/cmd/sqlite_lint@v1.0.2
sqlite_lint <path/to/schema.sql>
```
## Available Checks
All checks are enabled by default. To turn one off, you can use:
```yaml
with:
- schema-file: [...]
- require_not_null: false
```
This will disable the `require_not_null` check.
When running locally, you can use:
```bash
INPUT_REQUIRE_NOT_NULL=false sqlite_lint <path/to/schema.sql> # `require_not_null` check will be skipped
```
### `require_not_null`
Enforce that all columns should be marked as `not null` unless they are foreign keys.
**Explanation**: Nulls are a common source of unexpected bugs, because they're usually an invalid state but often get created by mistake (e.g., you forgot to set a value). Explicitly disabling nulls prevents such mistakes.
If you need to track "unset" values, add an `is_X_valid` column to make it explicit. Note that in many cases, `0` or `""` (empty string) are sufficient null values, and an `is_X_valid` flag might not even be required.
Foreign keys are exempt in this check because `null` is the only value the integrity checker will accept to represent "this row has no related item".
### `require_strict`
Enforce that all tables should be marked as `strict`.
**Explanation**: By default, SQLite is very loose with what it accepts, and basically doesn't enforce any type checking. "Strict" tables disable this "looseness" and enforce that inserted values match the stated type of the column.
"Strict" tables also limit to a small number of column types: `int`, `integer`, `real`, `text`, `blob` or `any`. To represent dates / times, use Unix epoch times in milliseconds, and convert to formatted dates (and timezones) only when displaying the value to a user. This is the most portable and least bug-prone method to handle dates.
See more about "strict" tables in SQLite's documentation: <https://sqlite.org/stricttables.html>
### `forbid_int_type`
Enforce that all columns should use `integer` type instead of `int`.
**Explanation**: This is an extension of "strict" tables, which allow two redundant integer types, `integer` and `int`. This check standardizes the types further, permitting only `integer`.
### `require_explicit_primary_key`
Enforce that all tables must have a primary key. If the primary key is `rowid`, it must be declared explicitly.
**Explanation**: All tables need to have a primary key, and it should usually be `rowid`. Declaring it explicitly improves the readability of the schema.
### `require_indexes_for_foreign_keys`
Enforce that columns referenced by foreign keys must have indexes.
**Explanation**: Foreign keys are usually used for `join`s. Joining on un-indexed columns is very slow. Ensuring that all foreign-key-referenced columns have indexes will greatly improve the performance of database operations.

26
action.yml
View File

@ -1,34 +1,10 @@
# action.yml
name: SQLite schema lint
description: Enforce constraints on SQLite schemas
inputs:
schema-file:
description: SQL schema file to lint
description: SQL schema file that will create
required: true
# List of checks
require_not_null:
description: Enforce that all columns should be marked as `not null` unless they are foreign keys.
required: false
default: true
require_strict:
description: Enforce that all tables should be marked as `strict`.
required: false
default: true
forbid_int_type:
description: Enforce that all columns should use `integer` type instead of `int`.
required: false
default: true
require_explicit_primary_key:
description: Enforce that all tables must have a primary key.
required: false
default: true
require_indexes_for_foreign_keys:
description: Enforce that columns referenced by foreign keys must have indexes.
required: false
default: true
runs:
using: 'docker'
image: 'Dockerfile'

84
cmd/sqlite_lint/main.go
View File

@ -1,84 +0,0 @@
package main
import (
"fmt"
"os"
"strings"
_ "github.com/mattn/go-sqlite3"
"github.com/playfulpachyderm/sqlite-lint/pkg/checks"
)
const (
GREEN = "\033[0;32m"
RED = "\033[0;31m"
RESET = "\033[0m"
)
func main() {
// Check if a filepath argument is provided
if len(os.Args) < 2 {
fmt.Println(RED + "Please provide a filepath as the first argument." + RESET)
os.Exit(1)
}
// Get the filepath from the first argument
filepath := os.Args[1]
fmt.Printf("-----------------\nLinting %s\n", filepath)
// Open the SQLite database
db, err := checks.OpenSchema(filepath)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
is_failure := false
// Execute each check against the database
for _, check := range checks.Checks {
// Checks can be disabled via Github config / environment variables
if !is_check_enabled(check) {
continue
}
results, err := check.Execute(db)
if err != nil {
panic(err)
}
// If there are results, print them as lint errors
if len(results) > 0 {
is_failure = true
fmt.Printf(RED+"Check '%s' failed:\n"+RESET, check.Name)
for _, result := range results {
fmt.Printf(RED+"- %s: %s.%s\n"+RESET, result.ErrorMsg, result.TableName, result.ColumnName)
}
fmt.Printf(RED+"Explanation: %s\n\n"+RESET, check.Explanation)
}
}
if is_failure {
fmt.Println(RED + "Errors found" + RESET)
os.Exit(1)
}
fmt.Println(GREEN + "Success" + RESET)
}
// github_actions_input_env_var converts an input name to the corresponding
// environment variable name used by GitHub Actions.
func github_actions_input_env_var(name string) string {
// GitHub normalizes both hyphens and underscores to underscores, then uppercases the name
normalized := strings.NewReplacer("-", "_", " ", "_").Replace(name)
return "INPUT_" + strings.ToUpper(normalized)
}
// Setting the environment variable INPUT_REQUIRE_NOT_NULL="false" disables the "require_not_null" check
func is_check_enabled(c checks.Check) bool {
val, is_set := os.LookupEnv(github_actions_input_env_var(c.Name))
if !is_set {
// Enable all checks by default
return true
}
// Anything except "false" is true
return val != "false"
}

23
entrypoint.sh Normal file
View File

@ -0,0 +1,23 @@
#!/bin/sh -l
set -x
set -e
if [ -z "$1" ]; then
echo "No SQL schema file given! Exiting..."
exit 1
fi
DB_PATH=/tmp/database.db
SCHEMA_PATH="$1"
pwd
echo $SCHEMA_PATH
# Create the database
sqlite3 $DB_PATH < $SCHEMA_PATH
sqlite3 -column -header $DB_PATH < /lints.sql | tee output.txt
if [ -s output.txt ]; then
echo "Some checks failed."
exit 2
fi

8
go.mod
View File

@ -1,8 +0,0 @@
module github.com/playfulpachyderm/sqlite-lint
go 1.24.0
require (
github.com/jmoiron/sqlx v1.4.0
github.com/mattn/go-sqlite3 v1.14.28
)

11
go.sum
View File

@ -1,11 +0,0 @@
filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
github.com/jmoiron/sqlx v1.4.0 h1:1PLqN7S1UYp5t4SrVVnt4nUVNemrDAtxlulVe+Qgm3o=
github.com/jmoiron/sqlx v1.4.0/go.mod h1:ZrZ7UsYB/weZdl2Bxg6jCRO9c3YHl8r3ahlKmRT4JLY=
github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/mattn/go-sqlite3 v1.14.28 h1:ThEiQrnbtumT+QMknw63Befp/ce/nUPgBPMlRFEum7A=
github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=

5
lints.sql
View File

@ -1,8 +1,3 @@
/*
* NOTE: This file is kept for historical reference. These checks have all been moved into the Go
* implementation, except the 'require_default_values' check. That one is TODO.
*/
create temporary view tables as
select l.*
from sqlite_schema s

128
pkg/checks/check.go
View File

@ -1,128 +0,0 @@
package checks
import (
"fmt"
"github.com/jmoiron/sqlx"
)
// Check represents a database check with a name, SQL statement, and an explanation.
type Check struct {
Name string
Sql string
Explanation string
}
// CheckResult represents a row in the query result with error message, table name, and column name.
type CheckResult struct {
ErrorMsg string `db:"error_msg"`
TableName string `db:"table_name"`
ColumnName string `db:"column_name"`
}
// Execute runs the SQL statement of the Check against the provided database and returns the
// resulting rows as a slice of CheckResult using sqlx.
func (c *Check) Execute(db *sqlx.DB) ([]CheckResult, error) {
var results []CheckResult
// return results, nil
// println(c.Sql)
if err := db.Select(&results, c.Sql); err != nil {
return nil, fmt.Errorf("failed to execute check '%s': %w", c.Name, err)
}
return results, nil
}
var Checks = map[string]Check{
"require_not_null": {
Name: "require_not_null",
Sql: `
select 'Column should should be "not null"' as error_msg,
table_name,
column_name
from columns
where columns."notnull" = 0
and fk_target_column is null
and is_primary_key = 0 -- primary keys are automatically not-null, but aren't listed as such in pragma_table_info
`,
Explanation: "All columns should be marked as `not null` unless they are foreign keys. (Primary keys are\n" +
"automatically not-null, and don't need to be specified.)",
},
// {
// Name: "require_default_values",
// Sql: `
// select 'Column should have a default value' as error_msg,
// table_name,
// column_name
// from columns
// where dflt_value is null
// and fk_target_column is null
// and is_primary_key = 0;
// `,
// Explanation: "All columns should have a default value specified, unless they are foreign keys or primary keys.",
// },
"require_strict": {
Name: "require_strict",
Sql: `
select 'Table should be marked "strict"' as error_msg,
name as table_name,
'' as column_name
from tables
where strict = 0;
`,
Explanation: "All tables should be marked as `strict` (must specify column types; types must be int,\n" +
"integer, real, text, blob or any). This disallows all 'date' and 'time' column types.\n" +
"See more: https://www.sqlite.org/stricttables.html",
},
"forbid_int_type": {
Name: "forbid_int_type",
Sql: `
select 'Column should use "integer" type instead of "int"' as error_msg,
table_name,
column_name
from columns
where column_type like 'int';
`,
Explanation: "All columns should use `integer` type instead of `int`.",
},
"require_explicit_primary_key": {
Name: "require_explicit_primary_key",
Sql: `
select 'Table should declare an explicit primary key' as error_msg,
tables.name as table_name,
'' as column_name
from tables
where not exists (select 1 from pragma_table_info(tables.name) where pk != 0);
`,
Explanation: "All tables must have a primary key. If it's rowid, it has to be named explicitly.",
},
"require_indexes_for_foreign_keys": {
Name: "require_indexes_for_foreign_keys",
Sql: `
with index_info as (
select tables.name as table_name,
columns.name as column_name
from tables
join pragma_index_list(tables.name) as indexes
join pragma_index_info(indexes.name) as columns
union
select table_name,
column_name
from columns
where column_name = 'rowid'
and is_primary_key != 0 -- 'pk' is either 0, or the 1-based index of the column within the primary key
), foreign_keys as (
select * from columns where fk_target_column is not null
)
select 'Foreign keys should point to indexed columns' as error_msg,
foreign_keys.table_name as table_name,
foreign_keys.column_name as column_name
from foreign_keys
left join index_info on foreign_keys.fk_target_table = index_info.table_name
and foreign_keys.fk_target_column = index_info.column_name
where index_info.column_name is null;
`,
Explanation: "Columns referenced by foreign keys must have indexes.",
},
}

71
pkg/checks/check_test.go
View File

@ -1,71 +0,0 @@
package checks_test
import (
"slices"
"testing"
_ "github.com/mattn/go-sqlite3"
"github.com/playfulpachyderm/sqlite-lint/pkg/checks"
)
func TestFailureCases(t *testing.T) {
test_cases := []struct {
sqlFile string
expectedFailures []string
}{
{"../../test_schemas/failure-has-foreign-key-no-index.sql", []string{"require_indexes_for_foreign_keys"}},
{"../../test_schemas/failure-has-ints.sql", []string{"forbid_int_type"}},
{"../../test_schemas/failure-has-nulls.sql", []string{"require_not_null"}},
{"../../test_schemas/failure-no-strict.sql", []string{"require_strict"}},
{"../../test_schemas/failure-total.sql", []string{
"require_not_null",
"require_explicit_primary_key",
"forbid_int_type",
"require_strict",
"require_indexes_for_foreign_keys",
}},
}
for _, test_case := range test_cases {
db, err := checks.OpenSchema(test_case.sqlFile)
if err != nil {
t.Fatalf("failed to open database: %v", err)
}
for _, check := range checks.Checks {
results, err := check.Execute(db)
if err != nil {
t.Errorf("failed to execute check '%s': %v", check.Name, err)
}
is_failure := len(results) > 0
is_failure_expected := slices.Contains(test_case.expectedFailures, check.Name)
if is_failure != is_failure_expected {
if is_failure_expected {
t.Errorf("Expected check '%s' to fail, but it passed: %s", check.Name, test_case.sqlFile)
} else {
t.Errorf("Expected check '%s' to pass, but it failed: %s", check.Name, test_case.sqlFile)
}
}
}
}
}
func TestSuccessCase(t *testing.T) {
db, err := checks.OpenSchema("../../test_schemas/success.sql")
if err != nil {
t.Fatalf("failed to open database: %v", err)
}
for _, check := range checks.Checks {
results, err := check.Execute(db)
if err != nil {
t.Errorf("failed to execute check '%s': %v", check.Name, err)
}
if len(results) > 0 {
t.Errorf("Should have passed, but didn't: %s", "test_schemas/success.sql")
}
}
}

52
pkg/checks/target.go
View File

@ -1,52 +0,0 @@
package checks
import (
"fmt"
"os"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
)
// OpenSchema opens a SQLite database in memory, executes the schema against it, and adds some views
func OpenSchema(filepath string) (*sqlx.DB, error) {
// Open a SQLite database in memory
db, err := sqlx.Open("sqlite3", ":memory:")
if err != nil {
return nil, fmt.Errorf("failed to open in-memory database: %w", err)
}
// Read the SQL file
sqlBytes, err := os.ReadFile(filepath)
if err != nil {
return nil, fmt.Errorf("failed to read SQL file: %w", err)
}
// Execute the SQL statements
db.MustExec(string(sqlBytes))
// Execute the SQL statements for creating views
db.MustExec(`
create view tables as
select l.*
from sqlite_schema s
left join pragma_table_list l on s.name = l.name
where s.type = 'table';
create view columns as
select tables.name as table_name,
table_info.name as column_name,
table_info.type as column_type,
"notnull",
dflt_value,
pk as is_primary_key,
fk."table" as fk_target_table,
fk."to" as fk_target_column
from tables
join pragma_table_info(tables.name) as table_info
left join pragma_foreign_key_list(tables.name) as fk on fk."from" = column_name;
`)
return db, nil
}

27
test.sh Executable file
View File

@ -0,0 +1,27 @@
#!/bin/sh
rm data/*
for file in test_schemas/failure-*; do
echo "Testing '$file'"
test -e output.txt && rm output.txt
db_path="data/$(basename $file).db"
sqlite3 $db_path < $file
sqlite3 -column -header $db_path < lints.sql | tee output.txt
if [ ! -s output.txt ]; then
echo "Should have failed, but didn't: $file"
exit 1
fi
done
file="test_schemas/success.sql"
echo "Testing '$file'"
test -e output.txt && rm output.txt
db_path="data/$(basename $file).db"
sqlite3 $db_path < $file
sqlite3 -column -header $db_path < lints.sql | tee output.txt
if [ -s output.txt ]; then
echo "Should have passed, but didn't: $file"
exit 1
fi
echo "Tests passed!"