ci: add fts5 build tag for tests

.gitea/workflows/CI.yaml

@@ -3,14 +3,14 @@ name: CI
 on: [push]
 
 jobs:
-  # These steps build the `gas/ci` docker image.
+  # These steps build the `gas` docker image.
   # There's a tricky bootstrapping problem here.  We'll try to cheat first, and if that doesn't
   # work, then we'll do a full bootstrapping build starting from Alpine and installing `gocheckout`
   # first thing.
   build-docker:
     continue-on-error: true
     container:
-      image: gas/ci
+      image: gas
       volumes:
         - woodpecker-gocache:/go-cache-volume
       env:
@@ -23,7 +23,7 @@ jobs:
         run: gocheckout
 
       - name: build CI container
-        run: docker build -t gas/ci ops/devcontainer
+        run: ops/devcontainer/build.sh
 
   # Fallback: full bootstrap build.  This one is quite slow.
   build-docker-bootstrap:
@@ -48,16 +48,16 @@ jobs:
           gocheckout
 
       - name: install docker
-        run: apk add docker
+        run: apk add sudo docker
 
       - name: build CI container
-        run: docker build -t gas/ci ops/devcontainer
+        run: ops/devcontainer/build.sh
 
   release-test:
     needs: build-docker-bootstrap
     if: "!failure()"
     container:
-      image: gas/ci
+      image: gas
       volumes:
         - woodpecker-gocache:/go-cache-volume
       env:
@@ -74,7 +74,7 @@ jobs:
           gocheckout
 
       - name: test
-        run: go test ./...
+        run: go test -tags fts5 ./...
 
       - name: lint
         run: golangci-lint run

cmd/subcmd_generate_models.go

@@ -57,13 +57,22 @@ var generate_model = &cobra.Command{
 								Name: ast.NewIdent("."),
 								Path: &ast.BasicLit{Kind: token.STRING, Value: `"git.offline-twitter.com/offline-labs/gas-stack/pkg/db"`},
 							},
+							&ast.ImportSpec{
+								Name: ast.NewIdent("."),
+								Path: &ast.BasicLit{
+									Kind:  token.STRING,
+									Value: `"git.offline-twitter.com/offline-labs/gas-stack/pkg/flowutils"`,
+								},
+							},
 						},
 					},
 					modelgenerate.GenerateIDType(table),
 					modelgenerate.GenerateModelAST(table),
+					modelgenerate.GenerateSQLFieldsConst(table),
 					modelgenerate.GenerateSaveItemFunc(table),
-					modelgenerate.GenerateGetItemByIDFunc(table),
 					modelgenerate.GenerateDeleteItemFunc(table),
+					modelgenerate.GenerateGetItemByIDFunc(table),
+					modelgenerate.GenerateGetAllItemsFunc(table),
 				},
 			}
 

doc/TODO.txt

@@ -5,14 +5,13 @@ TODO: auto-timestamps
 - SaveXyz shouldn't set created_at in the do-update branch
 - GetXyzByID should include `ErrItemIsDeleted` if item is soft-deleted
 
-TODO: primary-key
-- SaveXyz shouldn't set primary key if it's rowid
-
-TODO: foreign-key
-- Generated types should match foreign keys to the type of the column they point to
-
 TODO: modified-timestamps
 - set updated_at and created_at in SaveXYZ
 - soft delete option
 
-TODO: the `db_meta` table doesn't pass sqlite_lint
+TODO: generator-foreign-keys
+- add auto-foreign-key checking blocks to SaveXyz
+
+TODO: migration-structs
+- Right now, migrations are strings.  Could be a struct with "name", "up" and "down" fields
+- Adding a "down" operation enables handling newer DB versions with "down instead of error-out" for development (perhaps a flag)

doc/gas-stack.md

@@ -27,6 +27,41 @@
 - sample data
 	- https://git.tnd.gg/tnd/remichat/pulls/65#issuecomment-2035
 
+- `ops` directory
+
+### Borrowed from Ruby On Rails
+
+- Convention Over Configuration
+- scaffolding (code generators)
+- database migration system
+- focus on testing
+- ORM-like affordances (but not actually using an ORM)
+
+## Vendoring vs Package Management vs In-Sourcing
+
+Dependencies are technical debt.  See [Loris Cro's talk about "How To Write Better Software with Zig"](https://www.youtube.com/watch?v=AEybWzeAkho).
+
+## Scaffolding
+
+Scaffolding is not boilerplate code, generated code, or library code.  It's *starter code* which is *intended to be modified* as needed.
+
+Library code is provided as a pre-made, off-the-shelf solution.  If your problem is exactly the one the library is intended to solve, and the library does a good job, you should use it.  Lots of big stuff is like this; nobody implements their own HTTP server or SQL engine as part of an application, because the domain is big, stable and standardized.  It makes perfect sense to use libraries for this.
+
+Boilerplate code and generated code (the latter being a common solution to the former) are usually indicators of bad abstractions.  If there's truly something that needs to be done exactly the same way by rote, every time, then there should be a reusable library for it-- or maybe you're even using the wrong programming language.
+
+Scaffolding isn't either of those, because scaffolding is intended to be *changed*.  It's just a starting point as you flesh out your ideas.
+
+Consider [this parable](https://rcrowley.org/2022/rails-django-parable.html) comparing Rails and Django on their initial setup and tutorial.  The analogy isn't perfect, but his claim is basically that the original Rails official tutorial left you with a tiny app with almost on code and a huge amount of functionality; but since all the functionality was invisible "magic" provided by Rails, as soon as you want something custom, you're nearly starting from scratch'.  By comparison, the Django tutorial produces a large amount of code which makes the abstractions explicit.  The author refers to all those extra lines of code as "footholds", from which you can start working.
+
+Scaffolding *begins* as generic boilerplate, but evolves as your application logic becomes more custom and requirements change.  One piece of scaffolded code might never change, because the scaffolding was good enough; another piece might be tweaked over time, as you add more to it (or remove parts you don't need); and another piece might change so much that no traces of the original scaffolding existed.  One app could contain all three of these.
+
+Scaffolding is intended to make "in-sourcing" your code easier, by getting you to something bare-bones-but-working faster.
+
+## Dynamic vs Static linking
+
+TODO: write about this and why it matters for the GAS stack
+
+
 ## SQLite and ROWID
 
 Tables must be EITHER:

doc/inspirations.txt

@@ -0,0 +1,7 @@
+- Urbit
+- BCHS stack
+- Ruby on Rails
+- Hasen Judi's "Data Storage and Retrieval From First Principles": https://hasen.substack.com/p/data-storage-and-retrieval
+- Matklad's "Basic Things": https://matklad.github.io/2024/03/22/basic-things.html
+- Zig, Andrew Kelley, Loris Cro
+- Max Tagher's "8 Lints for your Postgres Schema": https://mercury.com/blog/lints-for-postgres-schema

doc/rails-scaffolding-comparison.md

@@ -0,0 +1,41 @@
+See: https://guides.rubyonrails.org/v3.2/getting_started.html#getting-up-and-running-quickly-with-scaffolding
+
+# DB
+
+db/migrate/20100207214725_create_posts.rb:
+	- Migration to create the posts table in your database (your name will include a different timestamp)
+app/models/post.rb:
+	- The Post model
+test/unit/post_test.rb:
+	- Unit testing harness for the posts model
+test/fixtures/posts.yml:
+	- Sample posts for use in testing
+
+# Web
+
+config/routes.rb:
+	- Edited to include routing information for posts
+app/controllers/posts_controller.rb:
+	- The Posts controller
+app/views/posts/index.html.erb:
+	- A view to display an index of all posts
+app/views/posts/edit.html.erb:
+	- A view to edit an existing post
+app/views/posts/show.html.erb:
+	- A view to display a single post
+app/views/posts/new.html.erb:
+	- A view to create a new post
+app/views/posts/_form.html.erb:
+	- A partial to control the overall look and feel of the form used in edit and new views
+test/functional/posts_controller_test.rb:
+	- Functional testing harness for the posts controller
+app/helpers/posts_helper.rb:
+	- Helper functions to be used from the post views
+test/unit/helpers/posts_helper_test.rb:
+	- Unit testing harness for the posts helper
+app/assets/javascripts/posts.js.coffee:
+	- CoffeeScript for the posts controller
+app/assets/stylesheets/posts.css.scss:
+	- Cascading style sheet for the posts controller
+app/assets/stylesheets/scaffolds.css.scss:
+	- Cascading style sheet to make the scaffolded views look better

doc/sqlite-schema-rules-and-linter.md

@@ -0,0 +1,88 @@
+# SQLite Schema Rules and Linter
+
+The `sqlite_lint` subcommand enforces some rules that the GAS stack considers best-practices.
+
+All checks are enabled by default.  Disabling checks isn't recommended; many GAS stack methodologies assume your schema is designed in accordance with these rules, and will be less effective if you don't follow them.
+
+Currently the only way to disable them is setting an environment variable with the check name in capitals prefixed with `INPUT_`, e.g., `INPUT_REQUIRE_NOT_NULL=false` disables the `require_not_null` check. 
+
+```bash
+INPUT_REQUIRE_NOT_NULL=false gas sqlite_lint <path/to/schema.sql>  # `require_not_null` check will be skipped
+```
+
+## Running the linter
+
+```bash
+gas sqlite_lint <path/to/schema.sql>
+```
+
+## Available Checks
+
+This is a list of currently available checks.
+
+### `require_not_null`
+
+Enforce that all columns should be marked as `not null`, unless they are foreign keys.
+
+**Explanation**: 
+
+- Nulls are a common source of unexpected bugs, because they're usually an invalid state but often get created by mistake (e.g., you forgot to set a value).  Explicitly disabling nulls prevents such mistakes.
+- If the "natural zero value" is a valid value in your application and you explicitly need to distinguish it from "missing data", use an `has_xyz` or `is_xyz_valid` flag of some kind, rather than a nullable field.
+  - This is usually unnecessary, because the natural zero-values `0` and `""` (empty string) are usually sufficient to indicate "no value".  This is called a "sentinel value", or "in-band null value", because you don't need a special data type (null) to declare absence of data.
+- Foreign keys are exempt in this check, because `null` is a special value the integrity checker uses to say "this row has no related item".
+
+### `require_strict`
+
+Enforce that all tables should be marked as `strict`.
+
+**Explanation**:
+
+- By default, SQLite tables are very loose with what values they accept, and don't enforce any type checking.   "Strict" disables this "looseness", and enforces that inserted values match the stated type of the column.
+- "Strict" tables also limit to a small number of column types: `int`, `integer`, `real`, `text`, `blob` or `any`.
+  - To represent dates / times, use Unix epoch times in milliseconds, and convert to formatted dates (and timezones) only when displaying the value to a user.  This is the most portable and least bug-prone method to handle dates.
+
+See more about "strict" tables in SQLite's documentation: <https://sqlite.org/stricttables.html>
+
+### `forbid_int_type`
+
+Enforce that all columns should use `integer` type instead of `int`.
+
+**Explanation**:
+
+- This is an extension of "strict" tables, which allow two redundant integer types, `integer` and `int`.  This check standardizes the types further, permitting only `integer`.
+
+### `require_explicit_primary_key`
+
+Enforce that all tables must have an explicitly declared primary key.
+
+**Explanation**: 
+
+- All tables need to have a primary key for storage reasons, so if you don't declare one, SQLite will auto-generate a hidden "rowid" column.  Making it explicit (rowid or otherwise) improves schema readability.
+
+See more about the special behavior of `rowid` in SQLite's documentation: <https://sqlite.org/lang_createtable.html#rowid>
+
+### `require_explicit_rowid`
+
+Enforce that any table that's not declared `without rowid` has an explicit `rowid integer primary key` column.
+
+**Explanation**:
+
+- In SQLite, all tables implicitly have a `rowid` column unless they are declared `without rowid`.  Making it explicit improves schema readability.
+
+See more about the `without rowid` modifier in SQLite's documentation: <https://sqlite.org/withoutrowid.html>
+
+### `forbid_rowid_on_without_rowid_table`
+
+Enforce that `without rowid` tables don't have a rowid column.
+
+**Explanation**:
+
+- This is pretty self explanatory.  You can technically give a `without rowid` table a rowid column.  But don't.
+
+### `require_indexes_for_foreign_keys`
+
+Enforce that columns referenced by foreign keys must have indexes.
+
+**Explanation**: 
+
+- Foreign keys are usually used for `join`s.  Joining on un-indexed columns is very slow.  Ensuring that all foreign-key-referenced columns have indexes will greatly improve the performance of database operations.

doc/using-devcontainer.md

@@ -0,0 +1,63 @@
+# Dev containers
+
+A container is a great development environment.  However, they tend to underperform because:
+1. it's easy to fall into the Docker Compose trap
+1. people don't use Alpine
+
+Using a dev container has multiple benefits:
+
+- "infrastructure as code", a.k.a. .  Your codebase itself (in the `ops/devcontainer` dir) defines explicitly defines all the tools and dependencies that you use, with .
+- reproducibility: a few simple commands to create a clean working setup anywhere Docker is supported (i.e., anywhere)
+- isolation: you have clean setup and teardown.  You can install a bunch of crap to try it out, and you don't have to remember what it was so you can purge it afterward.  Just delete the container.
+
+## Quick start
+
+Build the container:
+```bash
+ops/devcontainer/build.sh
+```
+
+Run the container:
+```bash
+ops/devcontainer/start.sh
+```
+
+## Concepts
+
+A dev container is meant to be short-lived, constantly thrown away and recreated as needed.  This explicitly divides the filesystem into "keep" (the source tree and any useful artifacts / caches) and "throw away" (everything else).  Frequently regenerating the container ensures that "your environment" never deviates too far from the Infrastructure As Code in your repo; it forces you to add any new tools to the Docker image build process.  In service of this, using `docker run --rm [...]` is always recommended.
+
+Contrary to conventional container ideology, it is not necessary to keep your images tiny and minimize container layers at all costs.  For example, conventional container ideology frequently suggests constructs like `RUN cmd1 && cmd2 && cmd3` rather than doing each `cmd` in its own `RUN` layer, in order to reduce the amount of layers generated from 3 to 1.  These practices are optimized for massive horizontal deployments, where you have a gazillion containers and images, and resource usage is a big problem.  Obviously, the GAS stack is the complete opposite; you want very few containers, ideally just 1 at most.  So having *more* layers is actually better, because it speeds up rebuild times by avoiding very heavy, frequently rebuilt layers.  It also makes the Dockerfile much easier to read.
+
+## Methodologies
+
+There's a few useful techniques and strategies when using dev containers:
+
+- user management
+- volumes for code
+- volumes for caching
+- openrc services
+- `--net host`
+
+### User management
+
+To make working in a dev container seamless, create a user on it that matches your host machine user (UID and GID).
+
+If you don't do this, Git will complain about conflicting ownership, and any tools or tests that create files will create them as "root", which then have to be constantly `chown`'d on the host.
+
+To make this work, it's necessary to have a `build.sh` script which passes the current user's UID as a build arg to the Docker build step.
+
+### Volumes
+
+Anything not in a volume (or built into the image) will be lost on container restart.  I like to mount the codebase on `/code`.
+
+For compiled languages (or anything that needs to "build" the project, e.g., linters), mounting build cache directories can also be useful.
+
+### OpenRC
+
+OpenRC is much simpler than systemd.  If you want to run background processes, or network services, making the root process OpenRC and writing an openrc service script is the best effort-to-value ratio.  ChatGPT can help you write openrc service scripts.
+
+### `--net host`
+
+Because this is a dev container, it's meant to make your life easier, not get you tangled up in security best-practices and so forth.  One of the biggest annoyances of using containers is having to do port mapping, which leads to an explosion of config.
+
+Using `docker run --net host [...]` makes the container use the host's networking, instead of creating a virtual network that you have to explicitly map ports back and forth between.

ops/compile.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+go build -tags fts5 -o gas ./cmd

ops/devcontainer/Dockerfile

@@ -2,7 +2,24 @@
 
 FROM alpine:3.22
 
-RUN apk add build-base git go sqlite shellcheck curl jq bash docker
+RUN apk add build-base git go sqlite vim shellcheck curl jq sudo bash docker
+
+# Busybox `less` doesn't appear to support colors (makes git diff lose color)
+RUN apk add less
 
 RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v2.0.2
 RUN GOBIN=/usr/local/bin go install git.offline-twitter.com/offline-labs/gocheckout@v0.0.2
+
+# Create a user in the container with the same UID as on the host machine, to avoid ownership conflicts.
+# The user gets sudo of course.
+#
+# If the host user is `root` (uid = 0), we skip creating a new user, because it will fail otherwise.
+ARG USERNAME
+ARG UID
+ARG GID
+RUN if [ "${UID}" -ne 0 ]; then \
+    addgroup -g "${GID}" "${USERNAME}" && \
+    adduser -D -u "${UID}" -G "${USERNAME}" "${USERNAME}" && \
+    echo "${USERNAME} ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/"${USERNAME}"; \
+    fi
+USER ${USERNAME}

ops/devcontainer/build.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+SCRIPT_PATH=$(cd "$(dirname "$0")" && pwd)
+
+sudo docker build  \
+    --build-arg USERNAME="$(whoami)" \
+    --build-arg UID="$(id -u)" \
+    --build-arg GID="$(id -g)" \
+    -t gas "$SCRIPT_PATH"

ops/devcontainer/start.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+sudo docker run --rm -it \
+    -v "$(pwd)":/code \
+    -v "$(go env GOCACHE):/gocache-vol" \
+    -e GOCACHE=/gocache-vol \
+    -v "$(go env GOMODCACHE):/gocache-vol/mod-cache" \
+    -e GOMODCACHE=/gocache-vol/mod-cache \
+    -e GOLANGCI_LINT_CACHE=/gocache-vol/lint-cache \
+    --workdir /code \
+    --net host \
+    gas

ops/gas_init_test.sh

@@ -13,7 +13,8 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.."
 
 # Compile `gas`
 gas="/tmp/gas"
-go build -o $gas ./cmd
+ops/compile.sh
+mv gas $gas
 
 test_project="/memory/test_gasproj"
 if [[ -e $test_project ]]; then
@@ -33,7 +34,7 @@ cat >> pkg/db/schema.sql <<EOF
 create table items (
     rowid integer primary key,
     description text not null default ''
-);
+) strict;
 EOF
 
 # Generate an item model and test file
@@ -44,5 +45,8 @@ go mod tidy
 # Run the tests
 go test ./...
 
+# Run sqlite_lint
+$gas sqlite_lint pkg/db/schema.sql
+
 # Notify success in green
 echo -e "\033[32mAll tests passed.  Finished successfully.\033[0m"

ops/sqlite_lint_test.sh

@@ -13,7 +13,8 @@ cd "$(dirname "${BASH_SOURCE[0]}")/.."
 
 # Compile `gas`
 gas="/tmp/gas"
-go build -o $gas ./cmd
+ops/compile.sh
+mv gas $gas
 
 test_schema_dir="pkg/schema/lint/test_schemas"
 

pkg/codegen/modelgenerate/generate_model.go

@@ -14,10 +14,10 @@ import (
 )
 
 func GenerateIDType(table schema.Table) *ast.GenDecl {
-	// e.g., `type FoodID uint64`
+	// e.g., `type FoodID int`
 	return &ast.GenDecl{
 		Tok:   token.TYPE,
-		Specs: []ast.Spec{&ast.TypeSpec{Name: ast.NewIdent(table.TypeIDName), Type: ast.NewIdent("uint64")}},
+		Specs: []ast.Spec{&ast.TypeSpec{Name: ast.NewIdent(table.TypeIDName), Type: ast.NewIdent("int")}},
 	}
 }
 
@@ -40,7 +40,7 @@ func GenerateModelAST(table schema.Table) *ast.GenDecl {
 			if col.IsForeignKey && strings.HasSuffix(col.Name, "_id") {
 				fields = append(fields, &ast.Field{
 					Names: []*ast.Ident{ast.NewIdent(textutils.SnakeToCamel(strings.TrimSuffix(col.Name, "_id")) + "ID")},
-					Type:  ast.NewIdent(textutils.SnakeToCamel(inflection.Singular(col.ForeignKeyTargetTable) + "ID")),
+					Type:  ast.NewIdent(schema.TypenameFromTablename(col.ForeignKeyTargetTable) + "ID"),
 					Tag:   &ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("`db:\"%s\" json:\"%s\"`", col.Name, col.Name)},
 				})
 			} else {
@@ -76,7 +76,7 @@ func GenerateModelAST(table schema.Table) *ast.GenDecl {
 	return &ast.GenDecl{
 		Tok: token.TYPE,
 		Specs: []ast.Spec{&ast.TypeSpec{
-			Name: ast.NewIdent(table.TypeName),
+			Name: ast.NewIdent(table.GoTypeName),
 			Type: &ast.StructType{Fields: &ast.FieldList{List: fields}},
 		}},
 	}
@@ -116,37 +116,62 @@ func GenerateSaveItemFunc(tbl schema.Table) *ast.FuncDecl {
 				Body: &ast.BlockStmt{
 					List: []ast.Stmt{
 						&ast.AssignStmt{
-							Lhs: []ast.Expr{ast.NewIdent("result"), ast.NewIdent("err")},
+							Lhs: []ast.Expr{ast.NewIdent("result")},
 							Tok: token.DEFINE,
-							Rhs: []ast.Expr{&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("NamedExec")}, Args: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: "`" + insertStmt + "`"}, ast.NewIdent(tbl.VarName)}}},
+							Rhs: []ast.Expr{&ast.CallExpr{
+								Fun: ast.NewIdent("Must"),
+								Args: []ast.Expr{&ast.CallExpr{
+									Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("NamedExec")},
+									Args: []ast.Expr{
+										&ast.BasicLit{Kind: token.STRING, Value: "`" + insertStmt + "`"},
+										ast.NewIdent(tbl.VarName),
+									},
+								}},
+							}},
 						},
-						&ast.IfStmt{Cond: &ast.BinaryExpr{X: ast.NewIdent("err"), Op: token.NEQ, Y: ast.NewIdent("nil")}, Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{ast.NewIdent("err")}}}}}},
 						&ast.AssignStmt{
-							Lhs: []ast.Expr{ast.NewIdent("id"), ast.NewIdent("err")},
+							Lhs: []ast.Expr{&ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")}},
-							Tok: token.DEFINE,
+							Tok: token.ASSIGN,
-							Rhs: []ast.Expr{&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("result"), Sel: ast.NewIdent("LastInsertId")}, Args: []ast.Expr{}}},
+							Rhs: []ast.Expr{&ast.CallExpr{
+								Fun: ast.NewIdent(tbl.TypeIDName),
+								Args: []ast.Expr{&ast.CallExpr{
+									Fun: ast.NewIdent("Must"),
+									Args: []ast.Expr{&ast.CallExpr{
+										Fun:  &ast.SelectorExpr{X: ast.NewIdent("result"), Sel: ast.NewIdent("LastInsertId")},
+										Args: []ast.Expr{},
+									}},
+								}},
+							}},
 						},
-						&ast.IfStmt{Cond: &ast.BinaryExpr{X: ast.NewIdent("err"), Op: token.NEQ, Y: ast.NewIdent("nil")}, Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{ast.NewIdent("err")}}}}}},
-						&ast.AssignStmt{Lhs: []ast.Expr{&ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")}}, Tok: token.ASSIGN, Rhs: []ast.Expr{&ast.CallExpr{Fun: ast.NewIdent(tbl.TypeIDName), Args: []ast.Expr{ast.NewIdent("id")}}}},
 					},
 				},
 				Else: &ast.BlockStmt{
 					List: []ast.Stmt{
 						&ast.AssignStmt{
-							Lhs: []ast.Expr{ast.NewIdent("result"), ast.NewIdent("err")},
+							Lhs: []ast.Expr{ast.NewIdent("result")},
 							Tok: token.DEFINE,
-							Rhs: []ast.Expr{&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("NamedExec")}, Args: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: "`" + updateStmt + "`"}, ast.NewIdent(tbl.VarName)}}},
+							Rhs: []ast.Expr{&ast.CallExpr{
+								Fun: ast.NewIdent("Must"),
+								Args: []ast.Expr{&ast.CallExpr{
+									Fun:  &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("NamedExec")},
+									Args: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: "`" + updateStmt + "`"}, ast.NewIdent(tbl.VarName)},
+								}},
+							}},
 						},
-						&ast.IfStmt{Cond: &ast.BinaryExpr{X: ast.NewIdent("err"), Op: token.NEQ, Y: ast.NewIdent("nil")}, Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{ast.NewIdent("err")}}}}}},
+
-						&ast.AssignStmt{
-							Lhs: []ast.Expr{ast.NewIdent("count"), ast.NewIdent("err")},
-							Tok: token.DEFINE,
-							Rhs: []ast.Expr{&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("result"), Sel: ast.NewIdent("RowsAffected")}, Args: []ast.Expr{}}},
-						},
-						&ast.IfStmt{Cond: &ast.BinaryExpr{X: ast.NewIdent("err"), Op: token.NEQ, Y: ast.NewIdent("nil")}, Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{ast.NewIdent("err")}}}}}},
 						&ast.IfStmt{
-							Cond: &ast.BinaryExpr{X: ast.NewIdent("count"), Op: token.NEQ, Y: &ast.BasicLit{Kind: token.INT, Value: "1"}},
+							Cond: &ast.BinaryExpr{
-							Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{&ast.CallExpr{Fun: ast.NewIdent("fmt.Errorf"), Args: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("\"got %s with ID (%%d), so attempted update, but it doesn't exist\"", strings.ToLower(tbl.TypeName))}, &ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")}}}}}}}},
+								X: &ast.CallExpr{
+									Fun: ast.NewIdent("Must"),
+									Args: []ast.Expr{&ast.CallExpr{
+										Fun:  &ast.SelectorExpr{X: ast.NewIdent("result"), Sel: ast.NewIdent("RowsAffected")},
+										Args: []ast.Expr{},
+									}},
+								},
+								Op: token.NEQ,
+								Y:  &ast.BasicLit{Kind: token.INT, Value: "1"},
+							},
+							Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{&ast.CallExpr{Fun: ast.NewIdent("fmt.Errorf"), Args: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("\"got %s with ID (%%d), so attempted update, but it doesn't exist\"", strings.ToLower(tbl.GoTypeName))}, &ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")}}}}}}}},
 						},
 					},
 				},
@@ -156,9 +181,9 @@ func GenerateSaveItemFunc(tbl schema.Table) *ast.FuncDecl {
 
 	funcDecl := &ast.FuncDecl{
 		Recv: &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("db")}, Type: ast.NewIdent("DB")}}},
-		Name: ast.NewIdent("Save" + tbl.TypeName),
+		Name: ast.NewIdent("Save" + tbl.GoTypeName),
 		Type: &ast.FuncType{
-			Params:  &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent(tbl.VarName)}, Type: &ast.StarExpr{X: ast.NewIdent(tbl.TypeName)}}}},
+			Params:  &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent(tbl.VarName)}, Type: &ast.StarExpr{X: ast.NewIdent(tbl.GoTypeName)}}}},
 			Results: nil,
 		},
 		Body: funcBody,
@@ -169,28 +194,33 @@ func GenerateSaveItemFunc(tbl schema.Table) *ast.FuncDecl {
 // GenerateGetItemByIDFunc produces an AST for the `GetXyzByID()` function.
 // E.g., a table with `table.TypeName = "foods"` will produce a "GetFoodByID()" function.
 func GenerateGetItemByIDFunc(tbl schema.Table) *ast.FuncDecl {
-	funcName := "Get" + tbl.TypeName + "ByID"
+	funcName := "Get" + tbl.GoTypeName + "ByID"
 
 	recv := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("db")}, Type: ast.NewIdent("DB")}}}
 	arg := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("id")}, Type: ast.NewIdent(tbl.TypeIDName)}}}
-	result := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("ret")}, Type: ast.NewIdent(tbl.TypeName)}, {Names: []*ast.Ident{ast.NewIdent("err")}, Type: ast.NewIdent("error")}}}
+	result := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("ret")}, Type: ast.NewIdent(tbl.GoTypeName)}, {Names: []*ast.Ident{ast.NewIdent("err")}, Type: ast.NewIdent("error")}}}
 
-	selectCols := make([]string, 0, len(tbl.Columns))
+	// Use the xyzSQLFields constant in the select query
-	for _, col := range tbl.Columns {
+	selectExpr := &ast.BinaryExpr{
-		selectCols = append(selectCols, col.Name)
+		X: &ast.BinaryExpr{
+			X:  &ast.BasicLit{Kind: token.STRING, Value: "`\n\t    select `"},
+			Op: token.ADD,
+			Y:  SQLFieldsConstIdent(tbl),
+		},
+		Op: token.ADD,
+		Y:  &ast.BasicLit{Kind: token.STRING, Value: "`\n\t      from " + tbl.TableName + "\n\t     where rowid = ?\n\t`"},
 	}
-	selectStmt := fmt.Sprintf("\n\t    select %s\n\t      from %s\n\t     where rowid = ?\n\t", strings.Join(selectCols, ", "), tbl.TableName)
 
 	funcBody := &ast.BlockStmt{
 		List: []ast.Stmt{
 			&ast.AssignStmt{
 				Lhs: []ast.Expr{ast.NewIdent("err")},
 				Tok: token.ASSIGN,
-				Rhs: []ast.Expr{&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("Get")}, Args: []ast.Expr{&ast.UnaryExpr{Op: token.AND, X: ast.NewIdent("ret")}, &ast.BasicLit{Kind: token.STRING, Value: "`" + selectStmt + "`"}, ast.NewIdent("id")}}},
+				Rhs: []ast.Expr{&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("Get")}, Args: []ast.Expr{&ast.UnaryExpr{Op: token.AND, X: ast.NewIdent("ret")}, selectExpr, ast.NewIdent("id")}}},
 			},
 			&ast.IfStmt{
 				Cond: &ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("errors"), Sel: ast.NewIdent("Is")}, Args: []ast.Expr{ast.NewIdent("err"), &ast.SelectorExpr{X: ast.NewIdent("sql"), Sel: ast.NewIdent("ErrNoRows")}}},
-				Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ReturnStmt{Results: []ast.Expr{&ast.CompositeLit{Type: ast.NewIdent(tbl.TypeName)}, ast.NewIdent("ErrNotInDB")}}}},
+				Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ReturnStmt{Results: []ast.Expr{&ast.CompositeLit{Type: ast.NewIdent(tbl.GoTypeName)}, ast.NewIdent("ErrNotInDB")}}}},
 			},
 			&ast.ReturnStmt{},
 		},
@@ -205,30 +235,105 @@ func GenerateGetItemByIDFunc(tbl schema.Table) *ast.FuncDecl {
 	return funcDecl
 }
 
+// GenerateGetAllItemsFunc produces an AST for the `GetAllXyzs()` function.
+// E.g., a table with `table.TypeName = "foods"` will produce a "GetAllFoods()" function.
+func GenerateGetAllItemsFunc(tbl schema.Table) *ast.FuncDecl {
+	funcName := "GetAll" + inflection.Plural(tbl.GoTypeName)
+	recv := &ast.FieldList{List: []*ast.Field{
+		{Names: []*ast.Ident{ast.NewIdent("db")}, Type: ast.NewIdent("DB")},
+	}}
+	result := &ast.FieldList{List: []*ast.Field{
+		{Names: []*ast.Ident{ast.NewIdent("ret")}, Type: &ast.ArrayType{Elt: ast.NewIdent(tbl.GoTypeName)}},
+	}}
+
+	selectCall := &ast.CallExpr{
+		Fun: ast.NewIdent("PanicIf"),
+		Args: []ast.Expr{
+			&ast.CallExpr{
+				Fun: &ast.SelectorExpr{
+					X:   ast.NewIdent("db.DB"),
+					Sel: ast.NewIdent("Select"),
+				},
+				Args: []ast.Expr{
+					&ast.UnaryExpr{Op: token.AND, X: ast.NewIdent("ret")},
+					&ast.BinaryExpr{
+						X: &ast.BinaryExpr{
+							X:  &ast.BasicLit{Kind: token.STRING, Value: "`SELECT `"},
+							Op: token.ADD,
+							Y:  SQLFieldsConstIdent(tbl),
+						},
+						Op: token.ADD,
+						Y:  &ast.BasicLit{Kind: token.STRING, Value: "` FROM " + tbl.TableName + "`"},
+					},
+				},
+			},
+		},
+	}
+
+	funcBody := &ast.BlockStmt{
+		List: []ast.Stmt{
+			&ast.ExprStmt{X: selectCall},
+			&ast.ReturnStmt{},
+		},
+	}
+
+	return &ast.FuncDecl{
+		Recv: recv,
+		Name: ast.NewIdent(funcName),
+		Type: &ast.FuncType{
+			Params:  &ast.FieldList{},
+			Results: result,
+		},
+		Body: funcBody,
+	}
+}
+
 // GenerateDeleteItemFunc produces an AST for the `DeleteXyz()` function.
 // E.g., a table with `table.TypeName = "foods"` will produce a "DeleteFood()" function.
 func GenerateDeleteItemFunc(tbl schema.Table) *ast.FuncDecl {
-	funcName := "Delete" + tbl.TypeName
+	funcName := "Delete" + tbl.GoTypeName
 	recv := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("db")}, Type: ast.NewIdent("DB")}}}
-	arg := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent(tbl.VarName)}, Type: ast.NewIdent(tbl.TypeName)}}}
+	arg := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent(tbl.VarName)}, Type: ast.NewIdent(tbl.GoTypeName)}}}
 
 	funcBody := &ast.BlockStmt{
 		List: []ast.Stmt{
 			&ast.AssignStmt{
-				Lhs: []ast.Expr{ast.NewIdent("result"), ast.NewIdent("err")},
+				Lhs: []ast.Expr{ast.NewIdent("result")},
 				Tok: token.DEFINE,
-				Rhs: []ast.Expr{&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("Exec")}, Args: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: "`delete from " + tbl.TableName + " where rowid = ?`"}, &ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")}}}},
+				Rhs: []ast.Expr{&ast.CallExpr{
+					Fun: ast.NewIdent("Must"),
+					Args: []ast.Expr{&ast.CallExpr{
+						Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("Exec")},
+						Args: []ast.Expr{
+							&ast.BasicLit{Kind: token.STRING, Value: "`delete from " + tbl.TableName + " where rowid = ?`"},
+							&ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")},
 						},
-			&ast.IfStmt{Cond: &ast.BinaryExpr{X: ast.NewIdent("err"), Op: token.NEQ, Y: ast.NewIdent("nil")}, Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{ast.NewIdent("err")}}}}}},
+					}},
-			&ast.AssignStmt{
+				}},
-				Lhs: []ast.Expr{ast.NewIdent("count"), ast.NewIdent("err")},
-				Tok: token.DEFINE,
-				Rhs: []ast.Expr{&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("result"), Sel: ast.NewIdent("RowsAffected")}, Args: []ast.Expr{}}},
 			},
-			&ast.IfStmt{Cond: &ast.BinaryExpr{X: ast.NewIdent("err"), Op: token.NEQ, Y: ast.NewIdent("nil")}, Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{ast.NewIdent("err")}}}}}},
 			&ast.IfStmt{
-				Cond: &ast.BinaryExpr{X: ast.NewIdent("count"), Op: token.NEQ, Y: &ast.BasicLit{Kind: token.INT, Value: "1"}},
+				Cond: &ast.BinaryExpr{
-				Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ExprStmt{X: &ast.CallExpr{Fun: ast.NewIdent("panic"), Args: []ast.Expr{&ast.CallExpr{Fun: ast.NewIdent("fmt.Errorf"), Args: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("\"tried to delete %s with ID (%%d) but it doesn't exist\"", strings.ToLower(tbl.TypeName))}, &ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")}}}}}}}},
+					X: &ast.CallExpr{
+						Fun: ast.NewIdent("Must"),
+						Args: []ast.Expr{
+							&ast.CallExpr{Fun: &ast.SelectorExpr{X: ast.NewIdent("result"), Sel: ast.NewIdent("RowsAffected")}, Args: []ast.Expr{}},
+						},
+					},
+					Op: token.NEQ,
+					Y:  &ast.BasicLit{Kind: token.INT, Value: "1"},
+				},
+				Body: &ast.BlockStmt{List: []ast.Stmt{
+					&ast.ExprStmt{X: &ast.CallExpr{
+						Fun: ast.NewIdent("panic"),
+						Args: []ast.Expr{&ast.CallExpr{
+							Fun: ast.NewIdent("fmt.Errorf"),
+							Args: []ast.Expr{
+								&ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("\"tried to delete %s with ID (%%d) but it doesn't exist\"", strings.ToLower(tbl.GoTypeName))},
+								&ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")},
+							},
+						}},
+					}},
+				}},
 			},
 		},
 	}
@@ -241,3 +346,31 @@ func GenerateDeleteItemFunc(tbl schema.Table) *ast.FuncDecl {
 	}
 	return funcDecl
 }
+
+// GenerateSQLFieldsConst produces an AST for the `const xyzSQLFields = ...` string.
+func GenerateSQLFieldsConst(tbl schema.Table) *ast.GenDecl {
+	columns := make([]string, 0, len(tbl.Columns))
+	for _, col := range tbl.Columns {
+		columns = append(columns, col.Name)
+	}
+	// Join with comma and space
+	value := "`" + strings.Join(columns, ", ") + "`"
+
+	return &ast.GenDecl{
+		Tok: token.CONST,
+		Specs: []ast.Spec{
+			&ast.ValueSpec{
+				Names:  []*ast.Ident{SQLFieldsConstIdent(tbl)},
+				Values: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: value}},
+			},
+		},
+	}
+}
+
+// ---------------
+// Helpers
+// ---------------
+
+func SQLFieldsConstIdent(tbl schema.Table) *ast.Ident {
+	return ast.NewIdent(strings.ToLower(tbl.GoTypeName) + "SQLFields")
+}

pkg/codegen/modelgenerate/generate_testfile.go

@@ -5,6 +5,8 @@ import (
 	"go/ast"
 	"go/token"
 
+	"github.com/jinzhu/inflection"
+
 	"git.offline-twitter.com/offline-labs/gas-stack/pkg/schema"
 )
 
@@ -87,6 +89,154 @@ func GenerateModelTestAST(tbl schema.Table, gomodName string) *ast.File {
 	description1 := `"an item"`
 	description2 := `"a big item"`
 
+	testCreateUpdateDelete := &ast.FuncDecl{
+		Name: ast.NewIdent("TestCreateUpdateDelete" + tbl.GoTypeName),
+		Type: &ast.FuncType{
+			Params: &ast.FieldList{
+				List: []*ast.Field{{
+					Names: []*ast.Ident{ast.NewIdent("t")},
+					Type:  ast.NewIdent("*testing.T"),
+				}},
+			},
+		},
+		Body: &ast.BlockStmt{
+			List: []ast.Stmt{
+				// item := Item{Description: "an item"}
+				&ast.AssignStmt{
+					Lhs: []ast.Expr{testObj},
+					Tok: token.DEFINE,
+					Rhs: []ast.Expr{&ast.CompositeLit{
+						Type: ast.NewIdent(tbl.GoTypeName),
+						Elts: []ast.Expr{
+							&ast.KeyValueExpr{
+								Key:   fieldName,
+								Value: &ast.BasicLit{Kind: token.STRING, Value: description1},
+							},
+						},
+					}},
+				},
+
+				// TestDB.SaveItem(&item)
+				&ast.ExprStmt{X: &ast.CallExpr{
+					Fun:  ast.NewIdent("TestDB.Save" + tbl.GoTypeName),
+					Args: []ast.Expr{&ast.UnaryExpr{Op: token.AND, X: testObj}},
+				}},
+
+				// require.NotZero(t, item.ID)
+				&ast.ExprStmt{X: &ast.CallExpr{
+					Fun:  ast.NewIdent("require.NotZero"),
+					Args: []ast.Expr{ast.NewIdent("t"), &ast.SelectorExpr{X: testObj, Sel: ast.NewIdent("ID")}},
+				}},
+
+				// item2 := Must(TestDB.GetItemByID(item.ID))
+				&ast.AssignStmt{
+					Lhs: []ast.Expr{testObj2},
+					Tok: token.DEFINE,
+					Rhs: []ast.Expr{&ast.CallExpr{
+						Fun: ast.NewIdent("Must"),
+						Args: []ast.Expr{&ast.CallExpr{
+							Fun:  ast.NewIdent("TestDB.Get" + tbl.GoTypeName + "ByID"),
+							Args: []ast.Expr{&ast.SelectorExpr{X: testObj, Sel: ast.NewIdent("ID")}},
+						}},
+					}},
+				},
+
+				// assert.Equal(t, "an item", item2.Description)
+				&ast.ExprStmt{X: &ast.CallExpr{
+					Fun: ast.NewIdent("assert.Equal"),
+					Args: []ast.Expr{
+						ast.NewIdent("t"),
+						&ast.BasicLit{Kind: token.STRING, Value: description1},
+						&ast.SelectorExpr{X: testObj2, Sel: fieldName},
+					},
+				}},
+
+				// item.Description = "a big item"
+				&ast.AssignStmt{
+					Lhs: []ast.Expr{&ast.SelectorExpr{X: testObj, Sel: fieldName}},
+					Tok: token.ASSIGN,
+					Rhs: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: description2}},
+				},
+
+				// TestDB.SaveItem(&item)
+				&ast.ExprStmt{X: &ast.CallExpr{
+					Fun:  ast.NewIdent("TestDB.Save" + tbl.GoTypeName),
+					Args: []ast.Expr{&ast.UnaryExpr{Op: token.AND, X: testObj}},
+				}},
+
+				// item2 = Must(TestDB.GetItemByID(item.ID))
+				&ast.AssignStmt{
+					Lhs: []ast.Expr{testObj2},
+					Tok: token.ASSIGN,
+					Rhs: []ast.Expr{&ast.CallExpr{
+						Fun: ast.NewIdent("Must"),
+						Args: []ast.Expr{&ast.CallExpr{
+							Fun:  ast.NewIdent("TestDB.Get" + tbl.GoTypeName + "ByID"),
+							Args: []ast.Expr{&ast.SelectorExpr{X: testObj, Sel: ast.NewIdent("ID")}},
+						}},
+					}},
+				},
+
+				// assert.Equal(t, item.Description, item2.Description)
+				&ast.ExprStmt{X: &ast.CallExpr{
+					Fun: ast.NewIdent("assert.Equal"),
+					Args: []ast.Expr{
+						ast.NewIdent("t"),
+						&ast.SelectorExpr{X: testObj, Sel: fieldName},
+						&ast.SelectorExpr{X: testObj2, Sel: fieldName},
+					},
+				}},
+
+				// TestDB.DeleteItem(item)
+				&ast.ExprStmt{X: &ast.CallExpr{
+					Fun:  ast.NewIdent("TestDB.Delete" + tbl.GoTypeName),
+					Args: []ast.Expr{testObj},
+				}},
+
+				// _, err := TestDB.GetItemByID(item.ID)
+				&ast.AssignStmt{
+					Lhs: []ast.Expr{ast.NewIdent("_"), ast.NewIdent("err")},
+					Tok: token.DEFINE,
+					Rhs: []ast.Expr{&ast.CallExpr{
+						Fun:  ast.NewIdent("TestDB.Get" + tbl.GoTypeName + "ByID"),
+						Args: []ast.Expr{&ast.SelectorExpr{X: testObj, Sel: ast.NewIdent("ID")}},
+					}},
+				},
+
+				// assert.ErrorIs(t, err, db.ErrNotInDB)
+				&ast.ExprStmt{X: &ast.CallExpr{
+					Fun: ast.NewIdent("assert.ErrorIs"),
+					Args: []ast.Expr{
+						ast.NewIdent("t"),
+						ast.NewIdent("err"),
+						&ast.SelectorExpr{X: ast.NewIdent("db"), Sel: ast.NewIdent("ErrNotInDB")},
+					},
+				}},
+			},
+		},
+	}
+
+	testGetAll := &ast.FuncDecl{
+		Name: ast.NewIdent("TestGetAll" + inflection.Plural(tbl.GoTypeName)),
+		Type: &ast.FuncType{Params: &ast.FieldList{List: []*ast.Field{
+			{Names: []*ast.Ident{ast.NewIdent("t")}, Type: &ast.StarExpr{X: ast.NewIdent("testing.T")}},
+		}}, Results: nil},
+		Body: &ast.BlockStmt{
+			List: []ast.Stmt{
+				&ast.AssignStmt{
+					Lhs: []ast.Expr{ast.NewIdent("_")},
+					Tok: token.ASSIGN,
+					Rhs: []ast.Expr{&ast.CallExpr{
+						Fun: &ast.SelectorExpr{
+							X:   ast.NewIdent("TestDB"),
+							Sel: ast.NewIdent("GetAll" + inflection.Plural(tbl.GoTypeName)),
+						},
+					}},
+				},
+			},
+		},
+	}
+
 	return &ast.File{
 		Name: ast.NewIdent(testpackageName),
 		Decls: []ast.Decl{
@@ -120,132 +270,8 @@ func GenerateModelTestAST(tbl schema.Table, gomodName string) *ast.File {
 			// func MakeDB(dbName string) *DB { db := Must(Create(fmt.Sprintf("file:%s?mode=memory&cache=shared", dbName))); return db }
 			makeDBHelperDecl,
 
-			&ast.FuncDecl{
+			testCreateUpdateDelete,
-				Name: ast.NewIdent("TestCreateUpdateDelete" + tbl.TypeName),
+			testGetAll,
-				Type: &ast.FuncType{
-					Params: &ast.FieldList{
-						List: []*ast.Field{{
-							Names: []*ast.Ident{ast.NewIdent("t")},
-							Type:  ast.NewIdent("*testing.T"),
-						}},
-					},
-				},
-				Body: &ast.BlockStmt{
-					List: []ast.Stmt{
-						// item := Item{Description: "an item"}
-						&ast.AssignStmt{
-							Lhs: []ast.Expr{testObj},
-							Tok: token.DEFINE,
-							Rhs: []ast.Expr{&ast.CompositeLit{
-								Type: ast.NewIdent(tbl.TypeName),
-								Elts: []ast.Expr{
-									&ast.KeyValueExpr{
-										Key:   fieldName,
-										Value: &ast.BasicLit{Kind: token.STRING, Value: description1},
-									},
-								},
-							}},
-						},
-
-						// TestDB.SaveItem(&item)
-						&ast.ExprStmt{X: &ast.CallExpr{
-							Fun:  ast.NewIdent("TestDB.Save" + tbl.TypeName),
-							Args: []ast.Expr{&ast.UnaryExpr{Op: token.AND, X: testObj}},
-						}},
-
-						// require.NotZero(t, item.ID)
-						&ast.ExprStmt{X: &ast.CallExpr{
-							Fun:  ast.NewIdent("require.NotZero"),
-							Args: []ast.Expr{ast.NewIdent("t"), &ast.SelectorExpr{X: testObj, Sel: ast.NewIdent("ID")}},
-						}},
-
-						// item2 := Must(TestDB.GetItemByID(item.ID))
-						&ast.AssignStmt{
-							Lhs: []ast.Expr{testObj2},
-							Tok: token.DEFINE,
-							Rhs: []ast.Expr{&ast.CallExpr{
-								Fun: ast.NewIdent("Must"),
-								Args: []ast.Expr{&ast.CallExpr{
-									Fun:  ast.NewIdent("TestDB.Get" + tbl.TypeName + "ByID"),
-									Args: []ast.Expr{&ast.SelectorExpr{X: testObj, Sel: ast.NewIdent("ID")}},
-								}},
-							}},
-						},
-
-						// assert.Equal(t, "an item", item2.Description)
-						&ast.ExprStmt{X: &ast.CallExpr{
-							Fun: ast.NewIdent("assert.Equal"),
-							Args: []ast.Expr{
-								ast.NewIdent("t"),
-								&ast.BasicLit{Kind: token.STRING, Value: description1},
-								&ast.SelectorExpr{X: testObj2, Sel: fieldName},
-							},
-						}},
-
-						// item.Description = "a big item"
-						&ast.AssignStmt{
-							Lhs: []ast.Expr{&ast.SelectorExpr{X: testObj, Sel: fieldName}},
-							Tok: token.ASSIGN,
-							Rhs: []ast.Expr{&ast.BasicLit{Kind: token.STRING, Value: description2}},
-						},
-
-						// TestDB.SaveItem(&item)
-						&ast.ExprStmt{X: &ast.CallExpr{
-							Fun:  ast.NewIdent("TestDB.Save" + tbl.TypeName),
-							Args: []ast.Expr{&ast.UnaryExpr{Op: token.AND, X: testObj}},
-						}},
-
-						// item2 = Must(TestDB.GetItemByID(item.ID))
-						&ast.AssignStmt{
-							Lhs: []ast.Expr{testObj2},
-							Tok: token.ASSIGN,
-							Rhs: []ast.Expr{&ast.CallExpr{
-								Fun: ast.NewIdent("Must"),
-								Args: []ast.Expr{&ast.CallExpr{
-									Fun:  ast.NewIdent("TestDB.Get" + tbl.TypeName + "ByID"),
-									Args: []ast.Expr{&ast.SelectorExpr{X: testObj, Sel: ast.NewIdent("ID")}},
-								}},
-							}},
-						},
-
-						// assert.Equal(t, item.Description, item2.Description)
-						&ast.ExprStmt{X: &ast.CallExpr{
-							Fun: ast.NewIdent("assert.Equal"),
-							Args: []ast.Expr{
-								ast.NewIdent("t"),
-								&ast.SelectorExpr{X: testObj, Sel: fieldName},
-								&ast.SelectorExpr{X: testObj2, Sel: fieldName},
-							},
-						}},
-
-						// TestDB.DeleteItem(item)
-						&ast.ExprStmt{X: &ast.CallExpr{
-							Fun:  ast.NewIdent("TestDB.Delete" + tbl.TypeName),
-							Args: []ast.Expr{testObj},
-						}},
-
-						// _, err := TestDB.GetItemByID(item.ID)
-						&ast.AssignStmt{
-							Lhs: []ast.Expr{ast.NewIdent("_"), ast.NewIdent("err")},
-							Tok: token.DEFINE,
-							Rhs: []ast.Expr{&ast.CallExpr{
-								Fun:  ast.NewIdent("TestDB.Get" + tbl.TypeName + "ByID"),
-								Args: []ast.Expr{&ast.SelectorExpr{X: testObj, Sel: ast.NewIdent("ID")}},
-							}},
-						},
-
-						// assert.ErrorIs(t, err, db.ErrNotInDB)
-						&ast.ExprStmt{X: &ast.CallExpr{
-							Fun: ast.NewIdent("assert.ErrorIs"),
-							Args: []ast.Expr{
-								ast.NewIdent("t"),
-								ast.NewIdent("err"),
-								&ast.SelectorExpr{X: ast.NewIdent("db"), Sel: ast.NewIdent("ErrNotInDB")},
-							},
-						}},
-					},
-				},
-			},
 		},
 	}
 }

pkg/codegen/tpl/schema.sql

@@ -5,6 +5,6 @@ PRAGMA foreign_keys = on;
 -- =======
 
 create table db_version (
-    version integer not null
+    version integer primary key
-) strict;
+) strict, without rowid;
 insert into db_version values(0);

pkg/db/errors.go

@@ -15,7 +15,7 @@ var (
 )
 
 type ForeignKey interface {
-	~uint64 | ~string
+	~int
 }
 
 type ForeignKeyError[T ForeignKey] struct {
@@ -65,6 +65,18 @@ func IsSqliteUniqError(err error) bool {
 	return errors.Is(sqliteErr.ExtendedCode, sqlite3.ErrConstraintUnique)
 }
 
+// IsSqlitePrimaryKeyError checks whether an error is a SQLite `primary key` constraint violation.
+//
+// TODO: it's kind of annoying that SQLite returns this instead of whether the violation was
+// .     uniqueness or null-ness.
+func IsSqlitePrimaryKeyError(err error) bool {
+	var sqliteErr sqlite3.Error
+	if !errors.As(err, &sqliteErr) {
+		return false
+	}
+	return errors.Is(sqliteErr.ExtendedCode, sqlite3.ErrConstraintPrimaryKey)
+}
+
 // IsSqliteNotNullError checks whether an error is a SQLite not null constraint violation.
 func IsSqliteNotNullError(err error) bool {
 	var sqliteErr sqlite3.Error

pkg/schema/lint/checks.go

@@ -23,7 +23,10 @@ var Checks = []Check{
 		Explanation: "All columns should be marked as `not null` unless they are foreign keys.  (Primary keys are\n" +
 			"automatically not-null, and don't need to be specified.)",
 		Execute: func(s schema.Schema) (ret []CheckResult) {
-			for tablename := range s.Tables {
+			for tablename, tbl := range s.Tables {
+				if tbl.TableType != "table" {
+					continue
+				}
 				for _, column := range s.Tables[tablename].Columns {
 					if !column.IsNotNull && !column.IsForeignKey && !column.IsPrimaryKey {
 						ret = append(ret, CheckResult{
@@ -43,7 +46,10 @@ var Checks = []Check{
 			"integer, real, text, blob or any).  This disallows all 'date' and 'time' column types.\n" +
 			"See more: https://www.sqlite.org/stricttables.html",
 		Execute: func(s schema.Schema) (ret []CheckResult) {
-			for tablename := range s.Tables {
+			for tablename, tbl := range s.Tables {
+				if tbl.TableType != "table" {
+					continue
+				}
 				if !s.Tables[tablename].IsStrict {
 					ret = append(ret, CheckResult{
 						ErrorMsg:   "Table should be marked \"strict\"",
@@ -78,7 +84,10 @@ var Checks = []Check{
 		Explanation: "All tables must have a primary key.  If it's rowid, it has to be named explicitly.",
 		Execute: func(s schema.Schema) (ret []CheckResult) {
 		tableloop:
-			for tablename := range s.Tables {
+			for tablename, tbl := range s.Tables {
+				if tbl.TableType != "table" {
+					continue
+				}
 				for _, column := range s.Tables[tablename].Columns {
 					if column.IsPrimaryKey {
 						continue tableloop
@@ -133,4 +142,66 @@ var Checks = []Check{
 			return
 		},
 	},
+	{
+		Name:        "forbid_rowid_on_without_rowid_table",
+		Explanation: "Tables that are `without rowid` may not have a `rowid` column",
+		Execute: func(s schema.Schema) (ret []CheckResult) {
+			for tblName, tbl := range s.Tables {
+				if !tbl.IsWithoutRowid {
+					continue
+				}
+				for _, column := range tbl.Columns {
+					if column.Name == "rowid" {
+						ret = append(ret, CheckResult{
+							ErrorMsg:   "rowid on 'without rowid' table",
+							TableName:  tblName,
+							ColumnName: column.Name,
+						})
+					}
+				}
+			}
+			return
+		},
+	},
+	{
+		Name: "require_explicit_rowid",
+		Explanation: "All tables should have an explicit `rowid integer primary key` column, unless they\n" +
+			"are declared `without rowid`.",
+		Execute: func(s schema.Schema) (ret []CheckResult) {
+		tbl_loop:
+			for tblName, tbl := range s.Tables {
+				if tbl.TableType != "table" {
+					continue
+				}
+				if tbl.IsWithoutRowid {
+					continue
+				}
+				for _, column := range tbl.Columns {
+					if column.Name == "rowid" {
+						if !column.IsPrimaryKey {
+							ret = append(ret, CheckResult{
+								ErrorMsg:   "`rowid` column not declared \"primary key\"",
+								TableName:  tblName,
+								ColumnName: column.Name,
+							})
+						}
+						if column.Type != "integer" {
+							ret = append(ret, CheckResult{
+								ErrorMsg:   "non-integer `rowid` column",
+								TableName:  tblName,
+								ColumnName: column.Name,
+							})
+						}
+						continue tbl_loop
+					}
+				}
+				ret = append(ret, CheckResult{
+					ErrorMsg:   "no `rowid` column",
+					TableName:  tblName,
+					ColumnName: "",
+				})
+			}
+			return
+		},
+	},
 }

pkg/schema/lint/checks_test.go

@@ -20,12 +20,17 @@ func TestFailureCases(t *testing.T) {
 		{"test_schemas/failure-has-ints.sql", []string{"forbid_int_type"}},
 		{"test_schemas/failure-has-nulls.sql", []string{"require_not_null"}},
 		{"test_schemas/failure-no-strict.sql", []string{"require_strict"}},
+		{"test_schemas/failure-rowid-on-without-rowid.sql", []string{"forbid_rowid_on_without_rowid_table"}},
+		{"test_schemas/failure-missing-rowid.sql", []string{"require_explicit_rowid"}},
+		{"test_schemas/failure-non-integer-rowid.sql", []string{"require_explicit_rowid"}},
+		{"test_schemas/failure-non-primary-key-rowid.sql", []string{"require_explicit_rowid"}},
 		{"test_schemas/failure-total.sql", []string{
 			"require_not_null",
 			"require_explicit_primary_key",
 			"forbid_int_type",
 			"require_strict",
 			"require_indexes_for_foreign_keys",
+			"require_explicit_rowid",
 		}},
 	}
 

pkg/schema/lint/test_schemas/failure-has-foreign-key-no-index.sql

@@ -5,7 +5,7 @@ create table stuff (
 ) strict;
 
 create table stuff2 (
-    weird_pk integer primary key,
+    rowid integer primary key,
     label text not null unique,
     stuff_id integer references stuff(rowid),
     alternative_stuff_id integer references stuff(amount)

pkg/schema/lint/test_schemas/failure-has-ints.sql

@@ -6,7 +6,7 @@ create table stuff (
 create index index_stuff_amount on stuff (amount);
 
 create table stuff2 (
-    weird_pk integer primary key,
+    rowid integer primary key,
     label text not null unique,
     stuff_id int references stuff(rowid),
     alternative_stuff_id integer references stuff(amount)

pkg/schema/lint/test_schemas/failure-has-nulls.sql

@@ -6,7 +6,7 @@ create table stuff (
 create index index_stuff_amount on stuff (amount);
 
 create table stuff2 (
-    weird_pk integer primary key,
+    rowid integer primary key,
     label text not null unique,
     stuff_id integer references stuff(rowid),
     alternative_stuff_id integer references stuff(amount)

pkg/schema/lint/test_schemas/failure-missing-rowid.sql

@@ -0,0 +1,3 @@
+create table missing_rowid (
+    a integer primary key
+) strict;

pkg/schema/lint/test_schemas/failure-no-strict.sql

@@ -6,7 +6,7 @@ create table stuff (
 create index index_stuff_amount on stuff (amount);
 
 create table stuff2 (
-    weird_pk integer primary key,
+    rowid integer primary key,
     label text not null unique,
     stuff_id integer references stuff(rowid),
     alternative_stuff_id integer references stuff(amount)

pkg/schema/lint/test_schemas/failure-non-integer-rowid.sql

@@ -0,0 +1,4 @@
+create table non_integer_rowid (
+    rowid text primary key,
+    a integer not null
+) strict;

pkg/schema/lint/test_schemas/failure-non-primary-key-rowid.sql

@@ -0,0 +1,4 @@
+create table withrowidbutnotprimarykey (
+    rowid integer not null,
+    a integer primary key
+) strict;

pkg/schema/lint/test_schemas/failure-rowid-on-without-rowid.sql

@@ -0,0 +1,4 @@
+create table weirdrowid (
+    rowid integer primary key,
+    a integer not null
+) strict, without rowid;

pkg/schema/lint/test_schemas/success.sql

@@ -5,14 +5,22 @@ create table stuff (
 ) strict;
 create index index_stuff_amount on stuff (amount);
 
+create virtual table stuff_fts using fts5(
+    data,
+    content='stuff',
+    content_rowid='rowid',
+    tokenize='trigram'
+);
+
+
 create table stuff2 (
     weird_pk integer primary key,
     label text not null unique,
     stuff_id integer references stuff(rowid),
     alternative_stuff_id integer references stuff(amount)
-) strict;
+) strict, without rowid;
 
 create table stuff3 (
-    weird_pk3 integer primary key,
+    rowid integer primary key,
     stuff2_id integer not null references stuff2(weird_pk)
 ) strict;

pkg/schema/parse.go

@@ -10,6 +10,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	_ "github.com/mattn/go-sqlite3"
 
+	. "git.offline-twitter.com/offline-labs/gas-stack/pkg/flowutils"
 	"git.offline-twitter.com/offline-labs/gas-stack/pkg/textutils"
 )
 
@@ -37,33 +38,25 @@ func SchemaFromDB(db *sqlx.DB) Schema {
 	ret := Schema{Tables: map[string]Table{}, Indexes: map[string]Index{}}
 
 	var tables []Table
-	err := db.Select(&tables, `select name, is_strict, is_without_rowid from tables`)
+	PanicIf(db.Select(&tables, `select name, table_type, is_strict, is_without_rowid from tables`))
-	if err != nil {
-		panic(err)
-	}
 	for _, tbl := range tables {
-		tbl.TypeName = textutils.SnakeToCamel(inflection.Singular(tbl.TableName))
+		tbl.GoTypeName = TypenameFromTablename(tbl.TableName)
-		tbl.TypeIDName = tbl.TypeName + "ID"
+		tbl.TypeIDName = tbl.GoTypeName + "ID"
 		tbl.VarName = strings.ToLower(string(tbl.TableName[0]))
 
-		err = db.Select(&tbl.Columns, `select * from columns where table_name = ?`, tbl.TableName)
+		PanicIf(db.Select(&tbl.Columns, `select * from columns where table_name = ?`, tbl.TableName))
-		if err != nil {
-			panic(err)
-		}
 		ret.Tables[tbl.TableName] = tbl
 	}
 
 	var indexes []Index
-	err = db.Select(&indexes, `select index_name, table_name, is_unique from indexes`)
+	PanicIf(db.Select(&indexes, `select index_name, table_name, is_unique from indexes`))
-	if err != nil {
-		panic(err)
-	}
 	for _, idx := range indexes {
-		err = db.Select(&idx.Columns, `select column_name from index_columns where index_name = ? order by rank`, idx.Name)
+		PanicIf(db.Select(&idx.Columns, `select column_name from index_columns where index_name = ? order by rank`, idx.Name))
-		if err != nil {
-			panic(err)
-		}
 		ret.Indexes[idx.Name] = idx
 	}
 	return ret
 }
+
+func TypenameFromTablename(tablename string) string {
+	return textutils.SnakeToCamel(inflection.Singular(tablename))
+}

pkg/schema/parse_test.go

@@ -25,7 +25,7 @@ func TestParseSchema(t *testing.T) {
 
 	foods := schema.Tables["foods"]
 	assert.Equal(foods.TableName, "foods")
-	assert.Equal(foods.TypeName, "Food")
+	assert.Equal(foods.GoTypeName, "Food")
 	assert.Equal(foods.TypeIDName, "FoodID")
 	assert.Equal(foods.IsStrict, true)
 	assert.Len(foods.Columns, 20)

pkg/schema/table.go

@@ -23,14 +23,21 @@ func (c Column) IsNullableForeignKey() bool {
 // Table is a single SQLite table.
 type Table struct {
 	TableName string `db:"name"`
+
+	// One of "table", "view", "shadow", or "virtual"
+	TableType      string `db:"table_type"`
 	IsStrict       bool   `db:"is_strict"`
 	IsWithoutRowid bool   `db:"is_without_rowid"`
 
 	Columns []Column
 
 	TypeIDName string
+
+	// Default variable name for variables of this type to use when generating Go code
 	VarName string
-	TypeName   string
+
+	// Name of corresponding model type to be generated
+	GoTypeName string
 }
 
 type Index struct {
@@ -39,6 +46,7 @@ type Index struct {
 	Columns   []string
 	IsUnique  bool `db:"is_unique"`
 	// TODO: `where ...` for partial indexes
+	// TODO: identify columns that are expressions
 }
 
 type Schema struct {

pkg/schema/views.sql

@@ -1,7 +1,7 @@
 create temporary view tables as
     select l.schema,
            l.name,
-           l.type,
+           l.type as table_type,
            l.wr as is_without_rowid,
            l.strict as is_strict
       from sqlite_schema s
@@ -33,7 +33,8 @@ create temporary view indexes as
 
 create temporary view index_columns as
     select indexes.index_name,
-           idx_cols.name as column_name,
+           idx_cols.cid as column_idx,
+           ifnull(idx_cols.name, '') as column_name,
            idx_cols.seqno as rank
       from indexes
       join pragma_index_info(indexes.index_name) as idx_cols;

sample_data/test_schemas/food.sql

@@ -43,6 +43,7 @@ create table foods (rowid integer primary key,
     cook_ratio real not null default 1
 ) strict;
 create index foods_protein on foods(protein);
+create index foods_fake_index on foods(protein + 1);
 
 
 create table units (rowid integer primary key,