codegen: implement auto-timestamps for created_at and updated_at

.gitignore

@@ -1 +1,5 @@
 sample_data/data
+
+# Legacy versions
+.cmd-old/
+pkg/.testapp

doc/TODO.txt

@@ -15,3 +15,17 @@ TODO: generator-foreign-keys
 TODO: migration-structs
 - Right now, migrations are strings.  Could be a struct with "name", "up" and "down" fields
 - Adding a "down" operation enables handling newer DB versions with "down instead of error-out" for development (perhaps a flag)
+
+IDEA: migrations-table
+- Store migrations in a table.  This makes the schema more self-documenting.
+- Possible schema: name, sql_up, sql_down, hash (computed from those fields plus previous migration hash)
+    - or just rowid instead of hash?  Migration sequence should be immutable after publishing, so there should never be "conflicts"
+
+TODO: language-server-for-TODO.txt
+
+TODO: auto-migration-checker
+- Use `pkg/schema` to test whether a base schema plus a migration equals a new schema
+
+TODO: codegen `without rowid` tables properly
+
+TODO: generated test file inclues global test DB setup, which is wrong

go.mod

@@ -3,6 +3,7 @@ module git.offline-twitter.com/offline-labs/gas-stack
 go 1.22.5
 
 require (
+	github.com/go-test/deep v1.1.1
 	github.com/jinzhu/inflection v1.0.0
 	github.com/jmoiron/sqlx v1.4.0
 	github.com/mattn/go-sqlite3 v1.14.24

go.sum

@@ -5,6 +5,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/go-sql-driver/mysql v1.8.1 h1:LedoTUt/eveggdHS9qUFC1EFSa8bU2+1pZjSRpvNJ1Y=
 github.com/go-sql-driver/mysql v1.8.1/go.mod h1:wEBSXgmK//2ZFJyE+qWnIsVGmvmEKlqwuVSjsCm7DZg=
+github.com/go-test/deep v1.1.1 h1:0r/53hagsehfO4bzD2Pgr/+RgHqhmf+k1Bpse2cTu1U=
+github.com/go-test/deep v1.1.1/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=

ops/devcontainer/Dockerfile

@@ -10,6 +10,8 @@ RUN apk add less
 RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b /usr/local/bin v2.0.2
 RUN GOBIN=/usr/local/bin go install git.offline-twitter.com/offline-labs/gocheckout@v0.0.2
 
+COPY etc/group /etc/group
+
 # Create a user in the container with the same UID as on the host machine, to avoid ownership conflicts.
 # The user gets sudo of course.
 #

ops/devcontainer/etc/group

@@ -0,0 +1,3 @@
+root:x:0:
+nogroup:x:65533:
+nobody:x:65534:

ops/gas_init_test.sh

@@ -9,7 +9,8 @@ set -e
 set -x
 
 PS4='+(${BASH_SOURCE}:${LINENO}): '
-cd "$(dirname "${BASH_SOURCE[0]}")/.."
+proj_root=$(readlink -f "$(dirname "${BASH_SOURCE[0]}")/..")
+cd "$proj_root"
 
 # Compile `gas`
 gas="/tmp/gas"
@@ -29,11 +30,19 @@ EOF
 
 cd $test_project
 
+# Add "replace" directive"
+echo "replace git.offline-twitter.com/offline-labs/gas-stack => $proj_root" >> go.mod
+
 # Create a new table in the schema
 cat >> pkg/db/schema.sql <<EOF
+create table item_flavor (rowid integer primary key, name text not null) strict;
+
 create table items (
     rowid integer primary key,
-    description text not null default ''
+    description text not null default '',
+    flavor integer references item_flavor(rowid),
+    created_at integer not null,
+    updated_at integer not null
 ) strict;
 EOF
 

pkg/codegen/modelgenerate/generate_model.go

@@ -21,6 +21,14 @@ func GenerateIDType(table schema.Table) *ast.GenDecl {
 	}
 }
 
+func fkFieldName(col schema.Column) string {
+	if col.IsNonCodeTableForeignKey() {
+		return textutils.SnakeToCamel(strings.TrimSuffix(col.Name, "_id")) + "ID"
+	} else {
+		return textutils.SnakeToCamel(col.Name)
+	}
+}
+
 // GenerateModelAST produces an AST for a struct type corresponding to the model.
 // TODO: generate the right field types here based on column types.
 func GenerateModelAST(table schema.Table) *ast.GenDecl {
@@ -37,9 +45,9 @@ func GenerateModelAST(table schema.Table) *ast.GenDecl {
 				Tag:   &ast.BasicLit{Kind: token.STRING, Value: "`db:\"rowid\" json:\"id\"`"},
 			})
 		default:
-			if col.IsForeignKey && strings.HasSuffix(col.Name, "_id") {
+			if col.IsNonCodeTableForeignKey() {
 				fields = append(fields, &ast.Field{
-					Names: []*ast.Ident{ast.NewIdent(textutils.SnakeToCamel(strings.TrimSuffix(col.Name, "_id")) + "ID")},
+					Names: []*ast.Ident{ast.NewIdent(fkFieldName(col))},
 					Type:  ast.NewIdent(schema.TypenameFromTablename(col.ForeignKeyTargetTable) + "ID"),
 					Tag:   &ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("`db:\"%s\" json:\"%s\"`", col.Name, col.Name)},
 				})
@@ -52,7 +60,7 @@ func GenerateModelAST(table schema.Table) *ast.GenDecl {
 					} else if strings.HasSuffix(col.Name, "_at") {
 						typeName = "Timestamp"
 					} else {
-						typeName = "int64"
+						typeName = "int"
 					}
 				case "text":
 					typeName = "string"
@@ -89,6 +97,9 @@ func GenerateSaveItemFunc(tbl schema.Table) *ast.FuncDecl {
 	insertVals := make([]string, 0, len(tbl.Columns))
 	updatePairs := make([]string, 0, len(tbl.Columns))
 
+	hasCreatedAt, hasUpdatedAt := tbl.HasAutoTimestamps()
+
+	// Assemble data for building SQL "insert" and "update" strings
 	for _, col := range tbl.Columns {
 		if col.Name == "rowid" {
 			continue
@@ -99,36 +110,248 @@ func GenerateSaveItemFunc(tbl schema.Table) *ast.FuncDecl {
 			val = fmt.Sprintf("nullif(%s, 0)", val)
 		}
 		insertVals = append(insertVals, val)
+		// created_at should not be updated after creation
+		if col.Name == "created_at" && hasCreatedAt {
+			continue
+		}
 		updatePairs = append(updatePairs, col.Name+"="+val)
 	}
-
-	insertStmt := fmt.Sprintf("\n\t\t    insert into %s (%s)\n\t\t          values (%s)\n\t\t", tbl.TableName, strings.Join(insertCols, ", "), strings.Join(insertVals, ", "))
+	insertStmt := fmt.Sprintf("\n\t\t    insert into %s (%s)\n\t\t         values (%s)\n\t\t", tbl.TableName, strings.Join(insertCols, ", "), strings.Join(insertVals, ", "))
 	updateStmt := fmt.Sprintf("\n\t\t    update %s\n\t\t       set %s\n\t\t     where rowid = :rowid\n\t\t", tbl.TableName, strings.Join(updatePairs, ",\n\t\t           "))
 
+	hasFks := false
+	checkForeignKeyFailuresAssignment := &ast.AssignStmt{
+		Lhs: []ast.Expr{ast.NewIdent("checkForeignKeyFailures")},
+		Tok: token.DEFINE,
+		Rhs: []ast.Expr{
+			&ast.FuncLit{
+				Type: &ast.FuncType{
+					Params: &ast.FieldList{
+						List: []*ast.Field{{
+							Names: []*ast.Ident{ast.NewIdent("err")},
+							Type:  ast.NewIdent("error"),
+						}},
+					},
+					Results: &ast.FieldList{
+						List: []*ast.Field{{Type: ast.NewIdent("error")}},
+					},
+				},
+				Body: &ast.BlockStmt{
+					List: func() []ast.Stmt {
+						ret := []ast.Stmt{}
+						// if !isSqliteFkError(err) { return nil }
+						ret = append(ret, &ast.IfStmt{
+							Cond: &ast.UnaryExpr{Op: token.NOT, X: &ast.CallExpr{Fun: ast.NewIdent("IsSqliteFkError"), Args: []ast.Expr{ast.NewIdent("err")}}},
+							Body: &ast.BlockStmt{List: []ast.Stmt{&ast.ReturnStmt{Results: []ast.Expr{ast.NewIdent("nil")}}}},
+						})
+
+						for _, col := range tbl.Columns {
+							if !col.IsForeignKey { // Check both "real" foreign keys and code table values
+								continue
+							}
+							hasFks = true
+
+							structFieldName := fkFieldName(col)
+							structField := &ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent(structFieldName)}
+
+							if col.IsNonCodeTableForeignKey() {
+								// Real foreign key; look up referent by ID to see if it exists
+								ret = append(ret, &ast.IfStmt{
+									Init: &ast.AssignStmt{
+										Lhs: []ast.Expr{ast.NewIdent("_"), ast.NewIdent("err")},
+										Tok: token.DEFINE,
+										Rhs: []ast.Expr{
+											&ast.CallExpr{
+												Fun: &ast.SelectorExpr{X: ast.NewIdent("db"), Sel: ast.NewIdent(getByIDFuncName(col.ForeignKeyTargetTable))},
+												Args: []ast.Expr{
+													structField,
+												},
+											},
+										},
+									},
+									Cond: &ast.CallExpr{
+										Fun: &ast.SelectorExpr{X: ast.NewIdent("errors"), Sel: ast.NewIdent("Is")},
+										Args: []ast.Expr{
+											ast.NewIdent("err"),
+											ast.NewIdent("ErrNotInDB"),
+										},
+									},
+									Body: &ast.BlockStmt{
+										List: []ast.Stmt{
+											&ast.ReturnStmt{
+												Results: []ast.Expr{
+													&ast.CallExpr{
+														Fun: ast.NewIdent("NewForeignKeyError"),
+														Args: []ast.Expr{
+															&ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("%q", structFieldName)},
+															&ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("%q", col.ForeignKeyTargetTable)},
+															structField,
+														},
+													},
+												},
+											},
+										},
+									},
+								})
+							} else {
+								// Code table value.  Query the table to see if it exists
+								ret = append(ret, &ast.IfStmt{
+									Init: &ast.AssignStmt{
+										Lhs: []ast.Expr{
+											ast.NewIdent("err"),
+										},
+										Tok: token.ASSIGN,
+										Rhs: []ast.Expr{
+											&ast.CallExpr{
+												Fun: &ast.SelectorExpr{X: &ast.SelectorExpr{X: ast.NewIdent("db"), Sel: ast.NewIdent("DB")}, Sel: ast.NewIdent("Get")},
+												Args: []ast.Expr{
+													&ast.CallExpr{Fun: ast.NewIdent("new"), Args: []ast.Expr{ast.NewIdent("int")}},
+													&ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("`select 1 from %s where rowid = ?`", col.ForeignKeyTargetTable)},
+													structField,
+												},
+											},
+										},
+									},
+									Cond: &ast.CallExpr{
+										Fun:  &ast.SelectorExpr{X: ast.NewIdent("errors"), Sel: ast.NewIdent("Is")},
+										Args: []ast.Expr{ast.NewIdent("err"), &ast.SelectorExpr{X: ast.NewIdent("sql"), Sel: ast.NewIdent("ErrNoRows")}},
+									},
+									Body: &ast.BlockStmt{
+										List: []ast.Stmt{
+											&ast.ReturnStmt{
+												Results: []ast.Expr{
+													&ast.CallExpr{
+														Fun: ast.NewIdent("NewForeignKeyError"),
+														Args: []ast.Expr{
+															&ast.BasicLit{Kind: token.STRING, Value: fmt.Sprintf("%q", structFieldName)},
+															ast.NewIdent(fmt.Sprintf("%q", col.ForeignKeyTargetTable)),
+															structField,
+														},
+													},
+												},
+											},
+										},
+									},
+								})
+							}
+						}
+						// final return nil
+						ret = append(ret, &ast.ReturnStmt{Results: []ast.Expr{ast.NewIdent("nil")}})
+						return ret
+					}(),
+				},
+			},
+		},
+	}
+
 	funcBody := &ast.BlockStmt{
-		List: []ast.Stmt{
-			&ast.IfStmt{
+		List: func() []ast.Stmt {
+			ret := []ast.Stmt{}
+			if hasFks {
+				ret = append(ret, checkForeignKeyFailuresAssignment)
+			}
+			if hasUpdatedAt {
+				// Auto-timestamps: updated_at
+				ret = append(ret, &ast.AssignStmt{
+					Lhs: []ast.Expr{&ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("UpdatedAt")}},
+					Tok: token.ASSIGN,
+					Rhs: []ast.Expr{&ast.CallExpr{Fun: ast.NewIdent("TimestampNow"), Args: []ast.Expr{}}},
+				})
+			}
+			// if item.ID == 0 {...} else {...}
+			ret = append(ret, &ast.IfStmt{
 				Cond: &ast.BinaryExpr{
 					X:  &ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")},
 					Op: token.EQL,
 					Y:  &ast.BasicLit{Kind: token.INT, Value: "0"},
 				},
 				Body: &ast.BlockStmt{
-					List: []ast.Stmt{
-						&ast.AssignStmt{
-							Lhs: []ast.Expr{ast.NewIdent("result")},
-							Tok: token.DEFINE,
-							Rhs: []ast.Expr{&ast.CallExpr{
-								Fun: ast.NewIdent("Must"),
-								Args: []ast.Expr{&ast.CallExpr{
-									Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("NamedExec")},
-									Args: []ast.Expr{
-										&ast.BasicLit{Kind: token.STRING, Value: "`" + insertStmt + "`"},
-										ast.NewIdent(tbl.VarName),
+					// Do create
+					List: append(
+						func() []ast.Stmt {
+							ret := []ast.Stmt{}
+							if hasCreatedAt {
+								// Auto-timestamps: created_at
+								ret = append(ret, &ast.AssignStmt{
+									Lhs: []ast.Expr{&ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("CreatedAt")}},
+									Tok: token.ASSIGN,
+									Rhs: []ast.Expr{&ast.CallExpr{Fun: ast.NewIdent("TimestampNow"), Args: []ast.Expr{}}},
+								})
+							}
+							namedExecStmt := &ast.CallExpr{
+								Fun: &ast.SelectorExpr{X: ast.NewIdent("db.DB"), Sel: ast.NewIdent("NamedExec")},
+								Args: []ast.Expr{
+									&ast.BasicLit{Kind: token.STRING, Value: "`" + insertStmt + "`"},
+									ast.NewIdent(tbl.VarName),
+								},
+							}
+							if !hasFks {
+								// No foreign key checking needed; just use `Must` for brevity
+								return append(ret, &ast.AssignStmt{
+									Lhs: []ast.Expr{ast.NewIdent("result")},
+									Tok: token.DEFINE,
+									Rhs: []ast.Expr{&ast.CallExpr{
+										Fun:  ast.NewIdent("Must"),
+										Args: []ast.Expr{namedExecStmt},
+									}},
+								})
+							}
+
+							return append(ret,
+								// result, err := db.DB.NamedExec(`...`, u)
+								&ast.AssignStmt{
+									Lhs: []ast.Expr{
+										ast.NewIdent("result"),
+										ast.NewIdent("err"),
 									},
-								}},
-							}},
-						},
+									Tok: token.DEFINE,
+									Rhs: []ast.Expr{namedExecStmt},
+								},
+
+								// if fkErr := checkForeignKeyFailures(err); fkErr != nil { return fkErr } else if err != nil { panic(err) }
+								&ast.IfStmt{
+									Init: &ast.AssignStmt{
+										Lhs: []ast.Expr{ast.NewIdent("fkErr")},
+										Tok: token.DEFINE,
+										Rhs: []ast.Expr{
+											&ast.CallExpr{
+												Fun:  ast.NewIdent("checkForeignKeyFailures"),
+												Args: []ast.Expr{ast.NewIdent("err")},
+											},
+										},
+									},
+									Cond: &ast.BinaryExpr{
+										X:  ast.NewIdent("fkErr"),
+										Op: token.NEQ,
+										Y:  ast.NewIdent("nil"),
+									},
+									Body: &ast.BlockStmt{
+										List: []ast.Stmt{
+											&ast.ReturnStmt{
+												Results: []ast.Expr{ast.NewIdent("fkErr")},
+											},
+										},
+									},
+									Else: &ast.IfStmt{
+										Cond: &ast.BinaryExpr{
+											X:  ast.NewIdent("err"),
+											Op: token.NEQ,
+											Y:  ast.NewIdent("nil"),
+										},
+										Body: &ast.BlockStmt{
+											List: []ast.Stmt{
+												&ast.ExprStmt{
+													X: &ast.CallExpr{
+														Fun:  ast.NewIdent("panic"),
+														Args: []ast.Expr{ast.NewIdent("err")},
+													},
+												},
+											},
+										},
+									},
+								},
+							)
+						}(),
 						&ast.AssignStmt{
 							Lhs: []ast.Expr{&ast.SelectorExpr{X: ast.NewIdent(tbl.VarName), Sel: ast.NewIdent("ID")}},
 							Tok: token.ASSIGN,
@@ -143,9 +366,10 @@ func GenerateSaveItemFunc(tbl schema.Table) *ast.FuncDecl {
 								}},
 							}},
 						},
-					},
+					),
 				},
 				Else: &ast.BlockStmt{
+					// Do update
 					List: []ast.Stmt{
 						&ast.AssignStmt{
 							Lhs: []ast.Expr{ast.NewIdent("result")},
@@ -175,27 +399,39 @@ func GenerateSaveItemFunc(tbl schema.Table) *ast.FuncDecl {
 						},
 					},
 				},
-			},
-		},
+			})
+			if hasFks {
+				// If there's foreign key checking, it needs to return an error (or nil)
+				ret = append(ret, &ast.ReturnStmt{Results: []ast.Expr{ast.NewIdent("nil")}})
+			}
+			return ret
+		}(),
 	}
 
 	funcDecl := &ast.FuncDecl{
 		Recv: &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("db")}, Type: ast.NewIdent("DB")}}},
 		Name: ast.NewIdent("Save" + tbl.GoTypeName),
 		Type: &ast.FuncType{
-			Params:  &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent(tbl.VarName)}, Type: &ast.StarExpr{X: ast.NewIdent(tbl.GoTypeName)}}}},
-			Results: nil,
+			Params: &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent(tbl.VarName)}, Type: &ast.StarExpr{X: ast.NewIdent(tbl.GoTypeName)}}}},
+			Results: func() *ast.FieldList {
+				if hasFks {
+					return &ast.FieldList{List: []*ast.Field{{Type: ast.NewIdent("error")}}}
+				}
+				return nil
+			}(),
 		},
 		Body: funcBody,
 	}
 	return funcDecl
 }
 
+func getByIDFuncName(tblname string) string {
+	return "Get" + schema.TypenameFromTablename(tblname) + "ByID"
+}
+
 // GenerateGetItemByIDFunc produces an AST for the `GetXyzByID()` function.
 // E.g., a table with `table.TypeName = "foods"` will produce a "GetFoodByID()" function.
 func GenerateGetItemByIDFunc(tbl schema.Table) *ast.FuncDecl {
-	funcName := "Get" + tbl.GoTypeName + "ByID"
-
 	recv := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("db")}, Type: ast.NewIdent("DB")}}}
 	arg := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("id")}, Type: ast.NewIdent(tbl.TypeIDName)}}}
 	result := &ast.FieldList{List: []*ast.Field{{Names: []*ast.Ident{ast.NewIdent("ret")}, Type: ast.NewIdent(tbl.GoTypeName)}, {Names: []*ast.Ident{ast.NewIdent("err")}, Type: ast.NewIdent("error")}}}
@@ -228,7 +464,7 @@ func GenerateGetItemByIDFunc(tbl schema.Table) *ast.FuncDecl {
 
 	funcDecl := &ast.FuncDecl{
 		Recv: recv,
-		Name: ast.NewIdent(funcName),
+		Name: ast.NewIdent(getByIDFuncName(tbl.TableName)),
 		Type: &ast.FuncType{Params: arg, Results: result},
 		Body: funcBody,
 	}
@@ -258,12 +494,12 @@ func GenerateGetAllItemsFunc(tbl schema.Table) *ast.FuncDecl {
 					&ast.UnaryExpr{Op: token.AND, X: ast.NewIdent("ret")},
 					&ast.BinaryExpr{
 						X: &ast.BinaryExpr{
-							X:  &ast.BasicLit{Kind: token.STRING, Value: "`SELECT `"},
+							X:  &ast.BasicLit{Kind: token.STRING, Value: "`select `"},
 							Op: token.ADD,
 							Y:  SQLFieldsConstIdent(tbl),
 						},
 						Op: token.ADD,
-						Y:  &ast.BasicLit{Kind: token.STRING, Value: "` FROM " + tbl.TableName + "`"},
+						Y:  &ast.BasicLit{Kind: token.STRING, Value: "` from " + tbl.TableName + "`"},
 					},
 				},
 			},
@@ -351,7 +587,11 @@ func GenerateDeleteItemFunc(tbl schema.Table) *ast.FuncDecl {
 func GenerateSQLFieldsConst(tbl schema.Table) *ast.GenDecl {
 	columns := make([]string, 0, len(tbl.Columns))
 	for _, col := range tbl.Columns {
-		columns = append(columns, col.Name)
+		if col.IsNullableForeignKey() {
+			columns = append(columns, fmt.Sprintf("ifnull(%s, 0) %s", col.Name, col.Name))
+		} else {
+			columns = append(columns, col.Name)
+		}
 	}
 	// Join with comma and space
 	value := "`" + strings.Join(columns, ", ") + "`"

pkg/codegen/modelgenerate/generate_testfile.go

@@ -83,6 +83,30 @@ func GenerateModelTestAST(tbl schema.Table, gomodName string) *ast.File {
 		},
 	}
 
+	// func MakeItem() Item { return Item{} }
+	makeItemFunc := &ast.FuncDecl{
+		Name: ast.NewIdent("Make" + tbl.GoTypeName),
+		Type: &ast.FuncType{
+			Params: &ast.FieldList{},
+			Results: &ast.FieldList{
+				List: []*ast.Field{
+					{Type: ast.NewIdent(tbl.GoTypeName)},
+				},
+			},
+		},
+		Body: &ast.BlockStmt{
+			List: []ast.Stmt{
+				&ast.ReturnStmt{
+					Results: []ast.Expr{
+						&ast.CompositeLit{
+							Type: ast.NewIdent(tbl.GoTypeName),
+						},
+					},
+				},
+			},
+		},
+	}
+
 	testObj := ast.NewIdent("item")
 	testObj2 := ast.NewIdent("item2")
 	fieldName := ast.NewIdent("Description")
@@ -209,7 +233,7 @@ func GenerateModelTestAST(tbl schema.Table, gomodName string) *ast.File {
 					Args: []ast.Expr{
 						ast.NewIdent("t"),
 						ast.NewIdent("err"),
-						&ast.SelectorExpr{X: ast.NewIdent("db"), Sel: ast.NewIdent("ErrNotInDB")},
+						ast.NewIdent("ErrNotInDB"),
 					},
 				}},
 			},
@@ -237,9 +261,126 @@ func GenerateModelTestAST(tbl schema.Table, gomodName string) *ast.File {
 		},
 	}
 
+	shouldIncludeTestFkCheck := false
+	testFkChecking := &ast.FuncDecl{
+		Name: ast.NewIdent("Test" + tbl.GoTypeName + "FkChecking"),
+		Type: &ast.FuncType{
+			Params: &ast.FieldList{
+				List: []*ast.Field{
+					{
+						Names: []*ast.Ident{ast.NewIdent("t")},
+						Type: &ast.StarExpr{
+							X: &ast.SelectorExpr{
+								X:   ast.NewIdent("testing"),
+								Sel: ast.NewIdent("T"),
+							},
+						},
+					},
+				},
+			},
+		},
+		Body: &ast.BlockStmt{
+			List: func() []ast.Stmt {
+				// post := MakePost()
+				stmts := []ast.Stmt{
+					&ast.AssignStmt{
+						Lhs: []ast.Expr{ast.NewIdent(tbl.VarName)},
+						Tok: token.DEFINE,
+						Rhs: []ast.Expr{
+							&ast.CallExpr{
+								Fun: ast.NewIdent("Make" + tbl.GoTypeName),
+							},
+						},
+					},
+				}
+				for _, col := range tbl.Columns {
+					if col.IsForeignKey {
+						shouldIncludeTestFkCheck = true
+						stmts = append(stmts, []ast.Stmt{
+
+							// post.QuotedPostID = 94354538969386985
+							&ast.AssignStmt{
+								Lhs: []ast.Expr{
+									&ast.SelectorExpr{
+										X:   ast.NewIdent(tbl.VarName),
+										Sel: ast.NewIdent(fkFieldName(col)),
+									},
+								},
+								Tok: token.ASSIGN,
+								Rhs: []ast.Expr{
+									&ast.BasicLit{
+										Kind:  token.INT,
+										Value: "94354538969386985",
+									},
+								},
+							},
+
+							// err := db.SavePost(&post)
+							&ast.AssignStmt{
+								Lhs: []ast.Expr{ast.NewIdent("err")},
+								Tok: token.DEFINE,
+								Rhs: []ast.Expr{
+									&ast.CallExpr{
+										Fun: &ast.SelectorExpr{
+											X:   ast.NewIdent("TestDB"),
+											Sel: ast.NewIdent("Save" + tbl.GoTypeName),
+										},
+										Args: []ast.Expr{
+											&ast.UnaryExpr{
+												Op: token.AND,
+												X:  ast.NewIdent(tbl.VarName),
+											},
+										},
+									},
+								},
+							},
+
+							// assertForeignKeyError(t, err, "QuotedPostID", post.QuotedPostID)
+							&ast.ExprStmt{
+								X: &ast.CallExpr{
+									Fun: ast.NewIdent("AssertForeignKeyError"),
+									Args: []ast.Expr{
+										ast.NewIdent("t"),
+										ast.NewIdent("err"),
+										&ast.BasicLit{
+											Kind:  token.STRING,
+											Value: fmt.Sprintf("%q", fkFieldName(col)),
+										},
+										&ast.SelectorExpr{
+											X:   ast.NewIdent(tbl.VarName),
+											Sel: ast.NewIdent(fkFieldName(col)),
+										},
+									},
+								},
+							},
+						}...)
+					}
+				}
+				return stmts
+			}(),
+		},
+	}
+
+	testList := []ast.Decl{
+		// var TestDB *DB
+		testDBDecl,
+		makeItemFunc,
+
+		// func init() { TestDB = MakeDB("tmp") }
+		initFuncDecl,
+
+		// func MakeDB(dbName string) *DB { db := Must(Create(fmt.Sprintf("file:%s?mode=memory&cache=shared", dbName))); return db }
+		makeDBHelperDecl,
+
+		testCreateUpdateDelete,
+		testGetAll,
+	}
+	if shouldIncludeTestFkCheck {
+		testList = append(testList, testFkChecking)
+	}
 	return &ast.File{
 		Name: ast.NewIdent(testpackageName),
-		Decls: []ast.Decl{
+		Decls: append([]ast.Decl{
 			&ast.GenDecl{
 				Tok: token.IMPORT,
 				Specs: []ast.Spec{
@@ -247,7 +388,7 @@ func GenerateModelTestAST(tbl schema.Table, gomodName string) *ast.File {
 					&ast.ImportSpec{Path: &ast.BasicLit{Kind: token.STRING, Value: `"testing"`}},
 					&ast.ImportSpec{
 						Path: &ast.BasicLit{Kind: token.STRING, Value: `"git.offline-twitter.com/offline-labs/gas-stack/pkg/db"`},
-						Name: ast.NewIdent("db"),
+						Name: ast.NewIdent("."),
 					},
 					&ast.ImportSpec{
 						Path: &ast.BasicLit{Kind: token.STRING, Value: `"git.offline-twitter.com/offline-labs/gas-stack/pkg/flowutils"`},
@@ -261,17 +402,6 @@ func GenerateModelTestAST(tbl schema.Table, gomodName string) *ast.File {
 					&ast.ImportSpec{Path: &ast.BasicLit{Kind: token.STRING, Value: `"github.com/stretchr/testify/require"`}},
 				},
 			},
-			// var TestDB *DB
-			testDBDecl,
-
-			// func init() { TestDB = MakeDB("tmp") }
-			initFuncDecl,
-
-			// func MakeDB(dbName string) *DB { db := Must(Create(fmt.Sprintf("file:%s?mode=memory&cache=shared", dbName))); return db }
-			makeDBHelperDecl,
-
-			testCreateUpdateDelete,
-			testGetAll,
-		},
+		}, testList...),
 	}
 }

pkg/db/errors_test_helper.go

@@ -0,0 +1,21 @@
+package db
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func AssertForeignKeyError[T ForeignKey](t *testing.T, err error, field string, val T) {
+	t.Helper()
+
+	var fkErr ForeignKeyError[T]
+	require.Error(t, err)
+	require.ErrorIs(t, err, ErrForeignKeyViolation)
+	// ErrorAs produces terrible error messages if it's a ForeignKeyError with a different type
+	// parameter (i.e., if it was a different field that failed).
+	require.ErrorAs(t, err, &fkErr, "expected error field: %q", field)
+	assert.Equal(t, field, fkErr.Field)
+	assert.Equal(t, val, fkErr.FkValue)
+}

pkg/schema/migration_verification_test.go

@@ -0,0 +1,167 @@
+package schema_test
+
+import (
+	"slices"
+	"testing"
+
+	"github.com/go-test/deep"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"git.offline-twitter.com/offline-labs/gas-stack/pkg/db"
+	"git.offline-twitter.com/offline-labs/gas-stack/pkg/flowutils"
+	"git.offline-twitter.com/offline-labs/gas-stack/pkg/schema"
+)
+
+var (
+	baseSchema = `
+		create table db_version (
+		    version integer primary key
+		) strict, without rowid;
+		insert into db_version values(0);
+
+		create table t1 (
+			rowid integer primary key,
+			data1 integer not null,
+			data2 text not null
+		);
+	`
+
+	fullSchema = `
+		create table db_version (
+		    version integer primary key
+		) strict, without rowid;
+		insert into db_version values(2);
+
+		create table t1 (
+			rowid integer primary key,
+			data1 integer not null,
+			data2 text not null,
+			data3 integer
+		);
+		create table t2 (
+			rowid integer primary key
+		);
+	`
+)
+
+func TestVerifyCorrectMigration(t *testing.T) {
+	db1 := schema.InitDB(fullSchema)
+	db1Schema := schema.SchemaFromDB(db1)
+
+	t.Run("migrate in 1 step", func(t *testing.T) {
+		migration := `
+			create table t2 (
+				rowid integer primary key
+			);
+			alter table t1 add column data3 integer;
+		`
+		db2Config := db.Init(&baseSchema, &[]string{migration})
+		db2 := flowutils.Must(db2Config.Create(":memory:"))
+		require.NoError(t, db2Config.CheckAndUpdateVersion(db2))
+		db2Schema := schema.SchemaFromDB(db2)
+
+		if diff := deep.Equal(db1Schema, db2Schema); diff != nil {
+			t.Error(diff)
+		}
+	})
+
+	t.Run("migrate in 2 steps", func(t *testing.T) {
+		migration1 := `
+			create table t2 (
+				rowid integer primary key
+			);
+		`
+		migration2 := `
+			alter table t1 add column data3 integer;
+		`
+
+		db2Config := db.Init(&baseSchema, &[]string{migration1, migration2})
+		db2 := flowutils.Must(db2Config.Create(":memory:"))
+		require.NoError(t, db2Config.CheckAndUpdateVersion(db2))
+		db2Schema := schema.SchemaFromDB(db2)
+
+		if diff := deep.Equal(db1Schema, db2Schema); diff != nil {
+			t.Error(diff)
+		}
+	})
+}
+
+func TestIncorrectMigrations(t *testing.T) {
+	db1 := schema.InitDB(fullSchema)
+	db1Schema := schema.SchemaFromDB(db1)
+
+	t.Run("missing migration", func(t *testing.T) {
+		db2Config := db.Init(&baseSchema, &[]string{})
+		db2 := flowutils.Must(db2Config.Create(":memory:"))
+		require.NoError(t, db2Config.CheckAndUpdateVersion(db2))
+		db2Schema := schema.SchemaFromDB(db2)
+
+		// Missing a table
+		assert.Len(t, db1Schema.Tables, len(db2Schema.Tables)+1)
+		assert.Contains(t, db1Schema.Tables, "t2")
+		assert.NotContains(t, db2Schema.Tables, "t2")
+
+		// Missing the new column
+		assert.Len(t, db1Schema.Tables["t1"].Columns, len(db2Schema.Tables["t1"].Columns)+1)
+		assert.True(t, slices.ContainsFunc(db1Schema.Tables["t1"].Columns, func(c schema.Column) bool { return c.Name == "data3" }), "t2")
+		assert.False(t, slices.ContainsFunc(db2Schema.Tables["t1"].Columns, func(c schema.Column) bool { return c.Name == "data3" }), "t2")
+	})
+
+	t.Run("incomplete migration", func(t *testing.T) {
+		db2Config := db.Init(&baseSchema, &[]string{`
+				create table t2 (
+					rowid integer primary key
+				);
+			`})
+		db2 := flowutils.Must(db2Config.Create(":memory:"))
+		require.NoError(t, db2Config.CheckAndUpdateVersion(db2))
+		db2Schema := schema.SchemaFromDB(db2)
+
+		// Has the new table
+		assert.Len(t, db1Schema.Tables, len(db2Schema.Tables))
+		assert.Contains(t, db1Schema.Tables, "t2")
+		assert.Contains(t, db2Schema.Tables, "t2")
+
+		// Still missing the new column
+		assert.Len(t, db1Schema.Tables["t1"].Columns, len(db2Schema.Tables["t1"].Columns)+1)
+		assert.True(t, slices.ContainsFunc(db1Schema.Tables["t1"].Columns, func(c schema.Column) bool { return c.Name == "data3" }), "t2")
+		assert.False(t, slices.ContainsFunc(db2Schema.Tables["t1"].Columns, func(c schema.Column) bool { return c.Name == "data3" }), "t2")
+	})
+
+	t.Run("incorrect migration (wrong data type)", func(t *testing.T) {
+		db2Config := db.Init(&baseSchema, &[]string{`
+			create table t2 (
+				rowid integer primary key
+			);
+			alter table t1 add column data3 text;
+		`})
+		db2 := flowutils.Must(db2Config.Create(":memory:"))
+		require.NoError(t, db2Config.CheckAndUpdateVersion(db2))
+		db2Schema := schema.SchemaFromDB(db2)
+
+		// Has the new table
+		assert.Len(t, db1Schema.Tables, len(db2Schema.Tables))
+		assert.Contains(t, db1Schema.Tables, "t2")
+		assert.Contains(t, db2Schema.Tables, "t2")
+
+		// Has the right column, but it's the wrong type
+		assert.Len(t, db1Schema.Tables["t1"].Columns, len(db2Schema.Tables["t1"].Columns))
+		col1 := db1Schema.Tables["t1"].Columns[slices.IndexFunc(
+			db1Schema.Tables["t1"].Columns,
+			func(c schema.Column) bool { return c.Name == "data3" },
+		)]
+		col2 := db2Schema.Tables["t1"].Columns[slices.IndexFunc(
+			db2Schema.Tables["t1"].Columns,
+			func(c schema.Column) bool { return c.Name == "data3" },
+		)]
+
+		assert.NotEqual(t, col1, col2)
+		assert.Equal(t, col1.Type, "integer") // Full schema has an integer column
+		assert.Equal(t, col2.Type, "text")    // Migration incorrectly uses a text column
+
+		// Other than that they are equal
+		col2.Type = "integer"
+		assert.Equal(t, col1, col2)
+	})
+}

pkg/schema/parse.go

@@ -21,7 +21,6 @@ var create_views string
 func InitDB(sql_schema string) *sqlx.DB {
 	db := sqlx.MustOpen("sqlite3", ":memory:")
 	db.MustExec(sql_schema)
-	db.MustExec(create_views)
 	return db
 }
 
@@ -36,6 +35,7 @@ func SchemaFromSQLFile(filepath string) (Schema, error) {
 // SchemaFromDB takes a DB connection, checks its schema metadata tables, and returns a Schema.
 func SchemaFromDB(db *sqlx.DB) Schema {
 	ret := Schema{Tables: map[string]Table{}, Indexes: map[string]Index{}}
+	db.MustExec(create_views)
 
 	var tables []Table
 	PanicIf(db.Select(&tables, `select name, table_type, is_strict, is_without_rowid from tables`))

pkg/schema/table.go

@@ -1,5 +1,7 @@
 package schema
 
+import "strings"
+
 // Column represents a single column in a table.
 type Column struct {
 	TableName              string `db:"table_name"`
@@ -20,6 +22,10 @@ func (c Column) IsNullableForeignKey() bool {
 	return !c.IsNotNull && !c.IsPrimaryKey && c.IsForeignKey
 }
 
+func (c Column) IsNonCodeTableForeignKey() bool {
+	return c.IsForeignKey && strings.HasSuffix(c.Name, "_id")
+}
+
 // Table is a single SQLite table.
 type Table struct {
 	TableName string `db:"name"`
@@ -40,6 +46,18 @@ type Table struct {
 	GoTypeName string
 }
 
+func (t Table) HasAutoTimestamps() (hasCreatedAt bool, hasUpdatedAt bool) {
+	for _, c := range t.Columns {
+		if c.Name == "created_at" && c.Type == "integer" {
+			hasCreatedAt = true
+		}
+		if c.Name == "updated_at" && c.Type == "integer" {
+			hasUpdatedAt = true
+		}
+	}
+	return
+}
+
 type Index struct {
 	Name      string `db:"index_name"`
 	TableName string `db:"table_name"`