From f1f5510aa0010b040b861c07250fabc61d293185 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh Date: Sun, 11 Oct 2026 11:39:49 +0530 Subject: [PATCH 1/2] perf(schema): batch loader queries per schema and drop nested cursors (#565) Load listed tables and then called LoadTable per table inside the open table-list cursor, issuing three more queries each: 3N+1 round trips against information_schema, with two connections held at once. SQLite's loadIndexes also nested PRAGMA index_info inside the index_list cursor. With db.SetMaxOpenConns(1) Load deadlocked, and with a pooled SQLite ":memory:" database the nested queries opened a second, empty database, so every table came back with no columns, indexes or foreign keys. All three loaders (postgres, mysql, sqlite) now run one schema-wide query per kind (table list, columns, indexes, foreign keys) and group rows by table in Go. Every cursor is drained and closed before the next statement runs, so one connection is enough. LoadTable reuses the same three query functions with a table filter, so Load and LoadTable share SQL and return identical tables. Load: 3N+1 -> 4 statements; LoadTable: 3+I -> 3. SQLite now uses the pragma_table_info / pragma_index_list / pragma_index_info / pragma_foreign_key_list table-valued functions with the table name as a bound parameter instead of %q-formatted PRAGMA text. Names with embedded double quotes now load correctly. Postgres pk/uq subqueries join on (table_name, column_name), so IsPrimary/IsUnique stay per table. MySQL LoadTable tolerates case-insensitive table-name matching (lower_case_table_names). Output matches the old code for existing fixtures (verified by golden diffs on SQLite and on a local PostgreSQL 17). The one difference: tables with no foreign keys (and MySQL tables with no indexes) now get nil slices instead of empty non-nil ones. Exported signatures are unchanged. benchstat, BenchmarkSQLiteLoader_Load (file DB, count=6): tables=10: 471.8us -> 282.4us (-40%), 75.7KiB -> 30.1KiB (-60%), 1805 -> 1106 allocs/op (-39%) tables=100: 3.59ms -> 1.92ms (-47%), 679KiB -> 250KiB (-63%), 16749 -> 9947 allocs/op (-41%) New regression tests cover in-memory full detail, MaxOpenConns=1 for Load and LoadTable, Load/LoadTable consistency, multi-column indexes, composite FKs, quoted table names and missing tables. The Postgres/MySQL equivalents run in CI under testcontainers. Fixes #565 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01UZSk1292jPc8ipDboNxeve --- pkg/schema/mysql/loader.go | 235 ++++++++++++------ pkg/schema/mysql/loader_test.go | 103 ++++++++ pkg/schema/postgres/loader.go | 205 ++++++++++------ pkg/schema/postgres/loader_test.go | 119 +++++++++ pkg/schema/sqlite/loader.go | 246 ++++++++++++------- pkg/schema/sqlite/loader_test.go | 381 +++++++++++++++++++++++++++++ 6 files changed, 1047 insertions(+), 242 deletions(-) diff --git a/pkg/schema/mysql/loader.go b/pkg/schema/mysql/loader.go index 190d8450..a761104d 100644 --- a/pkg/schema/mysql/loader.go +++ b/pkg/schema/mysql/loader.go @@ -31,55 +31,115 @@ type Loader struct{} func NewLoader() *Loader { return &Loader{} } // Load returns the full schema for all user tables in the given schemaName. +// +// Load issues four statements (tables, columns, indexes, foreign keys) for the +// whole schema and drains each cursor before running the next, so it works +// with a single-connection pool. func (l *Loader) Load(db *sql.DB, schemaName string) (*dbschema.DatabaseSchema, error) { - rows, err := db.Query(` - SELECT table_name - FROM information_schema.tables - WHERE table_schema = ? AND table_type = 'BASE TABLE' - ORDER BY table_name - `, schemaName) + names, err := l.listTables(db, schemaName) if err != nil { - return nil, fmt.Errorf("list tables: %w", err) + return nil, err + } + cols, err := l.loadColumns(db, schemaName, "") + if err != nil { + return nil, err + } + idxs, err := l.loadIndexes(db, schemaName, "") + if err != nil { + return nil, err + } + fks, err := l.loadForeignKeys(db, schemaName, "") + if err != nil { + return nil, err } - defer func() { _ = rows.Close() }() ds := &dbschema.DatabaseSchema{Name: schemaName} - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - return nil, err - } - tbl, err := l.LoadTable(db, schemaName, name) - if err != nil { - return nil, err - } - ds.Tables = append(ds.Tables, *tbl) + if len(names) > 0 { + ds.Tables = make([]dbschema.Table, 0, len(names)) + } + for _, n := range names { + ds.Tables = append(ds.Tables, dbschema.Table{ + Schema: schemaName, + Name: n, + Columns: cols[n], + Indexes: idxs[n], + ForeignKeys: fks[n], + }) } - return ds, rows.Err() + return ds, nil } // LoadTable returns schema for a single named table. func (l *Loader) LoadTable(db *sql.DB, schemaName, tableName string) (*dbschema.Table, error) { - tbl := &dbschema.Table{Schema: schemaName, Name: tableName} - var err error - tbl.Columns, err = l.loadColumns(db, schemaName, tableName) + cols, err := l.loadColumns(db, schemaName, tableName) if err != nil { return nil, err } - tbl.Indexes, err = l.loadIndexes(db, schemaName, tableName) + idxs, err := l.loadIndexes(db, schemaName, tableName) if err != nil { return nil, err } - tbl.ForeignKeys, err = l.loadForeignKeys(db, schemaName, tableName) + fks, err := l.loadForeignKeys(db, schemaName, tableName) if err != nil { return nil, err } - return tbl, nil + return &dbschema.Table{ + Schema: schemaName, + Name: tableName, + Columns: forTable(cols, tableName), + Indexes: forTable(idxs, tableName), + ForeignKeys: forTable(fks, tableName), + }, nil +} + +// forTable returns the entry for tableName from the result of a query filtered +// to that one table. MySQL may match table names case-insensitively (see +// lower_case_table_names), in which case the key is the stored spelling rather +// than tableName; since every row of a filtered query belongs to the same +// table, any single entry is the answer. +func forTable[T any](m map[string][]T, tableName string) []T { + if v, ok := m[tableName]; ok { + return v + } + for _, v := range m { + return v + } + return nil } -func (l *Loader) loadColumns(db *sql.DB, schemaName, tableName string) ([]dbschema.Column, error) { +// listTables returns the base tables in schemaName, ordered by name. +func (l *Loader) listTables(db *sql.DB, schemaName string) ([]string, error) { + rows, err := db.Query(` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = ? AND table_type = 'BASE TABLE' + ORDER BY table_name + `, schemaName) + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + defer func() { _ = rows.Close() }() + + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("list tables: scan: %w", err) + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + return names, nil +} + +// loadColumns returns columns grouped by table name, in ordinal order. +// An empty tableName loads every table in schemaName. +func (l *Loader) loadColumns(db *sql.DB, schemaName, tableName string) (map[string][]dbschema.Column, error) { rows, err := db.Query(` SELECT + c.table_name, c.column_name, c.ordinal_position, c.data_type, @@ -90,75 +150,87 @@ func (l *Loader) loadColumns(db *sql.DB, schemaName, tableName string) ([]dbsche c.numeric_scale, c.column_key FROM information_schema.columns c - WHERE c.table_schema = ? AND c.table_name = ? - ORDER BY c.ordinal_position - `, schemaName, tableName) + WHERE c.table_schema = ? AND (? = '' OR c.table_name = ?) + ORDER BY c.table_name, c.ordinal_position + `, schemaName, tableName, tableName) if err != nil { return nil, fmt.Errorf("load columns: %w", err) } defer func() { _ = rows.Close() }() - var cols []dbschema.Column + out := make(map[string][]dbschema.Column) for rows.Next() { + var tbl string var col dbschema.Column var isNullable, columnKey string err := rows.Scan( + &tbl, &col.Name, &col.OrdinalPos, &col.DataType, &isNullable, &col.DefaultValue, &col.MaxLength, &col.Precision, &col.Scale, &columnKey, ) if err != nil { - return nil, err + return nil, fmt.Errorf("load columns: scan: %w", err) } col.IsNullable = isNullable == "YES" col.IsPrimary = columnKey == "PRI" col.IsUnique = columnKey == "UNI" - cols = append(cols, col) + out[tbl] = append(out[tbl], col) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load columns: %w", err) } - return cols, rows.Err() + return out, nil } -func (l *Loader) loadIndexes(db *sql.DB, schemaName, tableName string) ([]dbschema.Index, error) { +// loadIndexes returns indexes grouped by table name, ordered by index name, +// with columns in key order. An empty tableName loads every table in schemaName. +func (l *Loader) loadIndexes(db *sql.DB, schemaName, tableName string) (map[string][]dbschema.Index, error) { rows, err := db.Query(` - SELECT index_name, non_unique, column_name + SELECT table_name, index_name, non_unique, column_name FROM information_schema.statistics - WHERE table_schema = ? AND table_name = ? - ORDER BY index_name, seq_in_index - `, schemaName, tableName) + WHERE table_schema = ? AND (? = '' OR table_name = ?) + ORDER BY table_name, index_name, seq_in_index + `, schemaName, tableName, tableName) if err != nil { return nil, fmt.Errorf("load indexes: %w", err) } defer func() { _ = rows.Close() }() - idxMap := make(map[string]*dbschema.Index) - var order []string + out := make(map[string][]dbschema.Index) for rows.Next() { - var name, col string + var tbl, name, col string var nonUnique int - if err := rows.Scan(&name, &nonUnique, &col); err != nil { - return nil, err + if err := rows.Scan(&tbl, &name, &nonUnique, &col); err != nil { + return nil, fmt.Errorf("load indexes: scan: %w", err) } - if _, ok := idxMap[name]; !ok { - idxMap[name] = &dbschema.Index{ - Name: name, - TableName: tableName, - IsUnique: nonUnique == 0, - IsPrimary: name == "PRIMARY", - } - order = append(order, name) + // Rows are ordered by (table, index), so an index's rows are + // contiguous and the last entry is the one to extend. + idxs := out[tbl] + if n := len(idxs); n > 0 && idxs[n-1].Name == name { + idxs[n-1].Columns = append(idxs[n-1].Columns, col) + continue } - idxMap[name].Columns = append(idxMap[name].Columns, col) + out[tbl] = append(idxs, dbschema.Index{ + Name: name, + TableName: tbl, + Columns: []string{col}, + IsUnique: nonUnique == 0, + IsPrimary: name == "PRIMARY", + }) } - result := make([]dbschema.Index, 0, len(order)) - for _, n := range order { - result = append(result, *idxMap[n]) + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load indexes: %w", err) } - return result, rows.Err() + return out, nil } -func (l *Loader) loadForeignKeys(db *sql.DB, schemaName, tableName string) ([]dbschema.ForeignKey, error) { +// loadForeignKeys returns foreign keys grouped by table name, ordered by +// constraint name. An empty tableName loads every table in schemaName. +func (l *Loader) loadForeignKeys(db *sql.DB, schemaName, tableName string) (map[string][]dbschema.ForeignKey, error) { rows, err := db.Query(` SELECT + kcu.table_name, kcu.constraint_name, kcu.column_name, kcu.referenced_table_name, @@ -169,40 +241,43 @@ func (l *Loader) loadForeignKeys(db *sql.DB, schemaName, tableName string) ([]db JOIN information_schema.referential_constraints rc ON rc.constraint_name = kcu.constraint_name AND rc.constraint_schema = kcu.table_schema - WHERE kcu.table_schema = ? AND kcu.table_name = ? + WHERE kcu.table_schema = ? AND (? = '' OR kcu.table_name = ?) AND kcu.referenced_table_name IS NOT NULL - ORDER BY kcu.constraint_name, kcu.ordinal_position - `, schemaName, tableName) + ORDER BY kcu.table_name, kcu.constraint_name, kcu.ordinal_position + `, schemaName, tableName, tableName) if err != nil { return nil, fmt.Errorf("load fks: %w", err) } defer func() { _ = rows.Close() }() - fkMap := make(map[string]*dbschema.ForeignKey) - var order []string + out := make(map[string][]dbschema.ForeignKey) for rows.Next() { - var name, col, refTable, refCol, onDelete, onUpdate string - if err := rows.Scan(&name, &col, &refTable, &refCol, &onDelete, &onUpdate); err != nil { - return nil, err + var tbl, name, col, refTable, refCol, onDelete, onUpdate string + if err := rows.Scan(&tbl, &name, &col, &refTable, &refCol, &onDelete, &onUpdate); err != nil { + return nil, fmt.Errorf("load fks: scan: %w", err) } - if _, ok := fkMap[name]; !ok { - fkMap[name] = &dbschema.ForeignKey{ - Name: name, - TableName: tableName, - RefTable: refTable, - OnDelete: onDelete, - OnUpdate: onUpdate, - } - order = append(order, name) + // Rows are ordered by (table, constraint), so a constraint's rows are + // contiguous and the last entry is the one to extend. + fks := out[tbl] + if n := len(fks); n > 0 && fks[n-1].Name == name { + fks[n-1].Columns = append(fks[n-1].Columns, col) + fks[n-1].RefColumns = append(fks[n-1].RefColumns, refCol) + continue } - fkMap[name].Columns = append(fkMap[name].Columns, col) - fkMap[name].RefColumns = append(fkMap[name].RefColumns, refCol) + out[tbl] = append(fks, dbschema.ForeignKey{ + Name: name, + TableName: tbl, + Columns: []string{col}, + RefTable: refTable, + RefColumns: []string{refCol}, + OnDelete: onDelete, + OnUpdate: onUpdate, + }) } - fks := make([]dbschema.ForeignKey, 0, len(order)) - for _, n := range order { - fks = append(fks, *fkMap[n]) + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load fks: %w", err) } - return fks, rows.Err() + return out, nil } // ensure Loader satisfies the interface (compile-time check). diff --git a/pkg/schema/mysql/loader_test.go b/pkg/schema/mysql/loader_test.go index 6aa28515..7c4aa9fa 100644 --- a/pkg/schema/mysql/loader_test.go +++ b/pkg/schema/mysql/loader_test.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" "os/exec" + "reflect" "runtime" "testing" "time" @@ -27,6 +28,7 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" + dbschema "github.com/ajitpratap0/GoSQLX/pkg/schema/db" myschema "github.com/ajitpratap0/GoSQLX/pkg/schema/mysql" ) @@ -141,6 +143,107 @@ func TestMySQLLoader_Load(t *testing.T) { if !names["products"] || !names["line_items"] { t.Errorf("missing tables in schema: %v", names) } + assertMySQLDetail(t, s) +} + +func findTable(s *dbschema.DatabaseSchema, name string) *dbschema.Table { + for i := range s.Tables { + if s.Tables[i].Name == name { + return &s.Tables[i] + } + } + return nil +} + +// assertMySQLDetail checks per-table columns, key flags, indexes and foreign +// keys of the fixture loaded via Load. +func assertMySQLDetail(t *testing.T, s *dbschema.DatabaseSchema) { + t.Helper() + products := findTable(s, "products") + lineItems := findTable(s, "line_items") + if products == nil || lineItems == nil { + t.Fatalf("missing products/line_items tables: %+v", s.Tables) + } + if len(products.Columns) != 3 { + t.Fatalf("products columns: got %d want 3", len(products.Columns)) + } + for _, c := range products.Columns { + switch c.Name { + case "id": + if !c.IsPrimary { + t.Error("products.id: expected primary key") + } + case "sku": + if !c.IsUnique { + t.Error("products.sku: expected unique") + } + } + } + if len(lineItems.Columns) != 3 { + t.Errorf("line_items columns: got %d want 3", len(lineItems.Columns)) + } + if len(lineItems.ForeignKeys) != 1 { + t.Fatalf("line_items foreign keys: got %d want 1", len(lineItems.ForeignKeys)) + } + fk := lineItems.ForeignKeys[0] + if fk.RefTable != "products" || fk.OnDelete != "RESTRICT" || + !reflect.DeepEqual(fk.Columns, []string{"product_id"}) || + !reflect.DeepEqual(fk.RefColumns, []string{"id"}) { + t.Errorf("line_items fk: got %+v", fk) + } + if len(products.Indexes) == 0 { + t.Error("products: expected indexes") + } + for _, idx := range products.Indexes { + if len(idx.Columns) == 0 || idx.TableName != "products" { + t.Errorf("products index: got %+v", idx) + } + } +} + +func TestMySQLLoader_Load_MaxOpenConns1(t *testing.T) { + db := startMySQL(t) + db.SetMaxOpenConns(1) + type result struct { + s *dbschema.DatabaseSchema + err error + } + done := make(chan result, 1) + go func() { + s, err := myschema.NewLoader().Load(db, "testdb") + done <- result{s, err} + }() + select { + case r := <-done: + if r.err != nil { + t.Fatalf("Load: %v", r.err) + } + assertMySQLDetail(t, r.s) + case <-time.After(30 * time.Second): + t.Fatal("Load deadlocked with MaxOpenConns=1") + } +} + +func TestMySQLLoader_LoadMatchesLoadTable(t *testing.T) { + db := startMySQL(t) + loader := myschema.NewLoader() + s, err := loader.Load(db, "testdb") + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(s.Tables) == 0 { + t.Fatal("Load returned no tables") + } + for i := range s.Tables { + want := s.Tables[i] + got, err := loader.LoadTable(db, "testdb", want.Name) + if err != nil { + t.Fatalf("LoadTable(%q): %v", want.Name, err) + } + if !reflect.DeepEqual(*got, want) { + t.Errorf("LoadTable(%q) differs from Load:\n got: %+v\nwant: %+v", want.Name, *got, want) + } + } } func TestMySQLLoader_LoadTable_Columns(t *testing.T) { diff --git a/pkg/schema/postgres/loader.go b/pkg/schema/postgres/loader.go index 5bc940cc..ccdac053 100644 --- a/pkg/schema/postgres/loader.go +++ b/pkg/schema/postgres/loader.go @@ -33,35 +33,45 @@ func NewLoader() *Loader { return &Loader{} } // Load returns the full schema for all user tables in the given schemaName. // If schemaName is empty, "public" is used. +// +// Load issues four statements (tables, columns, indexes, foreign keys) for the +// whole schema and drains each cursor before running the next, so it works +// with a single-connection pool. func (l *Loader) Load(db *sql.DB, schemaName string) (*dbschema.DatabaseSchema, error) { if schemaName == "" { schemaName = "public" } - rows, err := db.Query(` - SELECT table_name - FROM information_schema.tables - WHERE table_schema = $1 - AND table_type = 'BASE TABLE' - ORDER BY table_name - `, schemaName) + names, err := l.listTables(db, schemaName) if err != nil { - return nil, fmt.Errorf("list tables: %w", err) + return nil, err + } + cols, err := l.loadColumns(db, schemaName, "") + if err != nil { + return nil, err + } + idxs, err := l.loadIndexes(db, schemaName, "") + if err != nil { + return nil, err + } + fks, err := l.loadForeignKeys(db, schemaName, "") + if err != nil { + return nil, err } - defer func() { _ = rows.Close() }() ds := &dbschema.DatabaseSchema{Name: schemaName} - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - return nil, err - } - tbl, err := l.LoadTable(db, schemaName, name) - if err != nil { - return nil, err - } - ds.Tables = append(ds.Tables, *tbl) + if len(names) > 0 { + ds.Tables = make([]dbschema.Table, 0, len(names)) + } + for _, n := range names { + ds.Tables = append(ds.Tables, dbschema.Table{ + Schema: schemaName, + Name: n, + Columns: cols[n], + Indexes: idxs[n], + ForeignKeys: fks[n], + }) } - return ds, rows.Err() + return ds, nil } // LoadTable returns schema for a single named table. @@ -70,26 +80,61 @@ func (l *Loader) LoadTable(db *sql.DB, schemaName, tableName string) (*dbschema. if schemaName == "" { schemaName = "public" } - tbl := &dbschema.Table{Schema: schemaName, Name: tableName} - var err error - tbl.Columns, err = l.loadColumns(db, schemaName, tableName) + cols, err := l.loadColumns(db, schemaName, tableName) if err != nil { return nil, err } - tbl.Indexes, err = l.loadIndexes(db, schemaName, tableName) + idxs, err := l.loadIndexes(db, schemaName, tableName) if err != nil { return nil, err } - tbl.ForeignKeys, err = l.loadForeignKeys(db, schemaName, tableName) + fks, err := l.loadForeignKeys(db, schemaName, tableName) if err != nil { return nil, err } - return tbl, nil + return &dbschema.Table{ + Schema: schemaName, + Name: tableName, + Columns: cols[tableName], + Indexes: idxs[tableName], + ForeignKeys: fks[tableName], + }, nil } -func (l *Loader) loadColumns(db *sql.DB, schemaName, tableName string) ([]dbschema.Column, error) { +// listTables returns the base tables in schemaName, ordered by name. +func (l *Loader) listTables(db *sql.DB, schemaName string) ([]string, error) { + rows, err := db.Query(` + SELECT table_name + FROM information_schema.tables + WHERE table_schema = $1 + AND table_type = 'BASE TABLE' + ORDER BY table_name + `, schemaName) + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + defer func() { _ = rows.Close() }() + + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("list tables: scan: %w", err) + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + return names, nil +} + +// loadColumns returns columns grouped by table name, in ordinal order. +// An empty tableName loads every table in schemaName. +func (l *Loader) loadColumns(db *sql.DB, schemaName, tableName string) (map[string][]dbschema.Column, error) { rows, err := db.Query(` SELECT + c.table_name, c.column_name, c.ordinal_position, c.data_type, @@ -102,52 +147,60 @@ func (l *Loader) loadColumns(db *sql.DB, schemaName, tableName string) ([]dbsche COALESCE(uq.is_unique, false) FROM information_schema.columns c LEFT JOIN ( - SELECT kcu.column_name, true AS is_primary + SELECT kcu.table_name, kcu.column_name, true AS is_primary FROM information_schema.table_constraints tc JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema - WHERE tc.table_schema = $1 AND tc.table_name = $2 + WHERE tc.table_schema = $1 AND ($2::text = '' OR tc.table_name = $2) AND tc.constraint_type = 'PRIMARY KEY' - ) pk ON pk.column_name = c.column_name + ) pk ON pk.table_name = c.table_name AND pk.column_name = c.column_name LEFT JOIN ( - SELECT kcu.column_name, true AS is_unique + SELECT kcu.table_name, kcu.column_name, true AS is_unique FROM information_schema.table_constraints tc JOIN information_schema.key_column_usage kcu ON tc.constraint_name = kcu.constraint_name AND tc.table_schema = kcu.table_schema - WHERE tc.table_schema = $1 AND tc.table_name = $2 + WHERE tc.table_schema = $1 AND ($2::text = '' OR tc.table_name = $2) AND tc.constraint_type = 'UNIQUE' - ) uq ON uq.column_name = c.column_name - WHERE c.table_schema = $1 AND c.table_name = $2 - ORDER BY c.ordinal_position + ) uq ON uq.table_name = c.table_name AND uq.column_name = c.column_name + WHERE c.table_schema = $1 AND ($2::text = '' OR c.table_name = $2) + ORDER BY c.table_name, c.ordinal_position `, schemaName, tableName) if err != nil { return nil, fmt.Errorf("load columns: %w", err) } defer func() { _ = rows.Close() }() - var cols []dbschema.Column + out := make(map[string][]dbschema.Column) for rows.Next() { + var tbl string var col dbschema.Column var isNullable string err := rows.Scan( + &tbl, &col.Name, &col.OrdinalPos, &col.DataType, &isNullable, &col.DefaultValue, &col.MaxLength, &col.Precision, &col.Scale, &col.IsPrimary, &col.IsUnique, ) if err != nil { - return nil, err + return nil, fmt.Errorf("load columns: scan: %w", err) } col.IsNullable = isNullable == "YES" - cols = append(cols, col) + out[tbl] = append(out[tbl], col) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load columns: %w", err) } - return cols, rows.Err() + return out, nil } -func (l *Loader) loadIndexes(db *sql.DB, schemaName, tableName string) ([]dbschema.Index, error) { +// loadIndexes returns indexes grouped by table name, ordered by index name. +// An empty tableName loads every table in schemaName. +func (l *Loader) loadIndexes(db *sql.DB, schemaName, tableName string) (map[string][]dbschema.Index, error) { rows, err := db.Query(` SELECT + t.relname AS table_name, i.relname AS index_name, ix.indisunique, ix.indisprimary, @@ -157,32 +210,37 @@ func (l *Loader) loadIndexes(db *sql.DB, schemaName, tableName string) ([]dbsche JOIN pg_class i ON i.oid = ix.indexrelid JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = ANY(ix.indkey) JOIN pg_namespace n ON n.oid = t.relnamespace - WHERE n.nspname = $1 AND t.relname = $2 AND t.relkind = 'r' - GROUP BY i.relname, ix.indisunique, ix.indisprimary - ORDER BY i.relname + WHERE n.nspname = $1 AND ($2::text = '' OR t.relname = $2) AND t.relkind = 'r' + GROUP BY t.relname, i.relname, ix.indisunique, ix.indisprimary + ORDER BY t.relname, i.relname `, schemaName, tableName) if err != nil { return nil, fmt.Errorf("load indexes: %w", err) } defer func() { _ = rows.Close() }() - var indexes []dbschema.Index + out := make(map[string][]dbschema.Index) for rows.Next() { var idx dbschema.Index var colArray string - if err := rows.Scan(&idx.Name, &idx.IsUnique, &idx.IsPrimary, &colArray); err != nil { - return nil, err + if err := rows.Scan(&idx.TableName, &idx.Name, &idx.IsUnique, &idx.IsPrimary, &colArray); err != nil { + return nil, fmt.Errorf("load indexes: scan: %w", err) } - idx.TableName = tableName idx.Columns = parseArrayAgg(colArray) - indexes = append(indexes, idx) + out[idx.TableName] = append(out[idx.TableName], idx) } - return indexes, rows.Err() + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load indexes: %w", err) + } + return out, nil } -func (l *Loader) loadForeignKeys(db *sql.DB, schemaName, tableName string) ([]dbschema.ForeignKey, error) { +// loadForeignKeys returns foreign keys grouped by table name, ordered by +// constraint name. An empty tableName loads every table in schemaName. +func (l *Loader) loadForeignKeys(db *sql.DB, schemaName, tableName string) (map[string][]dbschema.ForeignKey, error) { rows, err := db.Query(` SELECT + tc.table_name, tc.constraint_name, kcu.column_name, ccu.table_name AS foreign_table_name, @@ -199,39 +257,42 @@ func (l *Loader) loadForeignKeys(db *sql.DB, schemaName, tableName string) ([]db JOIN information_schema.referential_constraints AS rc ON rc.constraint_name = tc.constraint_name WHERE tc.constraint_type = 'FOREIGN KEY' - AND tc.table_schema = $1 AND tc.table_name = $2 - ORDER BY tc.constraint_name, kcu.ordinal_position + AND tc.table_schema = $1 AND ($2::text = '' OR tc.table_name = $2) + ORDER BY tc.table_name, tc.constraint_name, kcu.ordinal_position `, schemaName, tableName) if err != nil { return nil, fmt.Errorf("load fks: %w", err) } defer func() { _ = rows.Close() }() - fkMap := make(map[string]*dbschema.ForeignKey) - var order []string + out := make(map[string][]dbschema.ForeignKey) for rows.Next() { - var name, col, refTable, refCol, onDelete, onUpdate string - if err := rows.Scan(&name, &col, &refTable, &refCol, &onDelete, &onUpdate); err != nil { - return nil, err + var tbl, name, col, refTable, refCol, onDelete, onUpdate string + if err := rows.Scan(&tbl, &name, &col, &refTable, &refCol, &onDelete, &onUpdate); err != nil { + return nil, fmt.Errorf("load fks: scan: %w", err) } - if _, ok := fkMap[name]; !ok { - fkMap[name] = &dbschema.ForeignKey{ - Name: name, - TableName: tableName, - RefTable: refTable, - OnDelete: onDelete, - OnUpdate: onUpdate, - } - order = append(order, name) + // Rows are ordered by (table, constraint), so a constraint's rows are + // contiguous and the last entry is the one to extend. + fks := out[tbl] + if n := len(fks); n > 0 && fks[n-1].Name == name { + fks[n-1].Columns = append(fks[n-1].Columns, col) + fks[n-1].RefColumns = append(fks[n-1].RefColumns, refCol) + continue } - fkMap[name].Columns = append(fkMap[name].Columns, col) - fkMap[name].RefColumns = append(fkMap[name].RefColumns, refCol) + out[tbl] = append(fks, dbschema.ForeignKey{ + Name: name, + TableName: tbl, + Columns: []string{col}, + RefTable: refTable, + RefColumns: []string{refCol}, + OnDelete: onDelete, + OnUpdate: onUpdate, + }) } - fks := make([]dbschema.ForeignKey, 0, len(order)) - for _, n := range order { - fks = append(fks, *fkMap[n]) + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load fks: %w", err) } - return fks, rows.Err() + return out, nil } // parseArrayAgg parses a Postgres array literal "{a,b,c}" into []string{"a","b","c"}. diff --git a/pkg/schema/postgres/loader_test.go b/pkg/schema/postgres/loader_test.go index 63086c22..aa2ac9c4 100644 --- a/pkg/schema/postgres/loader_test.go +++ b/pkg/schema/postgres/loader_test.go @@ -19,6 +19,7 @@ import ( "database/sql" "fmt" "os/exec" + "reflect" "runtime" "testing" "time" @@ -27,6 +28,7 @@ import ( "github.com/testcontainers/testcontainers-go" "github.com/testcontainers/testcontainers-go/wait" + dbschema "github.com/ajitpratap0/GoSQLX/pkg/schema/db" pgschema "github.com/ajitpratap0/GoSQLX/pkg/schema/postgres" ) @@ -140,6 +142,123 @@ func TestPostgresLoader_Load(t *testing.T) { if !tableNames["orders"] { t.Error("expected orders table") } + assertPostgresDetail(t, s) +} + +func findTable(s *dbschema.DatabaseSchema, name string) *dbschema.Table { + for i := range s.Tables { + if s.Tables[i].Name == name { + return &s.Tables[i] + } + } + return nil +} + +// assertPostgresDetail checks per-table columns, PK/UNIQUE flags, indexes and +// foreign keys of the fixture loaded via Load. +func assertPostgresDetail(t *testing.T, s *dbschema.DatabaseSchema) { + t.Helper() + users := findTable(s, "users") + orders := findTable(s, "orders") + if users == nil || orders == nil { + t.Fatalf("missing users/orders tables: %+v", s.Tables) + } + if len(users.Columns) != 3 { + t.Fatalf("users columns: got %d want 3", len(users.Columns)) + } + for _, c := range users.Columns { + switch c.Name { + case "id": + if !c.IsPrimary || c.IsUnique { + t.Errorf("users.id: primary=%v unique=%v, want primary only", c.IsPrimary, c.IsUnique) + } + case "email": + if !c.IsUnique || c.IsPrimary { + t.Errorf("users.email: primary=%v unique=%v, want unique only", c.IsPrimary, c.IsUnique) + } + case "name": + if c.IsUnique || c.IsPrimary { + t.Errorf("users.name: primary=%v unique=%v, want neither", c.IsPrimary, c.IsUnique) + } + } + } + if len(orders.Columns) != 3 { + t.Errorf("orders columns: got %d want 3", len(orders.Columns)) + } + for _, c := range orders.Columns { + // PK/UNIQUE flags must not leak across tables that share column names. + if c.IsUnique { + t.Errorf("orders.%s unexpectedly unique", c.Name) + } + if c.IsPrimary != (c.Name == "id") { + t.Errorf("orders.%s primary=%v", c.Name, c.IsPrimary) + } + } + if len(orders.ForeignKeys) != 1 { + t.Fatalf("orders foreign keys: got %d want 1", len(orders.ForeignKeys)) + } + fk := orders.ForeignKeys[0] + if fk.RefTable != "users" || fk.OnDelete != "CASCADE" || + !reflect.DeepEqual(fk.Columns, []string{"user_id"}) { + t.Errorf("orders fk: got %+v", fk) + } + var found bool + for _, idx := range orders.Indexes { + if idx.Name == "idx_orders_user" { + found = true + if !reflect.DeepEqual(idx.Columns, []string{"user_id"}) || idx.TableName != "orders" { + t.Errorf("idx_orders_user: got %+v", idx) + } + } + } + if !found { + t.Error("idx_orders_user not found") + } +} + +func TestPostgresLoader_Load_MaxOpenConns1(t *testing.T) { + db := startPostgres(t) + db.SetMaxOpenConns(1) + type result struct { + s *dbschema.DatabaseSchema + err error + } + done := make(chan result, 1) + go func() { + s, err := pgschema.NewLoader().Load(db, "public") + done <- result{s, err} + }() + select { + case r := <-done: + if r.err != nil { + t.Fatalf("Load: %v", r.err) + } + assertPostgresDetail(t, r.s) + case <-time.After(30 * time.Second): + t.Fatal("Load deadlocked with MaxOpenConns=1") + } +} + +func TestPostgresLoader_LoadMatchesLoadTable(t *testing.T) { + db := startPostgres(t) + loader := pgschema.NewLoader() + s, err := loader.Load(db, "public") + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(s.Tables) == 0 { + t.Fatal("Load returned no tables") + } + for i := range s.Tables { + want := s.Tables[i] + got, err := loader.LoadTable(db, "public", want.Name) + if err != nil { + t.Fatalf("LoadTable(%q): %v", want.Name, err) + } + if !reflect.DeepEqual(*got, want) { + t.Errorf("LoadTable(%q) differs from Load:\n got: %+v\nwant: %+v", want.Name, *got, want) + } + } } func TestPostgresLoader_LoadTable_Columns(t *testing.T) { diff --git a/pkg/schema/sqlite/loader.go b/pkg/schema/sqlite/loader.go index d0960665..4fe1386d 100644 --- a/pkg/schema/sqlite/loader.go +++ b/pkg/schema/sqlite/loader.go @@ -13,9 +13,12 @@ // limitations under the License. // Package sqlite provides a live schema Loader for SQLite databases. -// It uses PRAGMA commands (table_info, index_list, index_info, foreign_key_list) -// to retrieve tables, columns, indexes, and foreign keys. No CGO is required -// when using modernc.org/sqlite. +// It uses the pragma_table_info, pragma_index_list, pragma_index_info and +// pragma_foreign_key_list table-valued functions to retrieve tables, columns, +// indexes, and foreign keys with bound parameters. Load issues one query per +// kind for the whole database, and every cursor is drained and closed before +// the next statement runs, so a single connection is sufficient. No CGO is +// required when using modernc.org/sqlite. package sqlite import ( @@ -36,61 +39,115 @@ func NewLoader() *Loader { return &Loader{} } // schemaName is recorded in the returned DatabaseSchema but otherwise unused // (SQLite does not have named schemas in the PostgreSQL sense; use "main"). func (l *Loader) Load(db *sql.DB, schemaName string) (*dbschema.DatabaseSchema, error) { - rows, err := db.Query(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name`) + names, err := l.listTables(db) if err != nil { - return nil, fmt.Errorf("list tables: %w", err) + return nil, err + } + cols, err := l.loadColumns(db, "") + if err != nil { + return nil, err + } + idxs, err := l.loadIndexes(db, "") + if err != nil { + return nil, err + } + fks, err := l.loadForeignKeys(db, "") + if err != nil { + return nil, err } - defer func() { _ = rows.Close() }() ds := &dbschema.DatabaseSchema{Name: schemaName} - for rows.Next() { - var name string - if err := rows.Scan(&name); err != nil { - return nil, err - } - tbl, err := l.LoadTable(db, schemaName, name) - if err != nil { - return nil, err - } - ds.Tables = append(ds.Tables, *tbl) + if len(names) > 0 { + ds.Tables = make([]dbschema.Table, 0, len(names)) + } + for _, n := range names { + ds.Tables = append(ds.Tables, dbschema.Table{ + Schema: schemaName, + Name: n, + Columns: cols[n], + Indexes: idxs[n], + ForeignKeys: fks[n], + }) } - return ds, rows.Err() + return ds, nil } // LoadTable returns schema for a single named table. func (l *Loader) LoadTable(db *sql.DB, schemaName, tableName string) (*dbschema.Table, error) { - tbl := &dbschema.Table{Schema: schemaName, Name: tableName} - var err error - tbl.Columns, err = l.loadColumns(db, tableName) + cols, err := l.loadColumns(db, tableName) if err != nil { return nil, err } - tbl.Indexes, err = l.loadIndexes(db, tableName) + idxs, err := l.loadIndexes(db, tableName) if err != nil { return nil, err } - tbl.ForeignKeys, err = l.loadForeignKeys(db, tableName) + fks, err := l.loadForeignKeys(db, tableName) if err != nil { return nil, err } - return tbl, nil + return &dbschema.Table{ + Schema: schemaName, + Name: tableName, + Columns: cols[tableName], + Indexes: idxs[tableName], + ForeignKeys: fks[tableName], + }, nil +} + +// listTables returns the names of all user tables, ordered by name. +func (l *Loader) listTables(db *sql.DB) ([]string, error) { + rows, err := db.Query(`SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%' ORDER BY name`) + if err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + defer func() { _ = rows.Close() }() + + var names []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + return nil, fmt.Errorf("list tables: scan: %w", err) + } + names = append(names, name) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("list tables: %w", err) + } + return names, nil +} + +// tableSource returns the FROM-clause source of table names for the pragma +// table-valued functions: the single table tableName (as a bound parameter), +// or every user table when tableName is empty. The returned SQL is constant. +func tableSource(tableName string) (string, []any) { + if tableName != "" { + return `(SELECT ? AS name)`, []any{tableName} + } + return `(SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%')`, nil } -func (l *Loader) loadColumns(db *sql.DB, tableName string) ([]dbschema.Column, error) { - rows, err := db.Query(fmt.Sprintf("PRAGMA table_info(%q)", tableName)) +// loadColumns returns columns grouped by table name, in ordinal order. +// An empty tableName loads every user table. +func (l *Loader) loadColumns(db *sql.DB, tableName string) (map[string][]dbschema.Column, error) { + src, args := tableSource(tableName) + rows, err := db.Query(` + SELECT m.name, p.cid, p.name, p.type, p."notnull", p.dflt_value, p.pk + FROM `+src+` m + JOIN pragma_table_info(m.name) p + ORDER BY m.name, p.cid`, args...) if err != nil { - return nil, fmt.Errorf("pragma table_info: %w", err) + return nil, fmt.Errorf("load columns: %w", err) } defer func() { _ = rows.Close() }() - var cols []dbschema.Column + out := make(map[string][]dbschema.Column) for rows.Next() { - // cid, name, type, notnull, dflt_value, pk + var tbl, name, typ string var cid, notNull, pk int - var name, typ string var dflt sql.NullString - if err := rows.Scan(&cid, &name, &typ, ¬Null, &dflt, &pk); err != nil { - return nil, err + if err := rows.Scan(&tbl, &cid, &name, &typ, ¬Null, &dflt, &pk); err != nil { + return nil, fmt.Errorf("load columns: scan: %w", err) } col := dbschema.Column{ Name: name, @@ -102,88 +159,97 @@ func (l *Loader) loadColumns(db *sql.DB, tableName string) ([]dbschema.Column, e if dflt.Valid { col.DefaultValue = &dflt.String } - cols = append(cols, col) + out[tbl] = append(out[tbl], col) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load columns: %w", err) } - return cols, rows.Err() + return out, nil } -func (l *Loader) loadIndexes(db *sql.DB, tableName string) ([]dbschema.Index, error) { - rows, err := db.Query(fmt.Sprintf("PRAGMA index_list(%q)", tableName)) +// loadIndexes returns indexes grouped by table name, in index_list order, with +// each index's columns in key order. An empty tableName loads every user table. +func (l *Loader) loadIndexes(db *sql.DB, tableName string) (map[string][]dbschema.Index, error) { + src, args := tableSource(tableName) + rows, err := db.Query(` + SELECT m.name, il.name, il."unique", ii.name + FROM `+src+` m + JOIN pragma_index_list(m.name) il + JOIN pragma_index_info(il.name) ii + ORDER BY m.name, il.seq, ii.seqno`, args...) if err != nil { - return nil, fmt.Errorf("pragma index_list: %w", err) + return nil, fmt.Errorf("load indexes: %w", err) } defer func() { _ = rows.Close() }() - var indexes []dbschema.Index + out := make(map[string][]dbschema.Index) for rows.Next() { - // seq, name, unique, origin, partial - var seq, unique, partial int - var name, origin string - if err := rows.Scan(&seq, &name, &unique, &origin, &partial); err != nil { - return nil, err + var tbl, name, colName string + var unique int + if err := rows.Scan(&tbl, &name, &unique, &colName); err != nil { + return nil, fmt.Errorf("load indexes: scan: %w", err) + } + idxs := out[tbl] + if n := len(idxs); n > 0 && idxs[n-1].Name == name { + idxs[n-1].Columns = append(idxs[n-1].Columns, colName) + continue } - idx := dbschema.Index{ + out[tbl] = append(idxs, dbschema.Index{ Name: name, - TableName: tableName, + TableName: tbl, + Columns: []string{colName}, IsUnique: unique == 1, - } - // get columns in this index - icols, err := db.Query(fmt.Sprintf("PRAGMA index_info(%q)", name)) - if err != nil { - return nil, err - } - for icols.Next() { - var seqno, cid int - var colName string - if err := icols.Scan(&seqno, &cid, &colName); err != nil { - _ = icols.Close() - return nil, err - } - idx.Columns = append(idx.Columns, colName) - } - _ = icols.Close() - if err := icols.Err(); err != nil { - return nil, err - } - indexes = append(indexes, idx) + }) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load indexes: %w", err) } - return indexes, rows.Err() + return out, nil } -func (l *Loader) loadForeignKeys(db *sql.DB, tableName string) ([]dbschema.ForeignKey, error) { - rows, err := db.Query(fmt.Sprintf("PRAGMA foreign_key_list(%q)", tableName)) +// loadForeignKeys returns foreign keys grouped by table name, in id order, +// with columns in sequence order. An empty tableName loads every user table. +func (l *Loader) loadForeignKeys(db *sql.DB, tableName string) (map[string][]dbschema.ForeignKey, error) { + src, args := tableSource(tableName) + rows, err := db.Query(` + SELECT m.name, f.id, f."table", f."from", f."to", f.on_update, f.on_delete + FROM `+src+` m + JOIN pragma_foreign_key_list(m.name) f + ORDER BY m.name, f.id, f.seq`, args...) if err != nil { - return nil, fmt.Errorf("pragma foreign_key_list: %w", err) + return nil, fmt.Errorf("load fks: %w", err) } defer func() { _ = rows.Close() }() - fkMap := make(map[int]*dbschema.ForeignKey) - var order []int + out := make(map[string][]dbschema.ForeignKey) + lastTable, lastID := "", -1 for rows.Next() { - // id, seq, table, from, to, on_update, on_delete, match - var id, seq int - var refTable, fromCol, toCol, onUpdate, onDelete, match string - if err := rows.Scan(&id, &seq, &refTable, &fromCol, &toCol, &onUpdate, &onDelete, &match); err != nil { - return nil, err + var tbl, refTable, fromCol, toCol, onUpdate, onDelete string + var id int + if err := rows.Scan(&tbl, &id, &refTable, &fromCol, &toCol, &onUpdate, &onDelete); err != nil { + return nil, fmt.Errorf("load fks: scan: %w", err) } - if _, ok := fkMap[id]; !ok { - fkMap[id] = &dbschema.ForeignKey{ - Name: fmt.Sprintf("fk_%s_%d", strings.ToLower(tableName), id), - TableName: tableName, - RefTable: refTable, - OnDelete: onDelete, - OnUpdate: onUpdate, - } - order = append(order, id) + fks := out[tbl] + if n := len(fks); n > 0 && tbl == lastTable && id == lastID { + fks[n-1].Columns = append(fks[n-1].Columns, fromCol) + fks[n-1].RefColumns = append(fks[n-1].RefColumns, toCol) + continue } - fkMap[id].Columns = append(fkMap[id].Columns, fromCol) - fkMap[id].RefColumns = append(fkMap[id].RefColumns, toCol) + lastTable, lastID = tbl, id + out[tbl] = append(fks, dbschema.ForeignKey{ + Name: fmt.Sprintf("fk_%s_%d", strings.ToLower(tbl), id), + TableName: tbl, + Columns: []string{fromCol}, + RefTable: refTable, + RefColumns: []string{toCol}, + OnDelete: onDelete, + OnUpdate: onUpdate, + }) } - fks := make([]dbschema.ForeignKey, 0, len(order)) - for _, id := range order { - fks = append(fks, *fkMap[id]) + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("load fks: %w", err) } - return fks, rows.Err() + return out, nil } // ensure Loader satisfies the interface (compile-time check). diff --git a/pkg/schema/sqlite/loader_test.go b/pkg/schema/sqlite/loader_test.go index 8e311b5e..295ea390 100644 --- a/pkg/schema/sqlite/loader_test.go +++ b/pkg/schema/sqlite/loader_test.go @@ -16,10 +16,16 @@ package sqlite_test import ( "database/sql" + "fmt" + "path/filepath" + "reflect" + "strings" "testing" + "time" _ "modernc.org/sqlite" + dbschema "github.com/ajitpratap0/GoSQLX/pkg/schema/db" sqschema "github.com/ajitpratap0/GoSQLX/pkg/schema/sqlite" ) @@ -123,3 +129,378 @@ func TestSQLiteLoader_LoadTable_ForeignKeys(t *testing.T) { t.Errorf("fk ref table: got %q want items", tbl.ForeignKeys[0].RefTable) } } + +// richSchemaDDL creates a fixture with multiple indexes per table, multi-column +// indexes, and single and composite foreign keys. +const richSchemaDDL = ` + CREATE TABLE items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + qty INTEGER DEFAULT 0 + ); + CREATE TABLE cats ( + id INTEGER PRIMARY KEY, + label TEXT + ); + CREATE TABLE tags ( + item_id INTEGER NOT NULL REFERENCES items(id), + cat_id INTEGER REFERENCES cats(id) ON DELETE CASCADE, + tag TEXT NOT NULL, + UNIQUE(item_id, tag) + ); + CREATE INDEX idx_tags_item ON tags(item_id); + CREATE INDEX idx_tags_cat_tag ON tags(cat_id, tag); + CREATE TABLE pairs ( + a INTEGER, + b TEXT, + FOREIGN KEY(a, b) REFERENCES tags(item_id, tag) + ); +` + +// openSQLiteRich opens dsn and creates the rich fixture on it. +func openSQLiteRich(t *testing.T, dsn string) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + if _, err := db.Exec(richSchemaDDL); err != nil { + t.Fatalf("create tables: %v", err) + } + return db +} + +func findTable(s *dbschema.DatabaseSchema, name string) *dbschema.Table { + for i := range s.Tables { + if s.Tables[i].Name == name { + return &s.Tables[i] + } + } + return nil +} + +func findIndex(tbl *dbschema.Table, name string) *dbschema.Index { + for i := range tbl.Indexes { + if tbl.Indexes[i].Name == name { + return &tbl.Indexes[i] + } + } + return nil +} + +// assertRichDetail checks that the rich fixture was loaded with full column, +// index and foreign-key detail. +func assertRichDetail(t *testing.T, s *dbschema.DatabaseSchema) { + t.Helper() + tests := []struct { + name string + wantCols, wantIdx, wantFK int + }{ + {"cats", 2, 0, 0}, + {"items", 3, 0, 0}, + {"pairs", 2, 0, 1}, + {"tags", 3, 3, 2}, + } + if len(s.Tables) != len(tests) { + t.Errorf("tables: got %d want %d", len(s.Tables), len(tests)) + } + for _, tc := range tests { + tbl := findTable(s, tc.name) + if tbl == nil { + t.Errorf("table %q not found", tc.name) + continue + } + if len(tbl.Columns) != tc.wantCols { + t.Errorf("%s columns: got %d want %d", tc.name, len(tbl.Columns), tc.wantCols) + } + if len(tbl.Indexes) != tc.wantIdx { + t.Errorf("%s indexes: got %d want %d", tc.name, len(tbl.Indexes), tc.wantIdx) + } + if len(tbl.ForeignKeys) != tc.wantFK { + t.Errorf("%s foreign keys: got %d want %d", tc.name, len(tbl.ForeignKeys), tc.wantFK) + } + } + + tags := findTable(s, "tags") + if tags == nil { + return + } + assertTagsDetail(t, tags) + + pairs := findTable(s, "pairs") + if pairs != nil && len(pairs.ForeignKeys) == 1 { + fk := pairs.ForeignKeys[0] + if fk.RefTable != "tags" || + !reflect.DeepEqual(fk.Columns, []string{"a", "b"}) || + !reflect.DeepEqual(fk.RefColumns, []string{"item_id", "tag"}) { + t.Errorf("pairs composite fk: got %+v", fk) + } + } +} + +// assertTagsDetail checks the indexes and foreign keys of the tags table. +func assertTagsDetail(t *testing.T, tags *dbschema.Table) { + t.Helper() + wantIdx := []struct { + name string + cols []string + unique bool + }{ + {"idx_tags_cat_tag", []string{"cat_id", "tag"}, false}, + {"idx_tags_item", []string{"item_id"}, false}, + {"sqlite_autoindex_tags_1", []string{"item_id", "tag"}, true}, + } + for _, w := range wantIdx { + idx := findIndex(tags, w.name) + if idx == nil { + t.Errorf("index %q not found", w.name) + continue + } + if !reflect.DeepEqual(idx.Columns, w.cols) { + t.Errorf("index %q columns: got %v want %v", w.name, idx.Columns, w.cols) + } + if idx.IsUnique != w.unique { + t.Errorf("index %q unique: got %v want %v", w.name, idx.IsUnique, w.unique) + } + if idx.TableName != "tags" { + t.Errorf("index %q table: got %q want tags", w.name, idx.TableName) + } + } + + if len(tags.ForeignKeys) != 2 { + return + } + fk0, fk1 := tags.ForeignKeys[0], tags.ForeignKeys[1] + if fk0.Name != "fk_tags_0" || fk0.RefTable != "cats" || fk0.OnDelete != "CASCADE" || + !reflect.DeepEqual(fk0.Columns, []string{"cat_id"}) || + !reflect.DeepEqual(fk0.RefColumns, []string{"id"}) { + t.Errorf("tags fk[0]: got %+v", fk0) + } + if fk1.Name != "fk_tags_1" || fk1.RefTable != "items" || + !reflect.DeepEqual(fk1.Columns, []string{"item_id"}) || + !reflect.DeepEqual(fk1.RefColumns, []string{"id"}) { + t.Errorf("tags fk[1]: got %+v", fk1) + } +} + +// loadWithTimeout runs Load in a goroutine and fails the test if it does not +// return within d (a deadlock on a single-connection pool). +func loadWithTimeout(t *testing.T, db *sql.DB, d time.Duration) *dbschema.DatabaseSchema { + t.Helper() + type result struct { + s *dbschema.DatabaseSchema + err error + } + done := make(chan result, 1) + go func() { + s, err := sqschema.NewLoader().Load(db, "main") + done <- result{s, err} + }() + select { + case r := <-done: + if r.err != nil { + t.Fatalf("Load: %v", r.err) + } + return r.s + case <-time.After(d): + t.Fatal("Load deadlocked with MaxOpenConns=1") + } + return nil +} + +// TestSQLiteLoader_Load_InMemoryFullDetail guards against nested queries +// landing on a second pooled connection, which for ":memory:" is a different, +// empty database. +func TestSQLiteLoader_Load_InMemoryFullDetail(t *testing.T) { + db := openSQLiteRich(t, ":memory:") + s, err := sqschema.NewLoader().Load(db, "main") + if err != nil { + t.Fatalf("Load: %v", err) + } + assertRichDetail(t, s) +} + +func TestSQLiteLoader_Load_MaxOpenConns1(t *testing.T) { + db := openSQLiteRich(t, filepath.Join(t.TempDir(), "rich.db")) + db.SetMaxOpenConns(1) + s := loadWithTimeout(t, db, 10*time.Second) + assertRichDetail(t, s) +} + +func TestSQLiteLoader_LoadTable_MaxOpenConns1(t *testing.T) { + db := openSQLiteRich(t, filepath.Join(t.TempDir(), "rich.db")) + db.SetMaxOpenConns(1) + type result struct { + tbl *dbschema.Table + err error + } + done := make(chan result, 1) + go func() { + tbl, err := sqschema.NewLoader().LoadTable(db, "main", "tags") + done <- result{tbl, err} + }() + var r result + select { + case r = <-done: + case <-time.After(10 * time.Second): + t.Fatal("LoadTable deadlocked with MaxOpenConns=1") + } + if r.err != nil { + t.Fatalf("LoadTable: %v", r.err) + } + if len(r.tbl.Indexes) != 3 { + t.Fatalf("indexes: got %d want 3", len(r.tbl.Indexes)) + } + for _, idx := range r.tbl.Indexes { + if len(idx.Columns) == 0 { + t.Errorf("index %q has no columns", idx.Name) + } + } + assertTagsDetail(t, r.tbl) +} + +func TestSQLiteLoader_LoadMatchesLoadTable(t *testing.T) { + db := openSQLiteRich(t, filepath.Join(t.TempDir(), "rich.db")) + loader := sqschema.NewLoader() + s, err := loader.Load(db, "main") + if err != nil { + t.Fatalf("Load: %v", err) + } + if len(s.Tables) == 0 { + t.Fatal("Load returned no tables") + } + for i := range s.Tables { + want := s.Tables[i] + got, err := loader.LoadTable(db, "main", want.Name) + if err != nil { + t.Fatalf("LoadTable(%q): %v", want.Name, err) + } + if !reflect.DeepEqual(*got, want) { + t.Errorf("LoadTable(%q) differs from Load:\n got: %+v\nwant: %+v", want.Name, *got, want) + } + } +} + +func TestSQLiteLoader_IndexColumns_Multiple(t *testing.T) { + db := openSQLiteRich(t, ":memory:") + tbl, err := sqschema.NewLoader().LoadTable(db, "main", "tags") + if err != nil { + t.Fatalf("LoadTable: %v", err) + } + if len(tbl.Indexes) != 3 { + t.Fatalf("indexes: got %d want 3 (%+v)", len(tbl.Indexes), tbl.Indexes) + } + assertTagsDetail(t, tbl) +} + +func TestSQLiteLoader_ForeignKeys_Composite(t *testing.T) { + db := openSQLiteRich(t, ":memory:") + loader := sqschema.NewLoader() + + pairs, err := loader.LoadTable(db, "main", "pairs") + if err != nil { + t.Fatalf("LoadTable(pairs): %v", err) + } + if len(pairs.ForeignKeys) != 1 { + t.Fatalf("pairs foreign keys: got %d want 1", len(pairs.ForeignKeys)) + } + fk := pairs.ForeignKeys[0] + if fk.Name != "fk_pairs_0" || fk.TableName != "pairs" || fk.RefTable != "tags" { + t.Errorf("pairs fk: got %+v", fk) + } + if !reflect.DeepEqual(fk.Columns, []string{"a", "b"}) { + t.Errorf("pairs fk columns: got %v want [a b]", fk.Columns) + } + if !reflect.DeepEqual(fk.RefColumns, []string{"item_id", "tag"}) { + t.Errorf("pairs fk ref columns: got %v want [item_id tag]", fk.RefColumns) + } + + tags, err := loader.LoadTable(db, "main", "tags") + if err != nil { + t.Fatalf("LoadTable(tags): %v", err) + } + if len(tags.ForeignKeys) != 2 { + t.Fatalf("tags foreign keys: got %d want 2", len(tags.ForeignKeys)) + } + assertTagsDetail(t, tags) +} + +func TestSQLiteLoader_QuotedTableName(t *testing.T) { + db := openSQLiteRich(t, ":memory:") + if _, err := db.Exec(`CREATE TABLE "we""ird" (id INTEGER PRIMARY KEY)`); err != nil { + t.Fatalf("create table: %v", err) + } + loader := sqschema.NewLoader() + const name = `we"ird` + + tbl, err := loader.LoadTable(db, "main", name) + if err != nil { + t.Fatalf("LoadTable: %v", err) + } + if len(tbl.Columns) != 1 || tbl.Columns[0].Name != "id" || !tbl.Columns[0].IsPrimary { + t.Errorf("LoadTable columns: got %+v", tbl.Columns) + } + + s, err := loader.Load(db, "main") + if err != nil { + t.Fatalf("Load: %v", err) + } + fromLoad := findTable(s, name) + if fromLoad == nil { + t.Fatalf("table %q missing from Load", name) + } + if len(fromLoad.Columns) != 1 { + t.Errorf("Load columns: got %d want 1", len(fromLoad.Columns)) + } +} + +func TestSQLiteLoader_LoadTable_Missing(t *testing.T) { + db := openSQLiteRich(t, ":memory:") + tbl, err := sqschema.NewLoader().LoadTable(db, "main", "no_such_table") + if err != nil { + t.Fatalf("LoadTable: %v", err) + } + if tbl.Name != "no_such_table" || tbl.Schema != "main" { + t.Errorf("table identity: got %q.%q", tbl.Schema, tbl.Name) + } + if len(tbl.Columns) != 0 || len(tbl.Indexes) != 0 || len(tbl.ForeignKeys) != 0 { + t.Errorf("expected no columns, indexes or foreign keys, got %+v", tbl) + } +} + +func BenchmarkSQLiteLoader_Load(b *testing.B) { + for _, n := range []int{10, 100} { + b.Run(fmt.Sprintf("tables=%d", n), func(b *testing.B) { + db, err := sql.Open("sqlite", filepath.Join(b.TempDir(), "bench.db")) + if err != nil { + b.Fatalf("open sqlite: %v", err) + } + defer func() { _ = db.Close() }() + + var ddl strings.Builder + ddl.WriteString("CREATE TABLE parent (id INTEGER PRIMARY KEY, name TEXT);\n") + for i := 0; i < n; i++ { + fmt.Fprintf(&ddl, + "CREATE TABLE t%03d (id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES parent(id), val TEXT NOT NULL, qty INTEGER DEFAULT 0);\n"+ + "CREATE INDEX idx_t%03d_parent ON t%03d(parent_id);\n", i, i, i) + } + if _, err := db.Exec(ddl.String()); err != nil { + b.Fatalf("create tables: %v", err) + } + + loader := sqschema.NewLoader() + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + s, err := loader.Load(db, "main") + if err != nil { + b.Fatalf("Load: %v", err) + } + if len(s.Tables) != n+1 { + b.Fatalf("tables: got %d want %d", len(s.Tables), n+1) + } + } + }) + } +} From 09b7016c3a38ae8c50c135754421f9424711cab6 Mon Sep 17 00:00:00 2001 From: Ajit Pratap Singh Date: Sun, 11 Oct 2026 22:32:57 +0530 Subject: [PATCH 2/2] fix(schema): handle NULL index and FK targets in SQLite loader; reject empty table name SQLite Load failed on expression indexes (NULL index column name) and on REFERENCES without a column list (NULL "to"). Scan both as NullString: skip expression columns and resolve an omitted FK target to the parent's primary key columns. LoadTable now returns an error for an empty table name in the Postgres, MySQL and SQLite loaders. Co-Authored-By: Claude Opus 5.5 --- pkg/schema/mysql/loader.go | 4 ++ pkg/schema/mysql/loader_test.go | 7 ++++ pkg/schema/postgres/loader.go | 4 ++ pkg/schema/postgres/loader_test.go | 7 ++++ pkg/schema/sqlite/loader.go | 44 +++++++++++++++------- pkg/schema/sqlite/loader_test.go | 60 ++++++++++++++++++++++++++++++ 6 files changed, 112 insertions(+), 14 deletions(-) diff --git a/pkg/schema/mysql/loader.go b/pkg/schema/mysql/loader.go index a761104d..df45e959 100644 --- a/pkg/schema/mysql/loader.go +++ b/pkg/schema/mysql/loader.go @@ -19,6 +19,7 @@ package mysql import ( "database/sql" + "errors" "fmt" dbschema "github.com/ajitpratap0/GoSQLX/pkg/schema/db" @@ -71,6 +72,9 @@ func (l *Loader) Load(db *sql.DB, schemaName string) (*dbschema.DatabaseSchema, // LoadTable returns schema for a single named table. func (l *Loader) LoadTable(db *sql.DB, schemaName, tableName string) (*dbschema.Table, error) { + if tableName == "" { + return nil, errors.New("load table: table name must not be empty") + } cols, err := l.loadColumns(db, schemaName, tableName) if err != nil { return nil, err diff --git a/pkg/schema/mysql/loader_test.go b/pkg/schema/mysql/loader_test.go index 7c4aa9fa..2af05729 100644 --- a/pkg/schema/mysql/loader_test.go +++ b/pkg/schema/mysql/loader_test.go @@ -284,3 +284,10 @@ func TestMySQLLoader_LoadTable_ForeignKeys(t *testing.T) { t.Errorf("fk ref table: got %q want products", tbl.ForeignKeys[0].RefTable) } } + +// The name is validated before any query, so no database is needed. +func TestMySQLLoader_LoadTable_EmptyName(t *testing.T) { + if _, err := myschema.NewLoader().LoadTable(nil, "db", ""); err == nil { + t.Fatal("expected error for empty table name") + } +} diff --git a/pkg/schema/postgres/loader.go b/pkg/schema/postgres/loader.go index ccdac053..659baff3 100644 --- a/pkg/schema/postgres/loader.go +++ b/pkg/schema/postgres/loader.go @@ -19,6 +19,7 @@ package postgres import ( "database/sql" + "errors" "fmt" "strings" @@ -77,6 +78,9 @@ func (l *Loader) Load(db *sql.DB, schemaName string) (*dbschema.DatabaseSchema, // LoadTable returns schema for a single named table. // If schemaName is empty, "public" is used. func (l *Loader) LoadTable(db *sql.DB, schemaName, tableName string) (*dbschema.Table, error) { + if tableName == "" { + return nil, errors.New("load table: table name must not be empty") + } if schemaName == "" { schemaName = "public" } diff --git a/pkg/schema/postgres/loader_test.go b/pkg/schema/postgres/loader_test.go index aa2ac9c4..9f8b33ec 100644 --- a/pkg/schema/postgres/loader_test.go +++ b/pkg/schema/postgres/loader_test.go @@ -306,3 +306,10 @@ func TestPostgresLoader_LoadTable_ForeignKeys(t *testing.T) { t.Errorf("fk on delete: got %q want CASCADE", fk.OnDelete) } } + +// The name is validated before any query, so no database is needed. +func TestPostgresLoader_LoadTable_EmptyName(t *testing.T) { + if _, err := pgschema.NewLoader().LoadTable(nil, "public", ""); err == nil { + t.Fatal("expected error for empty table name") + } +} diff --git a/pkg/schema/sqlite/loader.go b/pkg/schema/sqlite/loader.go index 4fe1386d..d7371f42 100644 --- a/pkg/schema/sqlite/loader.go +++ b/pkg/schema/sqlite/loader.go @@ -23,6 +23,7 @@ package sqlite import ( "database/sql" + "errors" "fmt" "strings" @@ -74,6 +75,9 @@ func (l *Loader) Load(db *sql.DB, schemaName string) (*dbschema.DatabaseSchema, // LoadTable returns schema for a single named table. func (l *Loader) LoadTable(db *sql.DB, schemaName, tableName string) (*dbschema.Table, error) { + if tableName == "" { + return nil, errors.New("load table: table name must not be empty") + } cols, err := l.loadColumns(db, tableName) if err != nil { return nil, err @@ -172,7 +176,7 @@ func (l *Loader) loadColumns(db *sql.DB, tableName string) (map[string][]dbschem func (l *Loader) loadIndexes(db *sql.DB, tableName string) (map[string][]dbschema.Index, error) { src, args := tableSource(tableName) rows, err := db.Query(` - SELECT m.name, il.name, il."unique", ii.name + SELECT m.name, il.name, il."unique", ii.name -- NULL for expression columns FROM `+src+` m JOIN pragma_index_list(m.name) il JOIN pragma_index_info(il.name) ii @@ -184,22 +188,27 @@ func (l *Loader) loadIndexes(db *sql.DB, tableName string) (map[string][]dbschem out := make(map[string][]dbschema.Index) for rows.Next() { - var tbl, name, colName string + var tbl, name string + var colName sql.NullString var unique int if err := rows.Scan(&tbl, &name, &unique, &colName); err != nil { return nil, fmt.Errorf("load indexes: scan: %w", err) } idxs := out[tbl] - if n := len(idxs); n > 0 && idxs[n-1].Name == name { - idxs[n-1].Columns = append(idxs[n-1].Columns, colName) - continue + if n := len(idxs); n == 0 || idxs[n-1].Name != name { + out[tbl] = append(idxs, dbschema.Index{ + Name: name, + TableName: tbl, + IsUnique: unique == 1, + }) + idxs = out[tbl] + } + // Expression index columns (e.g. lower(x)) have a NULL name; they + // cannot be represented as a column name, so they are skipped. + if colName.Valid { + last := &idxs[len(idxs)-1] + last.Columns = append(last.Columns, colName.String) } - out[tbl] = append(idxs, dbschema.Index{ - Name: name, - TableName: tbl, - Columns: []string{colName}, - IsUnique: unique == 1, - }) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("load indexes: %w", err) @@ -212,7 +221,9 @@ func (l *Loader) loadIndexes(db *sql.DB, tableName string) (map[string][]dbschem func (l *Loader) loadForeignKeys(db *sql.DB, tableName string) (map[string][]dbschema.ForeignKey, error) { src, args := tableSource(tableName) rows, err := db.Query(` - SELECT m.name, f.id, f."table", f."from", f."to", f.on_update, f.on_delete + SELECT m.name, f.id, f."table", f."from", + COALESCE(f."to", (SELECT p.name FROM pragma_table_info(f."table") p WHERE p.pk = f.seq + 1)), + f.on_update, f.on_delete FROM `+src+` m JOIN pragma_foreign_key_list(m.name) f ORDER BY m.name, f.id, f.seq`, args...) @@ -224,11 +235,16 @@ func (l *Loader) loadForeignKeys(db *sql.DB, tableName string) (map[string][]dbs out := make(map[string][]dbschema.ForeignKey) lastTable, lastID := "", -1 for rows.Next() { - var tbl, refTable, fromCol, toCol, onUpdate, onDelete string + var tbl, refTable, fromCol, onUpdate, onDelete string + var toNull sql.NullString var id int - if err := rows.Scan(&tbl, &id, &refTable, &fromCol, &toCol, &onUpdate, &onDelete); err != nil { + if err := rows.Scan(&tbl, &id, &refTable, &fromCol, &toNull, &onUpdate, &onDelete); err != nil { return nil, fmt.Errorf("load fks: scan: %w", err) } + // "to" is NULL when REFERENCES omits the column list; the COALESCE + // above resolves it to the parent's primary key column at the same + // position. It stays empty only if the parent or its PK is missing. + toCol := toNull.String fks := out[tbl] if n := len(fks); n > 0 && tbl == lastTable && id == lastID { fks[n-1].Columns = append(fks[n-1].Columns, fromCol) diff --git a/pkg/schema/sqlite/loader_test.go b/pkg/schema/sqlite/loader_test.go index 295ea390..9076e16b 100644 --- a/pkg/schema/sqlite/loader_test.go +++ b/pkg/schema/sqlite/loader_test.go @@ -504,3 +504,63 @@ func BenchmarkSQLiteLoader_Load(b *testing.B) { }) } } + +func TestSQLiteLoader_ExpressionIndex(t *testing.T) { + db := openSQLite(t) + if _, err := db.Exec(` + CREATE INDEX idx_items_lower ON items(lower(name)); + CREATE INDEX idx_items_mixed ON items(qty, lower(name)); + `); err != nil { + t.Fatalf("create indexes: %v", err) + } + s, err := sqschema.NewLoader().Load(db, "main") + if err != nil { + t.Fatalf("Load with expression index: %v", err) + } + items := findTable(s, "items") + if items == nil { + t.Fatal("items table missing") + } + if idx := findIndex(items, "idx_items_lower"); idx == nil || len(idx.Columns) != 0 { + t.Errorf("idx_items_lower: got %+v, want present with no columns", idx) + } + if idx := findIndex(items, "idx_items_mixed"); idx == nil || !reflect.DeepEqual(idx.Columns, []string{"qty"}) { + t.Errorf("idx_items_mixed: got %+v, want columns [qty]", idx) + } +} + +func TestSQLiteLoader_ForeignKey_ImplicitParentKey(t *testing.T) { + db := openSQLite(t) + if _, err := db.Exec(` + CREATE TABLE pair (a INTEGER, b INTEGER, PRIMARY KEY (a, b)); + CREATE TABLE child1 (item_id INTEGER REFERENCES items); + CREATE TABLE child2 (x INTEGER, y INTEGER, FOREIGN KEY (x, y) REFERENCES pair); + CREATE TABLE child3 (z INTEGER REFERENCES missing_parent); + `); err != nil { + t.Fatalf("create tables: %v", err) + } + s, err := sqschema.NewLoader().Load(db, "main") + if err != nil { + t.Fatalf("Load with implicit FK target: %v", err) + } + check := func(table string, wantCols, wantRef []string) { + t.Helper() + tbl := findTable(s, table) + if tbl == nil || len(tbl.ForeignKeys) != 1 { + t.Fatalf("%s: expected one FK, got %+v", table, tbl) + } + fk := tbl.ForeignKeys[0] + if !reflect.DeepEqual(fk.Columns, wantCols) || !reflect.DeepEqual(fk.RefColumns, wantRef) { + t.Errorf("%s: got %v -> %v, want %v -> %v", table, fk.Columns, fk.RefColumns, wantCols, wantRef) + } + } + check("child1", []string{"item_id"}, []string{"id"}) + check("child2", []string{"x", "y"}, []string{"a", "b"}) + check("child3", []string{"z"}, []string{""}) +} + +func TestSQLiteLoader_LoadTable_EmptyName(t *testing.T) { + if _, err := sqschema.NewLoader().LoadTable(openSQLite(t), "main", ""); err == nil { + t.Fatal("expected error for empty table name") + } +}