Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions .changes/unreleased/Fixed-20260816-120000.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
kind: Fixed
body: 'SQLite-family queries that combine a `sqlc.slice` with a named argument (`sqlc.arg`, `@name`, `:name`) now bind their arguments to the right columns. sqlc numbers every placeholder of such a query (`?1`, `?2`, ...) and assumes the slice marker occupies one slot, but the marker expands to one slot per element, so every index after it named the wrong slot: the query returned wrong rows, or the driver rejected it. The generated SQL constant now carries bare `?` placeholders and the arguments are passed in text order, which is correct for any slice length including an empty one.'
time: 2026-08-16T12:00:00.0000000Z
custom:
Author: Rayakame
PR: "260"
7 changes: 7 additions & 0 deletions internal/config/constants.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,13 @@ func (dr SQLDriver) IsPsycopg() bool {
return dr == SQLDriverPsycopgAsync || dr == SQLDriverPsycopgSync
}

// IsSqliteFamily reports whether the driver executes SQLite SQL: the two
// sqlite modules and both pyturso flavors share sqlc's sqlite engine and its
// placeholder numbering.
func (dr SQLDriver) IsSqliteFamily() bool {
return driversEngine[dr] == engineSQLite
}

// IsTurso reports whether the driver is one of the two pyturso flavors,
// which share inline type conversion in both directions - pyturso has no
// adapter/converter registry and binds only None, numbers, strings, and
Expand Down
19 changes: 19 additions & 0 deletions internal/config/constants_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,25 @@ func TestSQLDriverIsPsycopg(t *testing.T) {
}
}

func TestSQLDriverIsSqliteFamily(t *testing.T) {
t.Parallel()
for driver, want := range map[config.SQLDriver]bool{
config.SQLDriverSQLite: true,
config.SQLDriverAioSQLite: true,
config.SQLDriverTursoSync: true,
config.SQLDriverTursoAsync: true,
config.SQLDriverAsyncpg: false,
config.SQLDriverPsycopgSync: false,
config.SQLDriverPsycopgAsync: false,
config.SQLDriverPymysql: false,
config.SQLDriverAsyncmy: false,
} {
if got := driver.IsSqliteFamily(); got != want {
t.Errorf("IsSqliteFamily(%q) = %v, want %v", driver, got, want)
}
}
}

func TestSQLDriverIsTurso(t *testing.T) {
t.Parallel()
for driver, want := range map[config.SQLDriver]bool{
Expand Down
29 changes: 24 additions & 5 deletions internal/sqllex/lex.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,13 +25,19 @@ const sliceMarkerPrefix = "/*SLICE:"
// dashComment is the line-comment introducer both engines share.
const dashComment = "--"

// decimalBase is the radix of a numbered placeholder's index.
const decimalBase = 10

// Token is one scanned span of the input, [Start, End).
type Token struct {
Kind Kind
Start int
End int
// Name is the sqlc.slice name of a KindSliceMarker token.
Name string
// Number is the explicit index of a numbered placeholder (sqlite's ?N),
// or 0 when the placeholder carries none.
Number int
// MarkerEnd splits a KindSliceMarker: [Start, MarkerEnd) is the comment,
// [MarkerEnd, End) the placeholder it binds. Zero for other kinds.
MarkerEnd int
Expand All @@ -43,6 +49,11 @@ type Token struct {
type Slot struct {
Name string
Marker string
// Number is the explicit index of a numbered placeholder (sqlite's ?N),
// or 0 when the placeholder carries none. SQLite binds ?N to slot N and
// a bare ? to one past the highest slot seen so far, so the two spell
// different bindings for the same text position.
Number int
}

// Scan splits sql into tokens under the given dialect. Unterminated strings,
Expand All @@ -53,12 +64,12 @@ func Scan(sql string, d Dialect) []Token {
text := 0
flush := func(end int) {
if end > text {
tokens = append(tokens, Token{Kind: KindText, Start: text, End: end, Name: "", MarkerEnd: 0})
tokens = append(tokens, Token{Kind: KindText, Start: text, End: end, Name: "", Number: 0, MarkerEnd: 0})
}
}
skip := func(start, end int) {
flush(start)
tokens = append(tokens, Token{Kind: KindSkipped, Start: start, End: end, Name: "", MarkerEnd: 0})
tokens = append(tokens, Token{Kind: KindSkipped, Start: start, End: end, Name: "", Number: 0, MarkerEnd: 0})
text = end
}

Expand All @@ -78,6 +89,7 @@ func Scan(sql string, d Dialect) []Token {
Start: i,
End: token,
Name: d.sliceName(sql[i+len(sliceMarkerPrefix) : end]),
Number: 0,
MarkerEnd: end,
})
i, text = token, token
Expand Down Expand Up @@ -119,12 +131,19 @@ func Scan(sql string, d Dialect) []Token {
case d.placeholder != "" && strings.HasPrefix(rest, d.placeholder):
flush(i)
end := i + len(d.placeholder)
number := 0
if d.numbered {
digits := end
for end < len(sql) && sql[end] >= '0' && sql[end] <= '9' {
end++
}
// Overflow cannot happen in practice: sqlc rejects gaps, so
// the highest index it emits is the parameter count.
for _, b := range []byte(sql[digits:end]) {
number = number*decimalBase + int(b-'0')
}
}
tokens = append(tokens, Token{Kind: KindPlaceholder, Start: i, End: end, Name: "", MarkerEnd: 0})
tokens = append(tokens, Token{Kind: KindPlaceholder, Start: i, End: end, Name: "", Number: number, MarkerEnd: 0})
i, text = end, end
default:
i++
Expand Down Expand Up @@ -156,9 +175,9 @@ func Slots(sql string, d Dialect) []Slot {
for _, token := range Scan(sql, d) {
switch token.Kind {
case KindPlaceholder:
slots = append(slots, Slot{Name: "", Marker: ""})
slots = append(slots, Slot{Name: "", Marker: "", Number: token.Number})
case KindSliceMarker:
slots = append(slots, Slot{Name: token.Name, Marker: sql[token.Start:token.End]})
slots = append(slots, Slot{Name: token.Name, Marker: sql[token.Start:token.End], Number: 0})
case KindText, KindSkipped:
}
}
Expand Down
32 changes: 18 additions & 14 deletions internal/sqllex/lex_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
)

// plain is a bind slot that is not a sqlc.slice marker.
var plain = sqllex.Slot{Name: "", Marker: ""} //nolint:gochecknoglobals
var plain = sqllex.Slot{Name: "", Marker: "", Number: 0} //nolint:gochecknoglobals

func TestSlotsMySQLRaw(t *testing.T) {
t.Parallel()
Expand All @@ -26,16 +26,16 @@ func TestSlotsMySQLRaw(t *testing.T) {
{
name: "slice marker carries its text",
sql: "SELECT a FROM t WHERE id IN (/*SLICE:ids*/?)",
want: []sqllex.Slot{{Name: "ids", Marker: "/*SLICE:ids*/?"}},
want: []sqllex.Slot{{Name: "ids", Marker: "/*SLICE:ids*/?", Number: 0}},
},
{
// The ordering case the drivers depend on.
name: "reused marker interleaved with a plain slot",
sql: "SELECT a FROM t WHERE x IN (/*SLICE:ids*/?) AND y = ? OR z IN (/*SLICE:ids*/?)",
want: []sqllex.Slot{
{Name: "ids", Marker: "/*SLICE:ids*/?"},
{Name: "ids", Marker: "/*SLICE:ids*/?", Number: 0},
plain,
{Name: "ids", Marker: "/*SLICE:ids*/?"},
{Name: "ids", Marker: "/*SLICE:ids*/?", Number: 0},
},
},
{
Expand Down Expand Up @@ -133,7 +133,7 @@ func TestSlotsMySQLPyformat(t *testing.T) {
{
name: "rewritten marker carries its text",
sql: "SELECT a FROM t WHERE id IN (/*SLICE:ids*/%s)",
want: []sqllex.Slot{{Name: "ids", Marker: "/*SLICE:ids*/%s"}},
want: []sqllex.Slot{{Name: "ids", Marker: "/*SLICE:ids*/%s", Number: 0}},
},
{
// The rewriter doubles a percent inside the marker; the name has
Expand All @@ -146,9 +146,9 @@ func TestSlotsMySQLPyformat(t *testing.T) {
name: "reused marker interleaved with a plain slot",
sql: "SELECT a FROM t WHERE x IN (/*SLICE:ids*/%s) AND y = %s OR z IN (/*SLICE:ids*/%s)",
want: []sqllex.Slot{
{Name: "ids", Marker: "/*SLICE:ids*/%s"},
{Name: "ids", Marker: "/*SLICE:ids*/%s", Number: 0},
plain,
{Name: "ids", Marker: "/*SLICE:ids*/%s"},
{Name: "ids", Marker: "/*SLICE:ids*/%s", Number: 0},
},
},
}
Expand All @@ -163,23 +163,27 @@ func TestSlotsSQLite(t *testing.T) {
want []sqllex.Slot
}{
{
// sqlc numbers SQLite parameters; the digits belong to the slot.
name: "numbered placeholders count once each",
// sqlc numbers SQLite parameters; the digits belong to the slot
// and name the index SQLite will bind it to.
name: "numbered placeholders report their index",
sql: "SELECT a FROM t WHERE b = ?1 AND c = ?12",
want: []sqllex.Slot{plain, plain},
want: []sqllex.Slot{
{Name: "", Marker: "", Number: 1},
{Name: "", Marker: "", Number: 12},
},
},
{
name: "slice marker carries its text",
sql: "SELECT a FROM t WHERE id IN (/*SLICE:ids*/?)",
want: []sqllex.Slot{{Name: "ids", Marker: "/*SLICE:ids*/?"}},
want: []sqllex.Slot{{Name: "ids", Marker: "/*SLICE:ids*/?", Number: 0}},
},
{
name: "reused marker interleaved with a numbered slot",
sql: "SELECT a FROM t WHERE x IN (/*SLICE:ids*/?) AND y = ?2 OR z IN (/*SLICE:ids*/?)",
want: []sqllex.Slot{
{Name: "ids", Marker: "/*SLICE:ids*/?"},
plain,
{Name: "ids", Marker: "/*SLICE:ids*/?"},
{Name: "ids", Marker: "/*SLICE:ids*/?", Number: 0},
{Name: "", Marker: "", Number: 2},
{Name: "ids", Marker: "/*SLICE:ids*/?", Number: 0},
},
},
{
Expand Down
10 changes: 10 additions & 0 deletions internal/transform/queries.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"strings"

"github.com/rayakame/sqlc-gen-better-python/internal/model"
"github.com/rayakame/sqlc-gen-better-python/internal/sqllex"
"github.com/rayakame/sqlc-gen-better-python/internal/utils"
"github.com/sqlc-dev/plugin-sdk-go/metadata"
"github.com/sqlc-dev/plugin-sdk-go/plugin"
Expand Down Expand Up @@ -287,6 +288,15 @@ func (t *Transformer) BuildQueries(tables []model.Table) []model.Query {
} else {
query.Params = t.plainParams(pluginQuery, pluginParams)
}
// sqlc numbers SQLite placeholders as soon as a query uses a named
// argument, and those indexes stop matching once a sqlc.slice marker
// expands. Strip them and bind by position instead - for the fields of
// a bundled Params class too, which the drivers expand positionally.
if t.config.SqlDriver.IsSqliteFamily() {
if slots := sqllex.Slots(query.SQL, sqllex.SQLite); needsSlotRewrite(slots) && reorderBindOrder(&query, slots) {
query.SQL = rewriteSQLiteSQL(query.SQL)
}
}

if query.Cmd == metadata.CmdExecLastId {
query.Returns.Type = model.PyType{Type: "int", IsNullable: true}
Expand Down
92 changes: 92 additions & 0 deletions internal/transform/queries_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package transform_test

import (
"reflect"
"strings"
"testing"

"github.com/rayakame/sqlc-gen-better-python/internal/config"
Expand Down Expand Up @@ -851,6 +852,97 @@ func TestBuildQueriesSqliteKeepsSameNamedParams(t *testing.T) {
}
}

func TestBuildQueriesSqliteSliceWithNamedArg(t *testing.T) {
t.Parallel()
// sqlc numbers the placeholders of a query that uses a named argument and
// assumes the marker takes one slot; it takes one per element. The indexes
// go, and the arguments follow the text instead - the reused one twice.
sliceCol := func() *plugin.Column {
column := queryCol("ids", "int4", nil)
column.IsSqlcSlice = true

return column
}
query := buildSingleQuery(t, &config.Config{SqlDriver: config.SQLDriverSQLite}, &plugin.Query{
Name: "ListAuthors",
Cmd: ":many",
Text: "SELECT id FROM test_authors WHERE id IN (/*SLICE:ids*/?) AND name = ?2 AND name != ?2",
Params: []*plugin.Parameter{
{Number: 1, Column: sliceCol()},
{Number: 2, Column: queryCol("n", "text", nil)},
},
})
want := []struct {
name string
repeated bool
}{{"ids", false}, {"n", false}, {"n", true}}
if len(query.Params) != len(want) {
t.Fatalf("params = %+v, want %d entries", query.Params, len(want))
}
for i, tc := range want {
if query.Params[i].Name != tc.name || query.Params[i].Repeated != tc.repeated {
t.Errorf("param %d = (%q, repeated=%v), want (%q, repeated=%v)",
i, query.Params[i].Name, query.Params[i].Repeated, tc.name, tc.repeated)
}
}
if strings.Contains(query.SQL, "?2") {
t.Errorf("SQL = %q, want the numbered placeholders stripped", query.SQL)
}
}

func TestBuildQueriesSqliteKeepsNumberingWithoutSlice(t *testing.T) {
t.Parallel()
// Without a marker every index holds, and sqlc's own numbering is left
// alone: the reused argument stays one bound value.
query := buildSingleQuery(t, &config.Config{SqlDriver: config.SQLDriverSQLite}, &plugin.Query{
Name: "ListAuthors",
Cmd: ":many",
Text: "SELECT id FROM test_authors WHERE name = ?1 AND name != ?1",
Params: []*plugin.Parameter{
{Number: 1, Column: queryCol("n", "text", nil)},
},
})
if len(query.Params) != 1 || query.Params[0].Name != "n" {
t.Fatalf("params = %+v, want a single n", query.Params)
}
if !strings.Contains(query.SQL, "?1") {
t.Errorf("SQL = %q, want sqlc's numbering kept", query.SQL)
}
}

func TestBuildQueriesSqliteBundledFieldsFollowBindOrder(t *testing.T) {
t.Parallel()
// A bundled Params class is expanded field by field, so its field order is
// the bind order: the marker binds first even though sqlc numbered its
// parameter second.
sliceCol := func() *plugin.Column {
column := queryCol("ids", "int4", nil)
column.IsSqlcSlice = true

return column
}
conf := &config.Config{SqlDriver: config.SQLDriverSQLite, QueryParameterLimit: utils.ToPtr(1)}
query := buildSingleQuery(t, conf, &plugin.Query{
Name: "ListAuthors",
Cmd: ":many",
Text: "SELECT id, name FROM test_authors WHERE id IN (/*SLICE:ids*/?) AND name = ?1",
Params: []*plugin.Parameter{
{Number: 1, Column: queryCol("n", "text", nil)},
{Number: 2, Column: sliceCol()},
},
})
if len(query.Params) != 1 || query.Params[0].Table == nil {
t.Fatalf("params = %+v, want a single bundled Params class", query.Params)
}
columns := query.Params[0].Table.Columns
if len(columns) != 2 || columns[0].Name != "ids" || columns[1].Name != "n" {
t.Fatalf("columns = %+v, want ids before n", columns)
}
if strings.Contains(query.SQL, "?1") {
t.Errorf("SQL = %q, want the numbered placeholder stripped", query.SQL)
}
}

func TestBuildQueriesMySQLSliceDedup(t *testing.T) {
t.Parallel()
sliceCol := func() *plugin.Column {
Expand Down
Loading