From e4b678cbe8515a7772652b8240ce3769d0e71de9 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Fri, 9 Jan 2026 11:48:11 -0300 Subject: [PATCH] perf: use wasm-free sqlite on supported platforms the library we are using uses wazero, and will take 70+mb of ram on init. modernc/sqlite is not supported on all platforms we currently ship to, so this duplicates the connect.go file for each driver with the according build tags. Signed-off-by: Carlos Alexandro Becker --- go.mod | 6 ++++++ go.sum | 28 +++++++++++++++++++++---- internal/db/connect.go | 27 +++---------------------- internal/db/connect_modernc.go | 30 +++++++++++++++++++++++++++ internal/db/connect_wasm.go | 37 ++++++++++++++++++++++++++++++++++ 5 files changed, 100 insertions(+), 28 deletions(-) create mode 100644 internal/db/connect_modernc.go create mode 100644 internal/db/connect_wasm.go diff --git a/go.mod b/go.mod index 379038df9c350329e5e1fa38dd5a46e597b07593..8d9e0171f44e6cd4e19992723a8d06791ff672a2 100644 --- a/go.mod +++ b/go.mod @@ -64,6 +64,7 @@ require ( golang.org/x/text v0.32.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 + modernc.org/sqlite v1.43.0 mvdan.cc/sh/moreinterp v0.0.0-20250902163504-3cf4fd5717a5 mvdan.cc/sh/v3 v3.12.1-0.20250902163504-3cf4fd5717a5 ) @@ -141,9 +142,11 @@ require ( github.com/muesli/mango-cobra v1.2.0 // indirect github.com/muesli/mango-pflag v0.1.0 // indirect github.com/muesli/roff v0.1.0 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/ncruces/julianday v1.0.0 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect github.com/sethvargo/go-retry v0.3.0 // indirect github.com/sourcegraph/jsonrpc2 v0.2.1 // indirect @@ -180,4 +183,7 @@ require ( google.golang.org/protobuf v1.36.10 // indirect gopkg.in/dnaeon/go-vcr.v4 v4.0.6-0.20251110073552-01de4eb40290 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + modernc.org/libc v1.66.10 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect ) diff --git a/go.sum b/go.sum index 0a28a008d4ac4ad3f1da125c1b6c599579f34f2e..71b6a2b0b921c031d8a5925bb09c40847a3530d7 100644 --- a/go.sum +++ b/go.sum @@ -184,6 +184,8 @@ github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/jsonschema-go v0.3.0 h1:6AH2TxVNtk3IlvkkhjrtbUc4S8AvO0Xii0DxIygDg+Q= github.com/google/jsonschema-go v0.3.0/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -501,14 +503,32 @@ gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -modernc.org/libc v1.66.3 h1:cfCbjTUcdsKyyZZfEUKfoHcP3S0Wkvz3jgSzByEWVCQ= -modernc.org/libc v1.66.3/go.mod h1:XD9zO8kt59cANKvHPXpx7yS2ELPheAey0vjIuZOhOU8= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= +modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/sqlite v1.38.2 h1:Aclu7+tgjgcQVShZqim41Bbw9Cho0y/7WzYptXqkEek= -modernc.org/sqlite v1.38.2/go.mod h1:cPTJYSlgg3Sfg046yBShXENNtPrWrDX8bsbAQBzgQ5E= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.43.0 h1:8YqiFx3G1VhHTXO2Q00bl1Wz9KhS9Q5okwfp9Y97VnA= +modernc.org/sqlite v1.43.0/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= mvdan.cc/sh/moreinterp v0.0.0-20250902163504-3cf4fd5717a5 h1:mO2lyKtGwu4mGQ+Qqjx0+fd5UU5BXhX/rslFmxd5aco= mvdan.cc/sh/moreinterp v0.0.0-20250902163504-3cf4fd5717a5/go.mod h1:Of9PCedbLDYT8b3EyiYG64rNnx5nOp27OLCVdDrjJyo= mvdan.cc/sh/v3 v3.12.1-0.20250902163504-3cf4fd5717a5 h1:e7Z/Lgw/zMijvQBVrfh/vUDZ+9FpuSLrJDVGBuoJtuo= diff --git a/internal/db/connect.go b/internal/db/connect.go index bfe768c7ae9a399afd61a9d0692841fbacbe164c..20f0c3f31b1506e32ed9d53327d839ac7616bbc9 100644 --- a/internal/db/connect.go +++ b/internal/db/connect.go @@ -7,42 +7,21 @@ import ( "log/slog" "path/filepath" - "github.com/ncruces/go-sqlite3" - "github.com/ncruces/go-sqlite3/driver" - _ "github.com/ncruces/go-sqlite3/embed" - "github.com/pressly/goose/v3" ) +// Connect opens a SQLite database connection and runs migrations. func Connect(ctx context.Context, dataDir string) (*sql.DB, error) { if dataDir == "" { return nil, fmt.Errorf("data.dir is not set") } dbPath := filepath.Join(dataDir, "crush.db") - // Set pragmas for better performance - pragmas := []string{ - "PRAGMA foreign_keys = ON;", - "PRAGMA journal_mode = WAL;", - "PRAGMA page_size = 4096;", - "PRAGMA cache_size = -8000;", - "PRAGMA synchronous = NORMAL;", - "PRAGMA secure_delete = ON;", - } - - db, err := driver.Open(dbPath, func(c *sqlite3.Conn) error { - for _, pragma := range pragmas { - if err := c.Exec(pragma); err != nil { - return fmt.Errorf("failed to set pragma `%s`: %w", pragma, err) - } - } - return nil - }) + db, err := openDB(dbPath) if err != nil { - return nil, fmt.Errorf("failed to open database: %w", err) + return nil, err } - // Verify connection if err = db.PingContext(ctx); err != nil { db.Close() return nil, fmt.Errorf("failed to connect to database: %w", err) diff --git a/internal/db/connect_modernc.go b/internal/db/connect_modernc.go new file mode 100644 index 0000000000000000000000000000000000000000..c80444ba42344943bfbf75308cdc2b01cd38f435 --- /dev/null +++ b/internal/db/connect_modernc.go @@ -0,0 +1,30 @@ +//go:build !openbsd && !netbsd && !android + +package db + +import ( + "database/sql" + "fmt" + "net/url" + + _ "modernc.org/sqlite" +) + +func openDB(dbPath string) (*sql.DB, error) { + // Set pragmas for better performance via _pragma query params. + // Format: _pragma=name(value) + params := url.Values{} + params.Add("_pragma", "foreign_keys(on)") + params.Add("_pragma", "journal_mode(WAL)") + params.Add("_pragma", "page_size(4096)") + params.Add("_pragma", "cache_size(-8000)") + params.Add("_pragma", "synchronous(NORMAL)") + params.Add("_pragma", "secure_delete(on)") + + dsn := fmt.Sprintf("file:%s?%s", dbPath, params.Encode()) + db, err := sql.Open("sqlite", dsn) + if err != nil { + return nil, fmt.Errorf("failed to open database: %w", err) + } + return db, nil +} diff --git a/internal/db/connect_wasm.go b/internal/db/connect_wasm.go new file mode 100644 index 0000000000000000000000000000000000000000..258b783324dac70f73a4f0fe03c7503ba3b29a0f --- /dev/null +++ b/internal/db/connect_wasm.go @@ -0,0 +1,37 @@ +//go:build openbsd || netbsd || android + +package db + +import ( + "database/sql" + "fmt" + + "github.com/ncruces/go-sqlite3" + "github.com/ncruces/go-sqlite3/driver" + _ "github.com/ncruces/go-sqlite3/embed" +) + +func openDB(dbPath string) (*sql.DB, error) { + // Set pragmas for better performance. + pragmas := []string{ + "PRAGMA foreign_keys = ON;", + "PRAGMA journal_mode = WAL;", + "PRAGMA page_size = 4096;", + "PRAGMA cache_size = -8000;", + "PRAGMA synchronous = NORMAL;", + "PRAGMA secure_delete = ON;", + } + + db, err := driver.Open(dbPath, func(c *sqlite3.Conn) error { + for _, pragma := range pragmas { + if err := c.Exec(pragma); err != nil { + return fmt.Errorf("failed to set pragma %q: %w", pragma, err) + } + } + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to open database: %w", err) + } + return db, nil +}