BSD WAL support. (#90)

Uses in-memory locks. Also supports illumos.
ncruces · Jun 4, 2024 · e9ed4c1 · e9ed4c1
1 parent d78a53a
commit e9ed4c1
Show file tree

Hide file tree

Showing 16 changed files with 314 additions and 32 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -131,7 +131,7 @@ jobs:
     - name: Test riscv64 (interpreter)
       run: GOARCH=riscv64 go test -v -short ./...
 
-    - name: Test s390x (big-endian, z/OS)
+    - name: Test s390x (big-endian, z/OS demo)
       run: GOARCH=s390x go test -v -short -tags sqlite3_flock ./...
 
   test-vm:

diff --git a/driver/driver_test.go b/driver/driver_test.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/ncruces/go-sqlite3"
 	_ "github.com/ncruces/go-sqlite3/embed"
+	_ "github.com/ncruces/go-sqlite3/internal/testcfg"
 	"github.com/ncruces/go-sqlite3/internal/util"
 	"github.com/ncruces/go-sqlite3/vfs"
 )

diff --git a/func_test.go b/func_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/ncruces/go-sqlite3"
 	_ "github.com/ncruces/go-sqlite3/embed"
 	"github.com/ncruces/go-sqlite3/ext/unicode"
+	_ "github.com/ncruces/go-sqlite3/internal/testcfg"
 )
 
 func ExampleConn_CreateCollation() {

diff --git a/go.mod b/go.mod
@@ -6,10 +6,10 @@ require (
 	github.com/ncruces/julianday v1.0.0
 	github.com/psanford/httpreadat v0.1.0
 	github.com/tetratelabs/wazero v1.7.2
-	golang.org/x/crypto v0.23.0
+	golang.org/x/crypto v0.24.0
 	golang.org/x/sync v0.7.0
-	golang.org/x/sys v0.20.0
-	golang.org/x/text v0.15.0
+	golang.org/x/sys v0.21.0
+	golang.org/x/text v0.16.0
 	lukechampine.com/adiantum v1.1.1
 )
 

diff --git a/go.sum b/go.sum
@@ -6,13 +6,13 @@ github.com/psanford/httpreadat v0.1.0 h1:VleW1HS2zO7/4c7c7zNl33fO6oYACSagjJIyMIw
 github.com/psanford/httpreadat v0.1.0/go.mod h1:Zg7P+TlBm3bYbyHTKv/EdtSJZn3qwbPwpfZ/I9GKCRE=
 github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
 github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
-golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
-golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
+golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI=
+golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM=
 golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M=
 golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
-golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4=
+golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
 lukechampine.com/adiantum v1.1.1 h1:4fp6gTxWCqpEbLy40ExiYDDED3oUNWx5cTqBCtPdZqA=
 lukechampine.com/adiantum v1.1.1/go.mod h1:LrAYVnTYLnUtE/yMp5bQr0HstAf060YUF8nM0B6+rUw=
diff --git a/go.work.sum b/go.work.sum
@@ -1,4 +1,7 @@
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
+golang.org/x/term v0.21.0/go.mod h1:ooXLefLobQVslOqselCNF4SxFAaoS6KujMbsGzSDmX0=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
+golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
diff --git a/gormlite/go.mod b/gormlite/go.mod
@@ -12,5 +12,5 @@ require (
 	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/ncruces/julianday v1.0.0 // indirect
 	github.com/tetratelabs/wazero v1.7.2 // indirect
-	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/sys v0.21.0 // indirect
 )
diff --git a/gormlite/go.sum b/gormlite/go.sum
@@ -8,8 +8,8 @@ github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt
 github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
 github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
 github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
-golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
-golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
+golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
 golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 gorm.io/gorm v1.25.10 h1:dQpO+33KalOA+aFYGlK+EfxcI5MbO7EP2yYygwh9h+s=

diff --git a/internal/testcfg/testcfg.go b/internal/testcfg/testcfg.go
@@ -1,6 +1,7 @@
 package testcfg
 
 import (
+	"math/bits"
 	"os"
 	"path/filepath"
 
@@ -9,6 +10,10 @@ import (
 )
 
 func init() {
+	if bits.UintSize < 64 {
+		return
+	}
+
 	sqlite3.RuntimeConfig = wazero.NewRuntimeConfig().
 		WithMemoryCapacityFromMax(true).
 		WithMemoryLimitPages(1024)

diff --git a/internal/util/mmap.go b/internal/util/mmap.go
@@ -1,4 +1,4 @@
-//go:build (darwin || linux) && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
+//go:build unix && (amd64 || arm64 || riscv64) && !(sqlite3_noshm || sqlite3_nosys)
 
 package util
 

diff --git a/internal/util/mmap_other.go b/internal/util/mmap_other.go
@@ -1,4 +1,4 @@
-//go:build !(darwin || linux) || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
+//go:build !unix || !(amd64 || arm64 || riscv64) || sqlite3_noshm || sqlite3_nosys
 
 package util
 

diff --git a/tests/stmt_test.go b/tests/stmt_test.go
@@ -3,6 +3,7 @@ package tests
 import (
 	"encoding/json"
 	"math"
+	"math/bits"
 	"testing"
 	"time"
 
@@ -617,6 +618,9 @@ func TestStmt_ColumnTime(t *testing.T) {
 }
 
 func TestStmt_Error(t *testing.T) {
+	if bits.UintSize < 64 {
+		t.Skip("skipping on 32-bit")
+	}
 	t.Parallel()
 
 	db, err := sqlite3.Open(":memory:")

diff --git a/vfs/README.md b/vfs/README.md
@@ -46,7 +46,7 @@ to check if your build supports file locking.
 
 ### Write-Ahead Logging
 
-On 64-bit Linux and macOS, this module uses `mmap` to implement
+On 64-bit Unix, this module uses `mmap` to implement
 [shared-memory for the WAL-index](https://sqlite.org/wal.html#implementation_of_shared_memory_for_the_wal_index),
 like SQLite.
 

diff --git a/vfs/shm.go b/vfs/shm.go
@@ -51,12 +51,7 @@ type vfsShm struct {
 	readOnly bool
 }
 
-func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
-	// Ensure size is a multiple of the OS page size.
-	if int(size)&(unix.Getpagesize()-1) != 0 {
-		return 0, _IOERR_SHMMAP
-	}
-
+func (s *vfsShm) shmOpen() error {
 	if s.File == nil {
 		var flag int
 		if s.readOnly {
@@ -67,29 +62,41 @@ func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, ext
 		f, err := os.OpenFile(s.path,
 			flag|unix.O_CREAT|unix.O_NOFOLLOW, 0666)
 		if err != nil {
-			return 0, _CANTOPEN
+			return _CANTOPEN
 		}
 		s.File = f
 	}
 
 	// Dead man's switch.
 	if lock, rc := osGetLock(s.File, _SHM_DMS, 1); rc != _OK {
-		return 0, _IOERR_LOCK
+		return _IOERR_LOCK
 	} else if lock == unix.F_WRLCK {
-		return 0, _BUSY
+		return _BUSY
 	} else if lock == unix.F_UNLCK {
 		if s.readOnly {
-			return 0, _READONLY_CANTINIT
+			return _READONLY_CANTINIT
 		}
 		if rc := osWriteLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
-			return 0, rc
+			return rc
 		}
 		if err := s.Truncate(0); err != nil {
-			return 0, _IOERR_SHMOPEN
+			return _IOERR_SHMOPEN
 		}
 	}
 	if rc := osReadLock(s.File, _SHM_DMS, 1, 0); rc != _OK {
-		return 0, rc
+		return rc
+	}
+	return nil
+}
+
+func (s *vfsShm) shmMap(ctx context.Context, mod api.Module, id, size int32, extend bool) (uint32, error) {
+	// Ensure size is a multiple of the OS page size.
+	if int(size)&(unix.Getpagesize()-1) != 0 {
+		return 0, _IOERR_SHMMAP
+	}
+
+	if err := s.shmOpen(); err != nil {
+		return 0, err
 	}
 
 	// Check if file is big enough.
@@ -165,9 +172,9 @@ func (s *vfsShm) shmUnmap(delete bool) {
 	s.regions = s.regions[:0]
 
 	// Close the file.
-	defer s.Close()
 	if delete {
-		os.Remove(s.Name())
+		os.Remove(s.path)
 	}
+	s.Close()
 	s.File = nil
 }