ext/crr: update to 0.15.1 of cr-sqlite

Based on https://github.com/tursodatabase/libsql/pull/434
and rebased.

libsql-sqlite3/ext/crr/.gitignore

@@ -2,3 +2,6 @@ node_modules/
 target/
 dist/
 dbg/
+dist-ios/
+dist-ios-sim/
+dist-macos/

libsql-sqlite3/ext/crr/CHANGELOG.md

@@ -1,5 +1,121 @@
 # @vlcn.io/crsqlite
 
+## 0.15.1-next.0
+
+### Patch Changes
+
+- ensure statements are finalized when closing db, allow automigrating fractindex tables, fractindex w/o list columns fix
+
+## 0.15.0
+
+### Minor Changes
+
+- 56df096: re-insertion, api naming consistencies, metadata size reduction, websocket server, websocket client, websocket demo
+
+### Patch Changes
+
+- 4022bd6: litefs support
+- 08f13fb: react strict mode fiex, migrator fixes, typed-sql basic support, ws replication, db provider hooks
+- f327068: rebuild
+
+## 0.15.0-next.2
+
+### Patch Changes
+
+- litefs support
+
+## 0.15.0-next.1
+
+### Patch Changes
+
+- react strict mode fiex, migrator fixes, typed-sql basic support, ws replication, db provider hooks
+
+## 0.15.0-next.0
+
+### Minor Changes
+
+- re-insertion, api naming consistencies, metadata size reduction, websocket server, websocket client, websocket demo
+
+## 0.14.0
+
+### Minor Changes
+
+- 68deb1c: binary encoded primary keys, no string encoding on values, cache prepared statements on merge, fix webkit JIT crash
+
+## 0.14.0-next.0
+
+### Minor Changes
+
+- binary encoded primary keys, no string encoding on values, cache prepared statements on merge, fix webkit JIT crash
+
+## 0.13.0
+
+### Minor Changes
+
+- 62912ad: split up large transactions, compact out unneeded delete records, coordinate dedicated workers for android, null merge fix
+
+## 0.13.0-next.0
+
+### Minor Changes
+
+- split up large transactions, compact out unneeded delete records, coordinate dedicated workers for android, null merge fix
+
+## 0.12.0
+
+### Minor Changes
+
+- 7885afd: 50x perf boost when pulling changesets
+
+## 0.12.0-next.0
+
+### Minor Changes
+
+- 15c8e04: 50x perf boost when pulling changesets
+
+## 0.11.0
+
+### Minor Changes
+
+- automigrate fixes for WASM, react fixes for referential equality, direct-connect networking implementations, sync in shared worker, dbProvider hooks for React
+
+### Patch Changes
+
+- 4e737a0: better error reporting on migration failure, handle schema swap
+
+## 0.10.2-next.0
+
+### Patch Changes
+
+- better error reporting on migration failure, handle schema swap
+
+## 0.10.1
+
+### Patch Changes
+
+- fts5, sqlite 3.42.1, direct-connect packages
+
+## 0.10.0
+
+### Minor Changes
+
+- e0de95c: ANSI SQL compliance for crsql_changes, all filters available for crsql_changes, removal of tracked_peers, simplified crsql_master table
+
+### Patch Changes
+
+- 9b483aa: npm is not updating on package publish -- bump versions to try to force it
+
+## 0.10.0-next.1
+
+### Patch Changes
+
+- npm is not updating on package publish -- bump versions to try to force it
+
+## 0.10.0-next.0
+
+### Minor Changes
+
+- ANSI SQL compliance for crsql_changes, all filters available for crsql_changes, removal of tracked_peers, simplified crsql_master table
+
 ## 0.9.3
 
 ### Patch Changes

libsql-sqlite3/ext/crr/Makefile

@@ -5,6 +5,7 @@ CC:=$(CI_GCC)
 endif
 
 LOADABLE_CFLAGS=-std=c99 -fPIC -shared -Wall
+STATIC_CFLAGS=-std=c99 -fPIC -c -Wall
 
 ifeq ($(shell uname -s),Darwin)
 CONFIG_DARWIN=y
@@ -16,16 +17,43 @@ endif
 
 ifdef CONFIG_DARWIN
 LOADABLE_EXTENSION=dylib
+# apparently `darwin-x86_64` also works on arm macs and is the proper host arch for ndk builds.
+NDK_HOSTARCH=darwin-x86_64
 endif
 
 ifdef CONFIG_LINUX
 LOADABLE_EXTENSION=so
+NDK_HOSTARCH=linux-x86_64
 endif
 
 ifdef CONFIG_WINDOWS
 LOADABLE_EXTENSION=dll
 endif
 
+ifdef IOS_TARGET
+CI_MAYBE_TARGET=$(IOS_TARGET)
+rs_build_flags = -Zbuild-std
+	ifeq ($(or $(findstring sim,$(CI_MAYBE_TARGET)),$(findstring x86_64,$(CI_MAYBE_TARGET))),)
+			# todo: run the xcode command to find this
+			sysroot_option = -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk
+		else
+			sysroot_option = -isysroot /Applications/Xcode.app/Contents/Developer/Platforms/iPhoneSimulator.platform/Developer/SDKs/iPhoneSimulator.sdk
+		endif
+endif
+
+# aarch64-linux-android
+# https://github.com/marketplace/actions/setup-android-ndk
+ifdef ANDROID_TARGET
+CI_MAYBE_TARGET=$(ANDROID_TARGET)
+NDK=$(ANDROID_NDK_HOME)
+LOADABLE_EXTENSION=so
+CC=$(NDK)/toolchains/llvm/prebuilt/$(NDK_HOSTARCH)/bin/clang
+rs_ndk=ndk -t $(ANDROID_TARGET)
+ANDROID_API_VERSION=33
+rs_build_flags=-Zbuild-std
+sysroot_option=--sysroot=$(NDK)/toolchains/llvm/prebuilt/$(NDK_HOSTARCH)/sysroot
+endif
+
 prefix=./dist
 dbg_prefix=./dbg
 
@@ -34,6 +62,7 @@ TARGET_DBG_LOADABLE=$(dbg_prefix)/crsqlite.$(LOADABLE_EXTENSION)
 TARGET_SQLITE3_EXTRA_C=$(prefix)/sqlite3-extra.c
 TARGET_SQLITE3=$(prefix)/sqlite3
 TARGET_SQLITE3_VANILLA=$(prefix)/vanilla-sqlite3
+TARGET_STATIC=$(prefix)/crsqlite.a
 TARGET_TEST=$(prefix)/test
 TARGET_FUZZ=$(prefix)/fuzz
 TARGET_TEST_ASAN=$(prefix)/test-asan
@@ -43,21 +72,13 @@ TARGET_TEST_ASAN=$(prefix)/test-asan
 ext_files=src/crsqlite.c \
 	src/util.c \
 	src/tableinfo.c \
-	src/triggers.c \
 	src/changes-vtab.c \
-	src/changes-vtab-read.c \
-	src/changes-vtab-common.c \
-	src/changes-vtab-write.c \
 	src/ext-data.c \
 	src/get-table.c
 ext_headers=src/crsqlite.h \
 	src/util.h \
 	src/tableinfo.h \
-	src/triggers.h \
 	src/changes-vtab.h \
-	src/changes-vtab-read.h \
-	src/changes-vtab-common.h \
-	src/changes-vtab-write.h \
 	src/ext-data.h
 
 $(prefix):
@@ -79,6 +100,7 @@ loadable: $(TARGET_LOADABLE)
 loadable_dbg: $(TARGET_DBG_LOADABLE)
 sqlite3: $(TARGET_SQLITE3)
 vanilla: $(TARGET_SQLITE3_VANILLA)
+static: $(TARGET_STATIC)
 test: $(TARGET_TEST)
 	$(prefix)/test
 # ASAN_OPTIONS=detect_leaks=1
@@ -99,8 +121,11 @@ fuzz: $(TARGET_FUZZ)
 	$(prefix)/fuzz
 
 sqlite_src = ../../
-shell.c = $(sqlite_src)/shell.c
-sqlite3.c = $(sqlite_src)/sqlite3.c
+shell.c = $(sqlite_src)shell.c
+sqlite3.c = $(sqlite_src)sqlite3.c
+
+rs_lib_static = ./rs/bundle/target/release/libcrsql_bundle.a
+rs_lib_static_cpy = ./dist/libcrsql_bundle-static.a
 
 rs_lib_dbg_static = ./rs/bundle/target/debug/libcrsql_bundle.a
 rs_lib_dbg_static_cpy = ./dbg/libcrsql_bundle-dbg-static.a
@@ -115,41 +140,53 @@ ifdef CI_MAYBE_TARGET
 	rs_lib_dbg_static = ./rs/bundle/target/$(CI_MAYBE_TARGET)/debug/libcrsql_bundle.a
 	rs_lib_loadable = ./rs/bundle/target/$(CI_MAYBE_TARGET)/release/libcrsql_bundle.a
 	rs_lib_dbg_loadable = ./rs/bundle/target/$(CI_MAYBE_TARGET)/debug/libcrsql_bundle.a
+	rs_lib_static = ./rs/bundle/target/$(CI_MAYBE_TARGET)/release/libcrsql_bundle.a
 	RS_TARGET = --target=$(CI_MAYBE_TARGET)
 	ifndef CI_GCC
-		C_TARGET = -target $(CI_MAYBE_TARGET)
+# clang has a different target triple than Rust for ios simuators
+		ifeq ($(findstring sim,$(CI_MAYBE_TARGET)),)
+			C_TARGET = --target=$(CI_MAYBE_TARGET)$(ANDROID_API_VERSION)
+		else
+			C_TARGET = --target=$(CI_MAYBE_TARGET)ulator
+		endif
 	endif
 endif
 
-$(rs_lib_dbg_static_cpy): FORCE $(dbg_prefix) $(sqlite3.c)
-	cd ./rs/bundle && $(rustflags_static) cargo build $(RS_TARGET) --features static,omit_load_extension $(rs_build_flags)
-	cp $(rs_lib_dbg_static) $(rs_lib_dbg_static_cpy)
-
-$(rs_lib_loadable_cpy): FORCE $(dbg_prefix) $(sqlite3.c)
-	cd ./rs/bundle && $(rustflags_static) cargo build $(RS_TARGET) --release --features loadable_extension $(rs_build_flags)
-	cp $(rs_lib_loadable) $(rs_lib_loadable_cpy)
-
-# we need separate output dirs based on selected features of the build
-$(rs_lib_dbg_loadable_cpy): FORCE $(dbg_prefix) $(sqlite3.c)
-	cd ./rs/bundle && $(rustflags_static) cargo build $(RS_TARGET) --features loadable_extension $(rs_build_flags)
-	cp $(rs_lib_dbg_loadable) $(rs_lib_dbg_loadable_cpy)
-
 $(shell.c):
 	cd $(sqlite_src) && make shell.c
 
 $(sqlite3.c):
 	cd $(sqlite_src) && make sqlite3.c
 
+$(rs_lib_dbg_static_cpy): FORCE $(dbg_prefix)
+	cd ./rs/bundle && cargo build $(RS_TARGET) --features static,omit_load_extension $(rs_build_flags)
+	cp $(rs_lib_dbg_static) $(rs_lib_dbg_static_cpy)
+
+$(rs_lib_static_cpy): FORCE $(prefix)
+	cd ./rs/bundle && cargo build $(RS_TARGET) --release --features static,omit_load_extension $(rs_build_flags)
+	cp $(rs_lib_static) $(rs_lib_static_cpy)
+
+$(rs_lib_loadable_cpy): FORCE $(prefix)
+	cd ./rs/bundle && cargo $(rs_ndk) build $(RS_TARGET) --release --features loadable_extension $(rs_build_flags)
+	cp $(rs_lib_loadable) $(rs_lib_loadable_cpy)
+
+$(rs_lib_dbg_loadable_cpy): FORCE $(dbg_prefix)
+	cd ./rs/bundle && cargo build $(RS_TARGET) --features loadable_extension $(rs_build_flags)
+	cp $(rs_lib_dbg_loadable) $(rs_lib_dbg_loadable_cpy)
+
 # Build the loadable extension.
-$(TARGET_LOADABLE): $(prefix) $(ext_files) $(rs_lib_loadable_cpy)
+$(TARGET_LOADABLE): $(prefix) $(ext_files) $(sqlite3.c) $(rs_lib_loadable_cpy)
 	$(CC) -O2 -I./src/ -I$(sqlite_src) \
 	$(LOADABLE_CFLAGS) \
 	$(C_TARGET) \
+	$(sysroot_option) \
 	$(ext_files) $(rs_lib_loadable_cpy) -o $@
 
-$(TARGET_DBG_LOADABLE): $(dbg_prefix) $(ext_files) $(rs_lib_dbg_loadable_cpy)
+$(TARGET_DBG_LOADABLE): $(dbg_prefix) $(ext_files) $(sqlite3.c) $(rs_lib_dbg_loadable_cpy)
 	$(CC) -g -I./src/ -I$(sqlite_src) \
 	$(LOADABLE_CFLAGS) \
+	$(C_TARGET) \
+	$(sysroot_option) \
 	$(ext_files) $(rs_lib_dbg_loadable_cpy) -o $@
 
 # Build a SQLite CLI that pre-loads cr-sqlite.
@@ -159,14 +196,29 @@ $(TARGET_SQLITE3): $(prefix) $(TARGET_SQLITE3_EXTRA_C) $(rs_lib_dbg_static_cpy)
 	-DSQLITE_THREADSAFE=0 \
 	-DSQLITE_OMIT_LOAD_EXTENSION=1 \
 	-DSQLITE_EXTRA_INIT=core_init \
+	-DSQLITE_ENABLE_BYTECODE_VTAB \
 	-I./src/ -I$(sqlite_src) \
 	$(TARGET_SQLITE3_EXTRA_C) $(shell.c) $(ext_files) $(rs_lib_dbg_static_cpy) \
 	$(LDLIBS) -o $@
 
+# Build the SQLite library w/ cr-sqlite statically linked in
+$(TARGET_STATIC): $(prefix) $(ext_files) $(sqlite3.c) $(rs_lib_static_cpy)
+	$(CC) -g \
+	-DHAVE_GETHOSTUUID=0 \
+	-I./src/ -I$(sqlite_src) \
+	$(STATIC_CFLAGS) \
+	$(C_TARGET) \
+	$(sysroot_option) \
+	$(ext_files)
+	mkdir -p $(prefix)/temp
+	rm -f $(prefix)/temp/*
+	mv *.o $(prefix)/temp
+	cd $(prefix)/temp && ar -x ../libcrsql_bundle-static.a && ar -rc crsqlite.a *.o && mv crsqlite.a ../crsqlite-$(CI_MAYBE_TARGET).a
+
 # Build a normal SQLite CLI that does not include cr-sqlite.
 # cr-sqlite can be laoded in via the `.load` pragma.
 # Useful for debugging.
-$(TARGET_SQLITE3_VANILLA): $(prefix) $(shell.c) $(sqlite3.c)
+$(TARGET_SQLITE3_VANILLA): $(prefix) $(sqlite3.c) $(shell.c)
 	$(CC) -g \
 	$(DEFINE_SQLITE_PATH) \
 	-DSQLITE_THREADSAFE=0 \
@@ -214,6 +266,6 @@ $(TARGET_FUZZ): $(prefix) $(TARGET_SQLITE3_EXTRA_C) src/fuzzer.cc $(ext_files)
 	sqlite3 \
 	correctness \
 	valgrind \
-	ubsan analyzer fuzz asan
+	ubsan analyzer fuzz asan static
 
-FORCE: ;
+FORCE: ;

libsql-sqlite3/ext/crr/README.md

@@ -49,15 +49,15 @@ The full documentation site is available [here](https://vlcn.io/docs/getting-sta
   - `SELECT crsql_as_crr('table_name')`
 - A virtual table (`crsql_changes`) to ask the database for changesets or to apply changesets from another database
   - `SELECT * FROM crsql_changes WHERE db_version > x AND site_id IS NULL` -- to get local changes
-  - `SELECT * FROM crsql_changes WHERE db_version > x AND site_id != some_site` -- to get all changes excluding those synced from some site
-  - `INSERT INTO crsql_changes VALUES ([patches receied from select on another peer])`
-- And `crsql_alter_begin('table_name')` & `crsql_alter_commit('table_name')` primitives to allow altering table definitions that have been upgraded to `crr`s.
+  - `SELECT * FROM crsql_changes WHERE db_version > x AND site_id IS NOT some_site` -- to get all changes excluding those synced from some site
+  - `INSERT INTO crsql_changes VALUES ([patches received from select on another peer])`
+- And `crsql_begin_alter('table_name')` & `crsql_alter_commit('table_name')` primitives to allow altering table definitions that have been upgraded to `crr`s.
   - Until we move forward with extending the syntax of SQLite to be CRR aware, altering CRRs looks like:
     ```sql
-    SELECT crsql_alter_begin('table_name');
+    SELECT crsql_begin_alter('table_name');
     -- 1 or more alterations to `table_name`
     ALTER TABLE table_name ...;
-    SELECT crsql_alter_commit('table_name');
+    SELECT crsql_commit_alter('table_name');
     ```
     A future version of cr-sqlite may extend the SQL syntax to make this more natural.
 

libsql-sqlite3/ext/crr/all-ios-loadable.sh

@@ -0,0 +1,77 @@
+#! /bin/bash
+
+# a hacky script to make all the various ios targets.
+# once we have something consistently working we'll streamline all of this.
+
+BUILD_DIR=./build
+DIST_PACKAGE_DIR=./dist
+
+function createXcframework() {
+  plist=$(cat << EOF
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>CFBundleDevelopmentRegion</key>
+  <string>en</string>
+  <key>CFBundleExecutable</key>
+  <string>crsqlite</string>
+  <key>CFBundleIdentifier</key>
+  <string>io.vlcn.crsqlite</string>
+  <key>CFBundleInfoDictionaryVersion</key>
+  <string>6.0</string>
+  <key>CFBundlePackageType</key>
+  <string>FMWK</string>
+  <key>CFBundleSignature</key>
+  <string>????</string>
+</dict>
+</plist>
+EOF
+)
+  printf "\n\n\t\t===================== create ios device framework =====================\n\n"
+  mkdir -p "${BUILD_DIR}/ios-arm64/crsqlite.framework"
+  echo "${plist}" > "${BUILD_DIR}/ios-arm64/crsqlite.framework/Info.plist"
+  cp -f "./dist-ios/crsqlite-aarch64-apple-ios.dylib" "${BUILD_DIR}/ios-arm64/crsqlite.framework/crsqlite"
+  install_name_tool -id "@rpath/crsqlite.framework/crsqlite" "${BUILD_DIR}/ios-arm64/crsqlite.framework/crsqlite"
+
+  printf "\n\n\t\t===================== create ios simulator framework =====================\n\n"
+  mkdir -p "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework"
+  echo "${plist}" > "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework/Info.plist"
+  cp -p "./dist-ios-sim/crsqlite-universal-ios-sim.dylib" "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework/crsqlite"
+  install_name_tool -id "@rpath/crsqlite.framework/crsqlite" "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework/crsqlite"
+
+  printf "\n\n\t\t===================== create ios xcframework =====================\n\n"
+  rm -rf "${BUILD_DIR}/crsqlite.xcframework"
+  xcodebuild -create-xcframework -framework "${BUILD_DIR}/ios-arm64/crsqlite.framework" -framework "${BUILD_DIR}/ios-arm64_x86_64-simulator/crsqlite.framework" -output "${BUILD_DIR}/crsqlite.xcframework"
+
+  mkdir -p ${DIST_PACKAGE_DIR}
+  cp -Rf "${BUILD_DIR}/crsqlite.xcframework" "${DIST_PACKAGE_DIR}/crsqlite.xcframework"
+  cd ${DIST_PACKAGE_DIR}
+  tar -czvf crsqlite-ios-dylib.xcframework.tar.gz crsqlite.xcframework
+  rm -rf ${BUILD_DIR}
+}
+
+# Make all the non-simulator libs
+# Package into a universal ios lib
+mkdir -p ./dist-ios
+
+# TODO: fix things up to not require a clean before each target.
+make clean
+export IOS_TARGET=aarch64-apple-ios; make loadable
+cp ./dist/crsqlite.dylib ./dist-ios/crsqlite-aarch64-apple-ios.dylib
+
+mkdir -p ./dist-ios-sim
+
+make clean
+export IOS_TARGET=aarch64-apple-ios-sim; make loadable
+cp ./dist/crsqlite.dylib ./dist-ios-sim/crsqlite-aarch64-apple-ios-sim.dylib
+
+make clean
+export IOS_TARGET=x86_64-apple-ios; make loadable
+cp ./dist/crsqlite.dylib ./dist-ios-sim/crsqlite-x86_64-apple-ios-sim.dylib
+
+cd ./dist-ios-sim
+lipo crsqlite-aarch64-apple-ios-sim.dylib crsqlite-x86_64-apple-ios-sim.dylib -create -output crsqlite-universal-ios-sim.dylib
+cd ..
+
+createXcframework

libsql-sqlite3/ext/crr/all-ios-static.sh

@@ -0,0 +1,54 @@
+#! /bin/bash
+
+# a hacky script to make all the various ios targets.
+# once we have something consistently working we'll streamline all of this.
+
+# Make all the non-simulator libs
+# Package into a universal ios lib
+mkdir -p ./dist-ios
+
+# TODO: fix things up to not require a clean before each target.
+make clean
+export IOS_TARGET=aarch64-apple-ios; make static
+cp ./dist/crsqlite-aarch64-apple-ios.a ./dist-ios
+
+make clean
+export IOS_TARGET=armv7-apple-ios; make static
+cp ./dist/crsqlite-armv7-apple-ios.a ./dist-ios
+
+make clean
+export IOS_TARGET=armv7s-apple-ios; make static
+cp ./dist/crsqlite-armv7s-apple-ios.a ./dist-ios
+
+cd ./dist-ios
+lipo crsqlite-aarch64-apple-ios.a crsqlite-armv7-apple-ios.a crsqlite-armv7s-apple-ios.a -create -output crsqlite-universal-ios.a
+
+cd ..
+# ===
+
+# Make the simlator libs
+# Package into a universal ios sim lib
+mkdir -p ./dist-ios-sim
+
+make clean
+export IOS_TARGET=aarch64-apple-ios-sim; make static
+cp ./dist/crsqlite-aarch64-apple-ios-sim.a ./dist-ios-sim
+
+make clean
+export IOS_TARGET=x86_64-apple-ios; make static
+cp ./dist/crsqlite-x86_64-apple-ios.a ./dist-ios-sim
+
+cd ./dist-ios-sim
+lipo crsqlite-aarch64-apple-ios-sim.a crsqlite-x86_64-apple-ios.a -create -output crsqlite-universal-ios-sim.a
+
+cd ..
+# ===
+
+# Make the macos static lib
+mkdir -p ./dist-macos
+make clean
+unset IOS_TARGET
+export CI_MAYBE_TARGET="aarch64-apple-darwin"; make static
+
+cp ./dist/crsqlite-aarch64-apple-darwin.a ./dist-macos
+

libsql-sqlite3/ext/crr/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vlcn.io/crsqlite",
-  "version": "0.9.3",
+  "version": "0.15.1-next.0",
   "description": "CR-SQLite loadable extension",
   "homepage": "https://vlcn.io",
   "repository": {

libsql-sqlite3/ext/crr/rs/bundle/Cargo.lock

@@ -36,6 +36,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bytes"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+
 [[package]]
 name = "cexpr"
 version = "0.6.0"
@@ -75,6 +81,9 @@ dependencies = [
 name = "crsql_core"
 version = "0.1.0"
 dependencies = [
+ "bytes",
+ "num-derive",
+ "num-traits",
  "sqlite_nostd",
 ]
 

libsql-sqlite3/ext/crr/rs/bundle/rust-toolchain.toml

@@ -0,0 +1,3 @@
+[toolchain]
+channel = "nightly-2023-06-17"
+components = [ "rust-src", "rustfmt", "clippy" ]

libsql-sqlite3/ext/crr/rs/bundle/src/lib.rs

@@ -1,6 +1,5 @@
 #![no_std]
 #![feature(core_intrinsics)]
-#![feature(alloc_error_handler)]
 #![feature(lang_items)]
 
 extern crate alloc;
@@ -15,58 +14,22 @@ use crsql_fractindex_core::sqlite3_crsqlfractionalindex_init;
 use sqlite_nostd as sqlite;
 use sqlite_nostd::SQLite3Allocator;
 
+// This must be our allocator so we can transfer ownership of memory to SQLite and have SQLite free that memory for us.
+// This drastically reduces copies when passing strings and blobs back and forth between Rust and C.
 #[global_allocator]
 static ALLOCATOR: SQLite3Allocator = SQLite3Allocator {};
 
+// This must be our panic handler for WASM builds. For simplicity, we make it our panic handler for
+// all builds. Abort is also more portable than unwind, enabling us to go to more embedded use cases.
 #[panic_handler]
 fn panic(_info: &PanicInfo) -> ! {
     core::intrinsics::abort()
 }
 
-#[alloc_error_handler]
-fn oom(_: Layout) -> ! {
-    core::intrinsics::abort()
-}
-
 #[cfg(not(target_family = "wasm"))]
 #[lang = "eh_personality"]
 extern "C" fn eh_personality() {}
 
-#[cfg(target_family = "wasm")]
-#[no_mangle]
-pub extern "C" fn __rust_alloc(size: usize, align: usize) -> *mut u8 {
-    unsafe { ALLOCATOR.alloc(Layout::from_size_align_unchecked(size, align)) }
-}
-
-#[cfg(target_family = "wasm")]
-#[no_mangle]
-pub extern "C" fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize) {
-    unsafe { ALLOCATOR.dealloc(ptr, Layout::from_size_align_unchecked(size, align)) }
-}
-
-#[cfg(target_family = "wasm")]
-#[no_mangle]
-pub extern "C" fn __rust_realloc(
-    ptr: *mut u8,
-    old_size: usize,
-    align: usize,
-    size: usize,
-) -> *mut u8 {
-    unsafe {
-        ALLOCATOR.realloc(
-            ptr,
-            Layout::from_size_align_unchecked(old_size, align),
-            size,
-        )
-    }
-}
-
-#[cfg(target_family = "wasm")]
-#[no_mangle]
-pub extern "C" fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8 {
-    unsafe { ALLOCATOR.alloc_zeroed(Layout::from_size_align_unchecked(size, align)) }
-}
-
 #[cfg(target_family = "wasm")]
 #[no_mangle]
 pub fn __rust_alloc_error_handler(_: Layout) -> ! {

libsql-sqlite3/ext/crr/rs/core/Cargo.lock

@@ -36,6 +36,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bytes"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+
 [[package]]
 name = "cexpr"
 version = "0.6.0"
@@ -66,6 +72,9 @@ dependencies = [
 name = "crsql_core"
 version = "0.1.0"
 dependencies = [
+ "bytes",
+ "num-derive",
+ "num-traits",
  "sqlite_nostd",
 ]
 

libsql-sqlite3/ext/crr/rs/core/Cargo.toml

@@ -12,6 +12,9 @@ crate-type = ["rlib"]
 
 [dependencies]
 sqlite_nostd = { path="../sqlite-rs-embedded/sqlite_nostd" }
+bytes = { version = "1.4", default-features = false }
+num-traits = { version = "0.2.15", default-features = false }
+num-derive = "0.3"
 
 [dev-dependencies]
 

libsql-sqlite3/ext/crr/rs/core/rust-toolchain.toml

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "nightly-2023-06-17"

libsql-sqlite3/ext/crr/rs/core/src/alter.rs

@@ -0,0 +1,136 @@
+// Not yet fully migrated from `crsqlite.c`
+
+use core::ffi::{c_char, c_int, CStr};
+
+use alloc::format;
+use alloc::string::String;
+use core::slice;
+#[cfg(not(feature = "std"))]
+use num_traits::FromPrimitive;
+use sqlite_nostd as sqlite;
+use sqlite_nostd::{sqlite3, Connection, ResultCode};
+
+use crate::c::{
+    crsql_ExtData, crsql_ensureTableInfosAreUpToDate, crsql_findTableInfo, crsql_getDbVersion,
+};
+
+#[no_mangle]
+pub unsafe extern "C" fn crsql_compact_post_alter(
+    db: *mut sqlite3,
+    tbl_name: *const c_char,
+    ext_data: *mut crsql_ExtData,
+    errmsg: *mut *mut c_char,
+) -> c_int {
+    match compact_post_alter(db, tbl_name, ext_data, errmsg) {
+        Ok(rc) | Err(rc) => rc as c_int,
+    }
+}
+
+unsafe fn compact_post_alter(
+    db: *mut sqlite3,
+    tbl_name: *const c_char,
+    ext_data: *mut crsql_ExtData,
+    errmsg: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    let tbl_name_str = CStr::from_ptr(tbl_name).to_str()?;
+    let c_rc = crsql_getDbVersion(db, ext_data, errmsg);
+    if c_rc != ResultCode::OK as c_int {
+        if let Some(rc) = ResultCode::from_i32(c_rc) {
+            return Err(rc);
+        }
+        return Err(ResultCode::ERROR);
+    }
+    let current_db_version = (*ext_data).dbVersion;
+
+    // If primary key columns change (in the schema)
+    // We need to drop, re-create and backfill
+    // the clock table.
+    // A change in pk columns means a change in all identities
+    // of all rows.
+    // We can determine this by comparing pks on clock table vs
+    // pks on source table
+    let stmt = db.prepare_v2(&format!(
+        "SELECT count(name) FROM (
+        SELECT name FROM pragma_table_info('{table_name}')
+          WHERE pk > 0 AND name NOT IN
+            (SELECT name FROM pragma_table_info('{table_name}__crsql_clock') WHERE pk > 0)
+          UNION SELECT name FROM pragma_table_info('{table_name}__crsql_clock') WHERE pk > 0 AND name NOT IN 
+            (SELECT name FROM pragma_table_info('{table_name}') WHERE pk > 0) AND name != '__crsql_col_name'
+        );",
+        table_name = crate::util::escape_ident_as_value(tbl_name_str),
+    ))?;
+    stmt.step()?;
+
+    let pk_diff = stmt.column_int(0)?;
+    // immediately drop stmt, otherwise clock table is considered locked.
+    drop(stmt);
+
+    if pk_diff > 0 {
+        // drop the clock table so we can re-create it
+        db.exec_safe(&format!(
+            "DROP TABLE \"{table_name}__crsql_clock\"",
+            table_name = crate::util::escape_ident(tbl_name_str),
+        ))?;
+    } else {
+        // clock table is still relevant but needs compacting
+        // in case columns were removed during the migration
+
+        // First delete entries that no longer have a column
+        let sql = format!(
+            "DELETE FROM \"{tbl_name_ident}__crsql_clock\" WHERE \"__crsql_col_name\" NOT IN (
+              SELECT name FROM pragma_table_info('{tbl_name_val}') UNION SELECT '{cl_sentinel}'
+            )",
+            tbl_name_ident = crate::util::escape_ident(tbl_name_str),
+            tbl_name_val = crate::util::escape_ident_as_value(tbl_name_str),
+            cl_sentinel = crate::c::DELETE_SENTINEL,
+        );
+        db.exec_safe(&sql)?;
+
+        // Next delete entries that no longer have a row
+        let mut sql = String::from(
+            format!(
+              "DELETE FROM \"{tbl_name}__crsql_clock\" WHERE (__crsql_col_name != '-1' OR (__crsql_col_name = '-1' AND __crsql_col_version % 2 != 0)) AND NOT EXISTS (SELECT 1 FROM \"{tbl_name}\" WHERE ",
+              tbl_name = crate::util::escape_ident(tbl_name_str),
+            ),
+        );
+        let c_rc = crsql_ensureTableInfosAreUpToDate(db, ext_data, errmsg);
+        if c_rc != ResultCode::OK as c_int {
+            if let Some(rc) = ResultCode::from_i32(c_rc) {
+                return Err(rc);
+            }
+            return Err(ResultCode::ERROR);
+        }
+        let table_info = crsql_findTableInfo(
+            (*ext_data).zpTableInfos,
+            (*ext_data).tableInfosLen,
+            tbl_name,
+        );
+        if table_info.is_null() {
+            return Err(ResultCode::ERROR);
+        }
+
+        // for each pk col, append \"%w\".\"%w\" = \"%w__crsql_clock\".\"%w\"
+        // to the where clause then close the statement.
+        let pk_cols = sqlite::args!((*table_info).pksLen, (*table_info).pks);
+        for (i, col) in pk_cols.iter().enumerate() {
+            if i > 0 {
+                sql.push_str(" AND ");
+            }
+            let col_name_str = CStr::from_ptr((*col).name).to_str()?;
+            sql.push_str(&format!(
+                "\"{tbl_name}\".\"{col_name}\" = \"{tbl_name}__crsql_clock\".\"{col_name}\"",
+                tbl_name = tbl_name_str,
+                col_name = col_name_str,
+            ));
+        }
+        sql.push_str(" LIMIT 1)");
+        db.exec_safe(&sql)?;
+    }
+
+    let stmt = db.prepare_v2(
+        "INSERT OR REPLACE INTO crsql_master (key, value) VALUES ('pre_compact_dbversion', ?)",
+    )?;
+    stmt.bind_int64(1, current_db_version)?;
+    stmt.step()?;
+    Ok(ResultCode::OK)
+}

libsql-sqlite3/ext/crr/rs/core/src/automigrate.rs

@@ -9,6 +9,7 @@ use alloc::string::ToString;
 use alloc::vec;
 use alloc::vec::Vec;
 use core::ffi::{c_char, c_int};
+use core::mem;
 use core::slice;
 use sqlite::ColumnType;
 use sqlite_nostd as sqlite;
@@ -34,15 +35,19 @@ pub extern "C" fn crsql_automigrate(
     argc: c_int,
     argv: *mut *mut sqlite::value,
 ) {
-    if argc != 1 {
-        ctx.result_error("Expected a single argument -- the schema string of create table statements to migrate to");
+    if argc < 1 {
+        ctx.result_error("Had no args. Expected a schema to migrate to");
         return;
     }
 
     let args = args!(argc, argv);
     if let Err(code) = automigrate_impl(ctx, args) {
-        ctx.result_error("failed to apply the updated schema");
-        ctx.result_error_code(code);
+        // We're using `Err(OK)` to signify that error message and code were already set.
+        if code != ResultCode::OK {
+            ctx.result_error(&format!("failed to apply the updated schema {:?}", code));
+            ctx.result_error_code(code);
+        }
+
         return;
     }
 
@@ -53,6 +58,14 @@ fn automigrate_impl(
     ctx: *mut sqlite::context,
     args: &[*mut sqlite::value],
 ) -> Result<ResultCode, ResultCode> {
+    let cleanup = |mem_db: ManagedConnection| {
+        if args.len() == 2 {
+            let cleanup_stmt = args[1].text();
+            mem_db.exec_safe(cleanup_stmt)
+        } else {
+            Ok(ResultCode::OK)
+        }
+    };
     let local_db = ctx.db_handle();
     let desired_schema = args[0].text();
     let stripped_schema = strip_crr_statements(desired_schema);
@@ -60,33 +73,43 @@ fn automigrate_impl(
     let result = sqlite::open(strlit!(":memory:"));
     if let Ok(mem_db) = result {
         if let Err(_) = mem_db.exec_safe(&stripped_schema) {
-            return Err(ResultCode::SCHEMA);
+            let mem_db_err_msg = mem_db.errmsg()?;
+            ctx.result_error(&mem_db_err_msg);
+            ctx.result_error_code(mem_db.errcode());
+            cleanup(mem_db)?;
+            return Err(ResultCode::OK);
         }
         local_db.exec_safe("SAVEPOINT automigrate_tables;")?;
-        if let Err(_) = migrate_to(local_db, mem_db) {
+
+        let migrate_result = migrate_to(local_db, &mem_db);
+
+        if let Err(_) = migrate_result {
             local_db.exec_safe("ROLLBACK TO automigrate_tables")?;
-            return Err(ResultCode::MISMATCH);
+            let mem_db_err_msg = mem_db.errmsg()?;
+            ctx.result_error(&mem_db_err_msg);
+            ctx.result_error_code(mem_db.errcode());
+            cleanup(mem_db)?;
+            return Err(ResultCode::OK);
+        } else {
+            cleanup(mem_db)?;
         }
-        // wait wait. This need not be done.
-        // We will run the schema against the local_db post migration.
-        // To pull in:
-        // - crr application
-        // - new index creation
-        // - new table creation
-        // - anything extra the user did like trigger creation
-        //
-        // In this way we simplify this automigrate code.
-        // The user's schema thus must then be idemptotent via `IF NOT EXISTS` statements.
+
         if !desired_schema.is_empty() {
             local_db.exec_safe(desired_schema)?;
         }
         local_db.exec_safe("RELEASE automigrate_tables")
     } else {
-        return Err(ResultCode::CANTOPEN);
+        ctx.result_error("could not open the temporary migration db");
+        ctx.result_error_code(ResultCode::CANTOPEN);
+        return Err(ResultCode::OK);
     }
 }
 
-fn migrate_to(local_db: *mut sqlite3, mem_db: ManagedConnection) -> Result<ResultCode, ResultCode> {
+fn migrate_to(
+    local_db: *mut sqlite3,
+    mem_db: &ManagedConnection,
+) -> Result<ResultCode, ResultCode> {
+    // TODO: why not HashSet?
     let mut mem_tables: BTreeSet<String> = BTreeSet::new();
 
     let sql = "SELECT name FROM sqlite_master WHERE type = 'table'
@@ -137,7 +160,10 @@ fn migrate_to(local_db: *mut sqlite3, mem_db: ManagedConnection) -> Result<Resul
 fn strip_crr_statements(schema: &str) -> String {
     schema
         .split("\n")
-        .filter(|line| !line.to_lowercase().contains("crsql_as_crr"))
+        .filter(|line| {
+            !line.to_lowercase().contains("crsql_as_crr")
+                && !line.to_lowercase().contains("crsql_fract_as_ordered")
+        })
         .collect::<Vec<_>>()
         .join("\n")
 }
@@ -146,7 +172,7 @@ fn drop_tables(local_db: *mut sqlite3, tables: Vec<String>) -> Result<ResultCode
     for table in tables {
         local_db.exec_safe(&format!(
             "DROP TABLE \"{table}\"",
-            table = crate::escape_ident(&table)
+            table = crate::util::escape_ident(&table)
         ))?;
     }
 
@@ -214,11 +240,15 @@ fn drop_columns(
     table: &str,
     columns: Vec<String>,
 ) -> Result<ResultCode, ResultCode> {
+    local_db.exec_safe(&format!(
+        "DROP VIEW IF EXISTS \"{table}_fractindex\"",
+        table = crate::util::escape_ident(table)
+    ))?;
     for col in columns {
         local_db.exec_safe(&format!(
             "ALTER TABLE \"{table}\" DROP \"{column}\"",
-            table = crate::escape_ident(table),
-            column = crate::escape_ident(&col)
+            table = crate::util::escape_ident(table),
+            column = crate::util::escape_ident(&col)
         ))?;
     }
 
@@ -289,8 +319,8 @@ fn add_column(
 
     local_db.exec_safe(&format!(
         "ALTER TABLE \"{table}\" ADD COLUMN \"{name}\" {col_type} {notnull} {dflt}",
-        table = crate::escape_ident(table),
-        name = crate::escape_ident(name),
+        table = crate::util::escape_ident(table),
+        name = crate::util::escape_ident(name),
         col_type = col_type,
         notnull = if notnull { "NOT NULL " } else { "" },
         dflt = dflt_val_str
@@ -347,7 +377,10 @@ fn drop_indices(local_db: *mut sqlite3, dropped: &Vec<String>) -> Result<ResultC
     // drop if exists given column dropping could have destroyed the index
     // already.
     for idx in dropped {
-        let sql = format!("DROP INDEX IF EXISTS \"{}\"", crate::escape_ident(&idx));
+        let sql = format!(
+            "DROP INDEX IF EXISTS \"{}\"",
+            crate::util::escape_ident(&idx)
+        );
         if let Err(e) = local_db.exec_safe(&sql) {
             return Err(e);
         }
@@ -386,7 +419,7 @@ fn maybe_recreate_index(
         // drop to finalize those statements
         drop(fetch_is_unique_mem);
         drop(fetch_is_unique_local);
-        return recreate_index(local_db, table, idx, mem_db);
+        return recreate_index(local_db, idx);
     }
 
     let fetch_idx_cols_mem = mem_db.prepare_v2(IDX_COLS_SQL)?;
@@ -400,25 +433,20 @@ fn maybe_recreate_index(
             // drop to finalize those statements
             drop(fetch_idx_cols_local);
             drop(fetch_idx_cols_mem);
-            return recreate_index(local_db, table, idx, mem_db);
+            return recreate_index(local_db, idx);
         }
         fetch_idx_cols_mem.step()?;
         fetch_idx_cols_local.step()?;
     }
 
     if mem_result != local_result {
-        return recreate_index(local_db, table, idx, mem_db);
+        return recreate_index(local_db, idx);
     }
 
     Ok(ResultCode::OK)
 }
 
-fn recreate_index(
-    local_db: *mut sqlite3,
-    table: &str,
-    idx: &str,
-    mem_db: &ManagedConnection,
-) -> Result<ResultCode, ResultCode> {
+fn recreate_index(local_db: *mut sqlite3, idx: &str) -> Result<ResultCode, ResultCode> {
     let indices = vec![idx.to_string()];
     drop_indices(local_db, &indices)?;
     // no need to call add_indices

libsql-sqlite3/ext/crr/rs/core/src/backfill.rs

@@ -1,6 +1,8 @@
 use sqlite_nostd::{sqlite3, Connection, Destructor, ManagedStmt, ResultCode};
 extern crate alloc;
+use crate::util::get_dflt_value;
 use alloc::format;
+use alloc::string::String;
 use alloc::{vec, vec::Vec};
 
 /**
@@ -11,43 +13,62 @@ pub fn backfill_table(
     table: &str,
     pk_cols: Vec<&str>,
     non_pk_cols: Vec<&str>,
+    is_commit_alter: bool,
+    no_tx: bool,
 ) -> Result<ResultCode, ResultCode> {
-    db.exec_safe("SAVEPOINT backfill")?;
+    if !no_tx {
+        db.exec_safe("SAVEPOINT backfill")?;
+    }
 
     let sql = format!(
-      "SELECT {pk_cols} FROM \"{table}\" as t1
-        LEFT JOIN \"{table}__crsql_clock\" as t2 ON {pk_on_conditions} WHERE t2.\"{first_pk}\" IS NULL",
-      table = crate::escape_ident(table),
-      pk_cols = pk_cols
-          .iter()
-          .map(|f| format!("t1.\"{}\"", crate::escape_ident(f)))
-          .collect::<Vec<_>>()
-          .join(", "),
-      pk_on_conditions = pk_cols
-          .iter()
-          .map(|f| format!("t1.\"{}\" = t2.\"{}\"", crate::escape_ident(f), crate::escape_ident(f)))
-          .collect::<Vec<_>>()
-          .join(" AND "),
-      first_pk = crate::escape_ident(pk_cols[0]),
+        "SELECT {pk_cols} FROM \"{table}\" AS t1
+        WHERE NOT EXISTS
+          (SELECT 1 FROM \"{table}__crsql_clock\" AS t2 WHERE {pk_where_conditions})",
+        table = crate::util::escape_ident(table),
+        pk_cols = pk_cols
+            .iter()
+            .map(|f| format!("t1.\"{}\"", crate::util::escape_ident(f)))
+            .collect::<Vec<_>>()
+            .join(", "),
+        pk_where_conditions = pk_cols
+            .iter()
+            .map(|f| format!(
+                "t1.\"{col_name}\" IS t2.\"{col_name}\"",
+                col_name = crate::util::escape_ident(f),
+            ))
+            .collect::<Vec<_>>()
+            .join(" AND "),
     );
     let stmt = db.prepare_v2(&sql);
 
     let result = match stmt {
-        Ok(stmt) => create_clock_rows_from_stmt(stmt, db, table, &pk_cols, &non_pk_cols),
+        Ok(stmt) => {
+            create_clock_rows_from_stmt(stmt, db, table, &pk_cols, &non_pk_cols, is_commit_alter)
+        }
         Err(e) => Err(e),
     };
 
     if let Err(e) = result {
-        db.exec_safe("ROLLBACK TO backfill")?;
+        if !no_tx {
+            db.exec_safe("ROLLBACK TO backfill")?;
+        }
+
         return Err(e);
     }
 
-    if let Err(e) = backfill_missing_columns(db, table, &pk_cols, &non_pk_cols) {
-        db.exec_safe("ROLLBACK TO backfill")?;
+    if let Err(e) = backfill_missing_columns(db, table, &pk_cols, &non_pk_cols, is_commit_alter) {
+        if !no_tx {
+            db.exec_safe("ROLLBACK TO backfill")?;
+        }
+
         return Err(e);
     }
 
-    db.exec_safe("RELEASE backfill")
+    if !no_tx {
+        db.exec_safe("RELEASE backfill")
+    } else {
+        Ok(ResultCode::OK)
+    }
 }
 
 /**
@@ -60,18 +81,31 @@ fn create_clock_rows_from_stmt(
     table: &str,
     pk_cols: &Vec<&str>,
     non_pk_cols: &Vec<&str>,
+    is_commit_alter: bool,
 ) -> Result<ResultCode, ResultCode> {
+    // We do not grab nextdbversion on migration.
+    // The idea is that other nodes will apply the same migration
+    // in the future so if they have already seen this node up
+    // to the current db version then the migration will place them into the correct
+    // state. No need to re-sync post migration.
+    // or-ignore since we do not drop sentinel values during compaction as they act as our metadata
+    // to determine if rows should resurrect on a future insertion event provided by a peer.
     let sql = format!(
-        "INSERT INTO \"{table}__crsql_clock\"
-          ({pk_cols}, __crsql_col_name, __crsql_col_version, __crsql_db_version) VALUES
-          ({pk_values}, ?, 1, crsql_nextdbversion())",
-        table = crate::escape_ident(table),
+        "INSERT OR IGNORE INTO \"{table}__crsql_clock\"
+          ({pk_cols}, __crsql_col_name, __crsql_col_version, __crsql_db_version, __crsql_seq) VALUES
+          ({pk_values}, ?, 1, {dbversion_getter}, crsql_increment_and_get_seq())",
+        table = crate::util::escape_ident(table),
         pk_cols = pk_cols
             .iter()
-            .map(|f| format!("\"{}\"", crate::escape_ident(f)))
+            .map(|f| format!("\"{}\"", crate::util::escape_ident(f)))
             .collect::<Vec<_>>()
             .join(", "),
         pk_values = pk_cols.iter().map(|_| "?").collect::<Vec<_>>().join(", "),
+        dbversion_getter = if is_commit_alter {
+            "crsql_db_version()"
+        } else {
+            "crsql_next_db_version()"
+        }
     );
     let write_stmt = db.prepare_v2(&sql)?;
 
@@ -84,11 +118,20 @@ fn create_clock_rows_from_stmt(
 
         for col in non_pk_cols.iter() {
             // We even backfill default values since we can't differentiate between an explicit
-            // reset to a default vs an implicit set to default on create.
+            // reset to a default vs an implicit set to default on create. Do we? I don't think we do set defaults.
             write_stmt.bind_text(pk_cols.len() as i32 + 1, col, Destructor::STATIC)?;
             write_stmt.step()?;
             write_stmt.reset()?;
         }
+        if non_pk_cols.len() == 0 {
+            write_stmt.bind_text(
+                pk_cols.len() as i32 + 1,
+                crate::c::INSERT_SENTINEL,
+                Destructor::STATIC,
+            )?;
+            write_stmt.step()?;
+            write_stmt.reset()?;
+        }
     }
 
     Ok(ResultCode::OK)
@@ -99,63 +142,64 @@ fn create_clock_rows_from_stmt(
 * If not, fill the data in for it for each row.
 *
 * Can we optimize and skip cases where it is equivalent to the default value?
-* E.g., adding a new column should not require a backfill...
+* E.g., adding a new column set to default values should not require a backfill...
 */
 fn backfill_missing_columns(
     db: *mut sqlite3,
     table: &str,
     pk_cols: &Vec<&str>,
     non_pk_cols: &Vec<&str>,
+    is_commit_alter: bool,
 ) -> Result<ResultCode, ResultCode> {
-    let has_col_stmt = db.prepare_v2(&format!(
-        "SELECT 1 FROM \"{table}__crsql_clock\" WHERE \"__crsql_col_name\" = ? LIMIT 1",
-        table = table,
-    ))?;
     for non_pk_col in non_pk_cols {
-        has_col_stmt.bind_text(1, non_pk_col, Destructor::STATIC)?;
-        let exists = has_col(&has_col_stmt)?;
-        has_col_stmt.reset()?;
-        if exists {
-            continue;
-        }
-        fill_column(db, table, &pk_cols, non_pk_col)?;
+        fill_column(db, table, &pk_cols, non_pk_col, is_commit_alter)?;
     }
 
     Ok(ResultCode::OK)
 }
 
-fn has_col(stmt: &ManagedStmt) -> Result<bool, ResultCode> {
-    let step_result = stmt.step()?;
-    if step_result == ResultCode::DONE {
-        Ok(false)
-    } else {
-        Ok(true)
-    }
-}
-
-/**
-*
-*/
+// This doesn't fill compeltely new columns...
+// Wel... does it not? The on condition x left join should do it.
 fn fill_column(
     db: *mut sqlite3,
     table: &str,
     pk_cols: &Vec<&str>,
     non_pk_col: &str,
+    is_commit_alter: bool,
 ) -> Result<ResultCode, ResultCode> {
-    // We don't technically need this join, right?
-    // There should never be a partially filled column.
-    // If there is there's likely a bug elsewhere.
+    // Only fill rows for which
+    // - a row does not exist for that pk combo _and_ the cid in the clock table.
+    // - the value is not the default value for that column.
+    let dflt_value = get_dflt_value(db, table, non_pk_col)?;
     let sql = format!(
-        "SELECT {pk_cols} FROM {table} as t1",
-        table = crate::escape_ident(table),
+        "SELECT {pk_cols} FROM {table} as t1
+          LEFT JOIN \"{table}__crsql_clock\" as t2 ON {pk_on_conditions} AND t2.__crsql_col_name = ?
+          WHERE t2.\"{first_pk}\" IS NULL {dflt_value_condition}",
+        table = crate::util::escape_ident(table),
         pk_cols = pk_cols
             .iter()
-            .map(|f| format!("t1.\"{}\"", crate::escape_ident(f)))
+            .map(|f| format!("t1.\"{}\"", crate::util::escape_ident(f)))
             .collect::<Vec<_>>()
             .join(", "),
+        pk_on_conditions = pk_cols
+            .iter()
+            .map(|f| format!(
+                "t1.\"{}\" = t2.\"{}\"",
+                crate::util::escape_ident(f),
+                crate::util::escape_ident(f)
+            ))
+            .collect::<Vec<_>>()
+            .join(" AND "),
+        first_pk = crate::util::escape_ident(pk_cols[0]),
+        dflt_value_condition = if let Some(dflt) = dflt_value {
+            format!("AND t1.\"{}\" IS NOT {}", non_pk_col, dflt)
+        } else {
+            String::from("")
+        },
     );
     let read_stmt = db.prepare_v2(&sql)?;
+    read_stmt.bind_text(1, non_pk_col, Destructor::STATIC)?;
 
     let non_pk_cols = vec![non_pk_col];
-    create_clock_rows_from_stmt(read_stmt, db, table, pk_cols, &non_pk_cols)
+    create_clock_rows_from_stmt(read_stmt, db, table, pk_cols, &non_pk_cols, is_commit_alter)
 }

libsql-sqlite3/ext/crr/rs/core/src/bootstrap.rs

@@ -0,0 +1,235 @@
+use core::ffi::{c_char, c_int, CStr};
+
+use crate::{c::crsql_TableInfo, consts};
+use alloc::{ffi::CString, format};
+use core::slice;
+use sqlite::{sqlite3, Connection, Destructor, ResultCode};
+use sqlite_nostd as sqlite;
+
+fn uuid() -> [u8; 16] {
+    let mut blob: [u8; 16] = [0; 16];
+    sqlite::randomness(&mut blob);
+    blob[6] = (blob[6] & 0x0f) + 0x40;
+    blob[8] = (blob[8] & 0x3f) + 0x80;
+    blob
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_init_site_id(db: *mut sqlite3, ret: *mut u8) -> c_int {
+    let buffer: &mut [u8] = unsafe { slice::from_raw_parts_mut(ret, 16) };
+    if let Ok(site_id) = init_site_id(db) {
+        buffer.copy_from_slice(&site_id);
+        ResultCode::OK as c_int
+    } else {
+        ResultCode::ERROR as c_int
+    }
+}
+
+fn insert_site_id(db: *mut sqlite3) -> Result<[u8; 16], ResultCode> {
+    let stmt = db.prepare_v2(&format!(
+        "INSERT INTO \"{tbl}\" (site_id, ordinal) VALUES (?, 0)",
+        tbl = consts::TBL_SITE_ID
+    ))?;
+
+    let site_id = uuid();
+    stmt.bind_blob(1, &site_id, Destructor::STATIC)?;
+    stmt.step()?;
+
+    Ok(site_id)
+}
+
+fn create_site_id_and_site_id_table(db: *mut sqlite3) -> Result<[u8; 16], ResultCode> {
+    db.exec_safe(&format!(
+        "CREATE TABLE \"{tbl}\" (site_id BLOB NOT NULL, ordinal INTEGER PRIMARY KEY AUTOINCREMENT);
+        CREATE UNIQUE INDEX {tbl}_site_id ON \"{tbl}\" (site_id);",
+        tbl = consts::TBL_SITE_ID
+    ))?;
+
+    insert_site_id(db)
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_init_peer_tracking_table(db: *mut sqlite3) -> c_int {
+    match db.exec_safe("CREATE TABLE IF NOT EXISTS crsql_tracked_peers (\"site_id\" BLOB NOT NULL, \"version\" INTEGER NOT NULL, \"seq\" INTEGER DEFAULT 0, \"tag\" INTEGER, \"event\" INTEGER, PRIMARY KEY (\"site_id\", \"tag\", \"event\")) STRICT;") {
+      Ok(_) => ResultCode::OK as c_int,
+      Err(code) => code as c_int
+    }
+}
+
+fn has_table(db: *mut sqlite3, table_name: &str) -> Result<bool, ResultCode> {
+    let stmt =
+        db.prepare_v2("SELECT 1 FROM sqlite_master WHERE type = 'table' AND tbl_name = ?")?;
+    stmt.bind_text(1, table_name, Destructor::STATIC)?;
+    let tbl_exists_result = stmt.step()?;
+    Ok(tbl_exists_result == ResultCode::ROW)
+}
+
+/**
+ * Loads the siteId into memory. If a site id
+ * cannot be found for the given database one is created
+ * and saved to the site id table.
+ */
+fn init_site_id(db: *mut sqlite3) -> Result<[u8; 16], ResultCode> {
+    if !has_table(db, consts::TBL_SITE_ID)? {
+        return create_site_id_and_site_id_table(db);
+    }
+
+    let stmt = db.prepare_v2(&format!(
+        "SELECT site_id FROM \"{}\" WHERE ordinal = 0",
+        consts::TBL_SITE_ID
+    ))?;
+    let result_code = stmt.step()?;
+
+    let ret = if result_code == ResultCode::DONE {
+        insert_site_id(db)?
+    } else {
+        let site_id_from_table = stmt.column_blob(0)?;
+        site_id_from_table.try_into()?
+    };
+
+    Ok(ret)
+}
+
+fn crsql_create_schema_table_if_not_exists(db: *mut sqlite3) -> Result<ResultCode, ResultCode> {
+    db.exec_safe("SAVEPOINT crsql_create_schema_table;")?;
+
+    if let Ok(_) = db.exec_safe(&format!(
+        "CREATE TABLE IF NOT EXISTS \"{}\" (\"key\" TEXT PRIMARY KEY, \"value\" ANY);",
+        consts::TBL_SCHEMA
+    )) {
+        db.exec_safe("RELEASE crsql_create_schema_table;")
+    } else {
+        let _ = db.exec_safe("ROLLBACK");
+        Err(ResultCode::ERROR)
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_maybe_update_db(db: *mut sqlite3, err_msg: *mut *mut c_char) -> c_int {
+    // No schema table? First time this DB has been opened with this extension.
+    if let Ok(has_schema_table) = has_table(db, consts::TBL_SCHEMA) {
+        if let Err(code) = crsql_create_schema_table_if_not_exists(db) {
+            return code as c_int;
+        }
+        let r = db.exec_safe("SAVEPOINT crsql_maybe_update_db;");
+        if let Err(code) = r {
+            return code as c_int;
+        }
+        if let Ok(_) = maybe_update_db_inner(db, has_schema_table == false, err_msg) {
+            let _ = db.exec_safe("RELEASE crsql_maybe_update_db;");
+            return ResultCode::OK as c_int;
+        } else {
+            let _ = db.exec_safe("ROLLBACK;");
+            return ResultCode::ERROR as c_int;
+        }
+    } else {
+        return ResultCode::ERROR as c_int;
+    }
+}
+
+fn maybe_update_db_inner(
+    db: *mut sqlite3,
+    is_blank_slate: bool,
+    err_msg: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    let mut recorded_version: i32 = 0;
+
+    // Completely new DBs need no migrations.
+    // We can set them to the current version.
+    if is_blank_slate {
+        recorded_version = consts::CRSQLITE_VERSION;
+    } else {
+        let stmt =
+            db.prepare_v2("SELECT value FROM crsql_master WHERE key = 'crsqlite_version'")?;
+        let step_result = stmt.step()?;
+        if step_result == ResultCode::ROW {
+            recorded_version = stmt.column_int(0)?;
+        }
+    }
+
+    if recorded_version < consts::CRSQLITE_VERSION && !is_blank_slate {
+        // todo: return an error message to the user that their version is
+        // not supported
+        let cstring = CString::new(format!("Opening a db created with cr-sqlite version {} is not supported. Upcoming release 0.15.0 is a breaking change.", recorded_version))?;
+        unsafe {
+            (*err_msg) = cstring.into_raw();
+            return Err(ResultCode::ERROR);
+        }
+    }
+
+    // if recorded_version < consts::CRSQLITE_VERSION_0_13_0 {
+    //     update_to_0_13_0(db)?;
+    // }
+
+    // if recorded_version < consts::CRSQLITE_VERSION_0_15_0 {
+    //     update_to_0_15_0(db)?;
+    // }
+
+    // write the db version if we migrated to a new one or we are a blank slate db
+    if recorded_version < consts::CRSQLITE_VERSION || is_blank_slate {
+        let stmt =
+            db.prepare_v2("INSERT OR REPLACE INTO crsql_master VALUES ('crsqlite_version', ?)")?;
+        stmt.bind_int(1, consts::CRSQLITE_VERSION)?;
+        stmt.step()?;
+    }
+
+    Ok(ResultCode::OK)
+}
+
+/**
+ * The clock table holds the versions for each column of a given row.
+ *
+ * These version are set to the dbversion at the time of the write to the
+ * column.
+ *
+ * The dbversion is updated on transaction commit.
+ * This allows us to find all columns written in the same transaction
+ * albeit with caveats.
+ *
+ * The caveats being that two partiall overlapping transactions will
+ * clobber the full transaction picture given we only keep latest
+ * state and not a full causal history.
+ *
+ * @param tableInfo
+ */
+#[no_mangle]
+pub extern "C" fn crsql_create_clock_table(
+    db: *mut sqlite3,
+    table_info: *mut crsql_TableInfo,
+    err: *mut *mut c_char,
+) -> c_int {
+    match create_clock_table(db, table_info, err) {
+        Ok(_) => ResultCode::OK as c_int,
+        Err(code) => code as c_int,
+    }
+}
+
+fn create_clock_table(
+    db: *mut sqlite3,
+    table_info: *mut crsql_TableInfo,
+    _err: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    let columns = sqlite::args!((*table_info).pksLen, (*table_info).pks);
+    let pk_list = crate::util::as_identifier_list(columns, None)?;
+    let table_name = unsafe { CStr::from_ptr((*table_info).tblName).to_str() }?;
+
+    db.exec_safe(&format!(
+        "CREATE TABLE IF NOT EXISTS \"{table_name}__crsql_clock\" (
+      {pk_list},
+      __crsql_col_name TEXT NOT NULL,
+      __crsql_col_version INT NOT NULL,
+      __crsql_db_version INT NOT NULL,
+      __crsql_site_id INT,
+      __crsql_seq INT NOT NULL,
+      PRIMARY KEY ({pk_list}, __crsql_col_name)
+    )",
+        pk_list = pk_list,
+        table_name = crate::util::escape_ident(table_name)
+    ))?;
+
+    db.exec_safe(
+      &format!(
+        "CREATE INDEX IF NOT EXISTS \"{table_name}__crsql_clock_dbv_idx\" ON \"{table_name}__crsql_clock\" (\"__crsql_db_version\")",
+        table_name = crate::util::escape_ident(table_name),
+      ))
+}

libsql-sqlite3/ext/crr/rs/core/src/c.rs

@@ -0,0 +1,642 @@
+extern crate alloc;
+use core::ffi::{c_char, c_int};
+#[cfg(not(feature = "std"))]
+use num_derive::FromPrimitive;
+
+// Structs that still exist in C but will eventually be moved to Rust
+// As well as functions re-defined in Rust but not yet deleted from C
+use sqlite_nostd as sqlite;
+
+pub static INSERT_SENTINEL: &str = "-1";
+pub static DELETE_SENTINEL: &str = "-1";
+
+#[derive(FromPrimitive, PartialEq, Debug)]
+pub enum CrsqlChangesColumn {
+    Tbl = 0,
+    Pk = 1,
+    Cid = 2,
+    Cval = 3,
+    ColVrsn = 4,
+    DbVrsn = 5,
+    SiteId = 6,
+    Cl = 7,
+    Seq = 8,
+}
+
+#[derive(FromPrimitive, PartialEq, Debug)]
+pub enum ClockUnionColumn {
+    Tbl = 0,
+    Pks = 1,
+    Cid = 2,
+    ColVrsn = 3,
+    DbVrsn = 4,
+    SiteId = 5,
+    RowId = 6,
+    Seq = 7,
+    Cl = 8,
+}
+
+#[derive(FromPrimitive, PartialEq, Debug)]
+pub enum ChangeRowType {
+    Update = 0,
+    Delete = 1,
+    PkOnly = 2,
+}
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_snake_case, non_camel_case_types)]
+pub struct crsql_TableInfo {
+    pub tblName: *mut ::core::ffi::c_char,
+    pub baseCols: *mut crsql_ColumnInfo,
+    pub baseColsLen: ::core::ffi::c_int,
+    pub pks: *mut crsql_ColumnInfo,
+    pub pksLen: ::core::ffi::c_int,
+    pub nonPks: *mut crsql_ColumnInfo,
+    pub nonPksLen: ::core::ffi::c_int,
+}
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_snake_case, non_camel_case_types)]
+pub struct crsql_ColumnInfo {
+    pub cid: ::core::ffi::c_int,
+    pub name: *mut ::core::ffi::c_char,
+    pub type_: *mut ::core::ffi::c_char,
+    pub notnull: ::core::ffi::c_int,
+    pub pk: ::core::ffi::c_int,
+}
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_snake_case, non_camel_case_types)]
+pub struct crsql_ExtData {
+    pub pPragmaSchemaVersionStmt: *mut sqlite::stmt,
+    pub pPragmaDataVersionStmt: *mut sqlite::stmt,
+    pub pragmaDataVersion: ::core::ffi::c_int,
+    pub dbVersion: sqlite::int64,
+    pub pendingDbVersion: sqlite::int64,
+    pub pragmaSchemaVersion: ::core::ffi::c_int,
+    pub pragmaSchemaVersionForTableInfos: ::core::ffi::c_int,
+    pub siteId: *mut ::core::ffi::c_uchar,
+    pub pDbVersionStmt: *mut sqlite::stmt,
+    pub zpTableInfos: *mut *mut crsql_TableInfo,
+    pub tableInfosLen: ::core::ffi::c_int,
+    pub rowsImpacted: ::core::ffi::c_int,
+    pub seq: ::core::ffi::c_int,
+    pub pSetSyncBitStmt: *mut sqlite::stmt,
+    pub pClearSyncBitStmt: *mut sqlite::stmt,
+    pub pSetSiteIdOrdinalStmt: *mut sqlite::stmt,
+    pub pSelectSiteIdOrdinalStmt: *mut sqlite::stmt,
+    pub pStmtCache: *mut ::core::ffi::c_void,
+}
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_snake_case, non_camel_case_types)]
+pub struct crsql_Changes_vtab {
+    pub base: sqlite::vtab,
+    pub db: *mut sqlite::sqlite3,
+    pub pExtData: *mut crsql_ExtData,
+}
+
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
+#[allow(non_snake_case, non_camel_case_types)]
+pub struct crsql_Changes_cursor {
+    pub base: sqlite::vtab_cursor,
+    pub pTab: *mut crsql_Changes_vtab,
+    pub pChangesStmt: *mut sqlite::stmt,
+    pub pRowStmt: *mut sqlite::stmt,
+    pub dbVersion: sqlite::int64,
+    pub rowType: ::core::ffi::c_int,
+    pub changesRowid: sqlite::int64,
+    pub tblInfoIdx: ::core::ffi::c_int,
+}
+
+extern "C" {
+    pub fn crsql_indexofTableInfo(
+        tblInfos: *mut *mut crsql_TableInfo,
+        len: ::core::ffi::c_int,
+        tblName: *const ::core::ffi::c_char,
+    ) -> ::core::ffi::c_int;
+    pub fn crsql_findTableInfo(
+        tblInfos: *mut *mut crsql_TableInfo,
+        len: c_int,
+        tblName: *const c_char,
+    ) -> *mut crsql_TableInfo;
+    pub fn crsql_ensureTableInfosAreUpToDate(
+        db: *mut sqlite::sqlite3,
+        pExtData: *mut crsql_ExtData,
+        errmsg: *mut *mut c_char,
+    ) -> c_int;
+    pub fn crsql_getDbVersion(
+        db: *mut sqlite::sqlite3,
+        ext_data: *mut crsql_ExtData,
+        err_msg: *mut *mut c_char,
+    ) -> c_int;
+    pub fn crsql_createCrr(
+        db: *mut sqlite::sqlite3,
+        schemaName: *const c_char,
+        tblName: *const c_char,
+        isCommitAlter: c_int,
+        noTx: c_int,
+        err: *mut *mut c_char,
+    ) -> c_int;
+}
+
+#[test]
+fn bindgen_test_layout_crsql_Changes_vtab() {
+    const UNINIT: ::core::mem::MaybeUninit<crsql_Changes_vtab> = ::core::mem::MaybeUninit::uninit();
+    let ptr = UNINIT.as_ptr();
+    assert_eq!(
+        ::core::mem::size_of::<crsql_Changes_vtab>(),
+        40usize,
+        concat!("Size of: ", stringify!(crsql_Changes_vtab))
+    );
+    assert_eq!(
+        ::core::mem::align_of::<crsql_Changes_vtab>(),
+        8usize,
+        concat!("Alignment of ", stringify!(crsql_Changes_vtab))
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).base) as usize - ptr as usize },
+        0usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_vtab),
+            "::",
+            stringify!(base)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).db) as usize - ptr as usize },
+        24usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_vtab),
+            "::",
+            stringify!(db)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).pExtData) as usize - ptr as usize },
+        32usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_vtab),
+            "::",
+            stringify!(pExtData)
+        )
+    );
+}
+
+#[test]
+fn bindgen_test_layout_crsql_Changes_cursor() {
+    const UNINIT: ::core::mem::MaybeUninit<crsql_Changes_cursor> =
+        ::core::mem::MaybeUninit::uninit();
+    let ptr = UNINIT.as_ptr();
+    assert_eq!(
+        ::core::mem::size_of::<crsql_Changes_cursor>(),
+        64usize,
+        concat!("Size of: ", stringify!(crsql_Changes_cursor))
+    );
+    assert_eq!(
+        ::core::mem::align_of::<crsql_Changes_cursor>(),
+        8usize,
+        concat!("Alignment of ", stringify!(crsql_Changes_cursor))
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).base) as usize - ptr as usize },
+        0usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_cursor),
+            "::",
+            stringify!(base)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).pTab) as usize - ptr as usize },
+        8usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_cursor),
+            "::",
+            stringify!(pTab)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).pChangesStmt) as usize - ptr as usize },
+        16usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_cursor),
+            "::",
+            stringify!(pChangesStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).pRowStmt) as usize - ptr as usize },
+        24usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_cursor),
+            "::",
+            stringify!(pRowStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).dbVersion) as usize - ptr as usize },
+        32usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_cursor),
+            "::",
+            stringify!(dbVersion)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).rowType) as usize - ptr as usize },
+        40usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_cursor),
+            "::",
+            stringify!(rowType)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).changesRowid) as usize - ptr as usize },
+        48usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_cursor),
+            "::",
+            stringify!(changesRowid)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).tblInfoIdx) as usize - ptr as usize },
+        56usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_Changes_cursor),
+            "::",
+            stringify!(tblInfoIdx)
+        )
+    );
+}
+
+#[test]
+#[allow(non_snake_case)]
+fn bindgen_test_layout_crsql_ColumnInfo() {
+    const UNINIT: ::core::mem::MaybeUninit<crsql_ColumnInfo> = ::core::mem::MaybeUninit::uninit();
+    let ptr = UNINIT.as_ptr();
+    assert_eq!(
+        ::core::mem::size_of::<crsql_ColumnInfo>(),
+        32usize,
+        concat!("Size of: ", stringify!(crsql_ColumnInfo))
+    );
+    assert_eq!(
+        ::core::mem::align_of::<crsql_ColumnInfo>(),
+        8usize,
+        concat!("Alignment of ", stringify!(crsql_ColumnInfo))
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).cid) as usize - ptr as usize },
+        0usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ColumnInfo),
+            "::",
+            stringify!(cid)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).name) as usize - ptr as usize },
+        8usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ColumnInfo),
+            "::",
+            stringify!(name)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).type_) as usize - ptr as usize },
+        16usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ColumnInfo),
+            "::",
+            stringify!(type_)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).notnull) as usize - ptr as usize },
+        24usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ColumnInfo),
+            "::",
+            stringify!(notnull)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).pk) as usize - ptr as usize },
+        28usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ColumnInfo),
+            "::",
+            stringify!(pk)
+        )
+    );
+}
+
+#[test]
+#[allow(non_snake_case)]
+fn bindgen_test_layout_crsql_TableInfo() {
+    const UNINIT: ::core::mem::MaybeUninit<crsql_TableInfo> = ::core::mem::MaybeUninit::uninit();
+    let ptr = UNINIT.as_ptr();
+    assert_eq!(
+        ::core::mem::size_of::<crsql_TableInfo>(),
+        56usize,
+        concat!("Size of: ", stringify!(crsql_TableInfo))
+    );
+    assert_eq!(
+        ::core::mem::align_of::<crsql_TableInfo>(),
+        8usize,
+        concat!("Alignment of ", stringify!(crsql_TableInfo))
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).tblName) as usize - ptr as usize },
+        0usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_TableInfo),
+            "::",
+            stringify!(tblName)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).baseCols) as usize - ptr as usize },
+        8usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_TableInfo),
+            "::",
+            stringify!(baseCols)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).baseColsLen) as usize - ptr as usize },
+        16usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_TableInfo),
+            "::",
+            stringify!(baseColsLen)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).pks) as usize - ptr as usize },
+        24usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_TableInfo),
+            "::",
+            stringify!(pks)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).pksLen) as usize - ptr as usize },
+        32usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_TableInfo),
+            "::",
+            stringify!(pksLen)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).nonPks) as usize - ptr as usize },
+        40usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_TableInfo),
+            "::",
+            stringify!(nonPks)
+        )
+    );
+    assert_eq!(
+        unsafe { ::core::ptr::addr_of!((*ptr).nonPksLen) as usize - ptr as usize },
+        48usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_TableInfo),
+            "::",
+            stringify!(nonPksLen)
+        )
+    );
+}
+
+#[test]
+#[allow(non_snake_case)]
+fn bindgen_test_layout_crsql_ExtData() {
+    const UNINIT: ::std::mem::MaybeUninit<crsql_ExtData> = ::std::mem::MaybeUninit::uninit();
+    let ptr = UNINIT.as_ptr();
+    assert_eq!(
+        ::std::mem::size_of::<crsql_ExtData>(),
+        128usize,
+        concat!("Size of: ", stringify!(crsql_ExtData))
+    );
+    assert_eq!(
+        ::std::mem::align_of::<crsql_ExtData>(),
+        8usize,
+        concat!("Alignment of ", stringify!(crsql_ExtData))
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pPragmaSchemaVersionStmt) as usize - ptr as usize },
+        0usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pPragmaSchemaVersionStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pPragmaDataVersionStmt) as usize - ptr as usize },
+        8usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pPragmaDataVersionStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pragmaDataVersion) as usize - ptr as usize },
+        16usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pragmaDataVersion)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).dbVersion) as usize - ptr as usize },
+        24usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(dbVersion)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pendingDbVersion) as usize - ptr as usize },
+        32usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pendingDbVersion)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pragmaSchemaVersion) as usize - ptr as usize },
+        40usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pragmaSchemaVersion)
+        )
+    );
+    assert_eq!(
+        unsafe {
+            ::std::ptr::addr_of!((*ptr).pragmaSchemaVersionForTableInfos) as usize - ptr as usize
+        },
+        44usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pragmaSchemaVersionForTableInfos)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).siteId) as usize - ptr as usize },
+        48usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(siteId)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pDbVersionStmt) as usize - ptr as usize },
+        56usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pDbVersionStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).zpTableInfos) as usize - ptr as usize },
+        64usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(zpTableInfos)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).tableInfosLen) as usize - ptr as usize },
+        72usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(tableInfosLen)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).rowsImpacted) as usize - ptr as usize },
+        76usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(rowsImpacted)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).seq) as usize - ptr as usize },
+        80usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(seq)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pSetSyncBitStmt) as usize - ptr as usize },
+        88usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pSetSyncBitStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pClearSyncBitStmt) as usize - ptr as usize },
+        96usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pClearSyncBitStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pSetSiteIdOrdinalStmt) as usize - ptr as usize },
+        104usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pSetSiteIdOrdinalStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pSelectSiteIdOrdinalStmt) as usize - ptr as usize },
+        112usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pSelectSiteIdOrdinalStmt)
+        )
+    );
+    assert_eq!(
+        unsafe { ::std::ptr::addr_of!((*ptr).pStmtCache) as usize - ptr as usize },
+        120usize,
+        concat!(
+            "Offset of field: ",
+            stringify!(crsql_ExtData),
+            "::",
+            stringify!(pStmtCache)
+        )
+    );
+}

libsql-sqlite3/ext/crr/rs/core/src/changes_vtab.rs

@@ -0,0 +1,600 @@
+extern crate alloc;
+use crate::alloc::string::ToString;
+use crate::changes_vtab_write::crsql_merge_insert;
+use crate::stmt_cache::{
+    get_cache_key, get_cached_stmt, reset_cached_stmt, set_cached_stmt, CachedStmtType,
+};
+use alloc::format;
+use alloc::string::String;
+use core::ffi::{c_char, c_int, CStr};
+use core::mem::forget;
+use core::ptr::null_mut;
+use core::slice;
+
+use alloc::ffi::CString;
+#[cfg(not(feature = "std"))]
+use num_traits::FromPrimitive;
+use sqlite::{ColumnType, Connection, Context, Stmt, Value};
+use sqlite_nostd as sqlite;
+use sqlite_nostd::ResultCode;
+
+use crate::c::{
+    crsql_Changes_cursor, crsql_Changes_vtab, crsql_ensureTableInfosAreUpToDate, ChangeRowType,
+    ClockUnionColumn, CrsqlChangesColumn,
+};
+use crate::changes_vtab_read::{changes_union_query, row_patch_data_query};
+use crate::pack_columns::bind_package_to_stmt;
+use crate::unpack_columns;
+
+fn changes_crsr_finalize(crsr: *mut crsql_Changes_cursor) -> c_int {
+    // Assign pointers to null after freeing
+    // since we can get into this twice for the same cursor object.
+    unsafe {
+        let mut rc = 0;
+        rc += match (*crsr).pChangesStmt.finalize() {
+            Ok(rc) => rc as c_int,
+            Err(rc) => rc as c_int,
+        };
+        (*crsr).pChangesStmt = null_mut();
+        let reset_rc = reset_cached_stmt((*crsr).pRowStmt);
+        match reset_rc {
+            Ok(r) | Err(r) => rc += r as c_int,
+        }
+        (*crsr).pRowStmt = null_mut();
+        (*crsr).dbVersion = crate::consts::MIN_POSSIBLE_DB_VERSION;
+
+        return rc;
+    }
+}
+
+// A very c-style port. We can get more idiomatic once we finish the rust port and have test and perf parity
+#[no_mangle]
+pub unsafe extern "C" fn crsql_changes_best_index(
+    vtab: *mut sqlite::vtab,
+    index_info: *mut sqlite::index_info,
+) -> c_int {
+    match changes_best_index(vtab, index_info) {
+        Ok(rc) => rc as c_int,
+        Err(rc) => rc as c_int,
+    }
+}
+
+fn changes_best_index(
+    _vtab: *mut sqlite::vtab,
+    index_info: *mut sqlite::index_info,
+) -> Result<ResultCode, ResultCode> {
+    let mut idx_num: i32 = 0;
+
+    let mut first_constraint = true;
+    let mut str = String::new();
+    let constraints = sqlite::args!((*index_info).nConstraint, (*index_info).aConstraint);
+    let constraint_usage =
+        sqlite::args_mut!((*index_info).nConstraint, (*index_info).aConstraintUsage);
+    let mut arg_v_index = 1;
+    for (i, constraint) in constraints.iter().enumerate() {
+        if !constraint_is_usable(constraint) {
+            continue;
+        }
+        let col = CrsqlChangesColumn::from_i32(constraint.iColumn);
+        if let Some(col_name) = get_clock_table_col_name(&col) {
+            if let Some(op_string) = get_operator_string(constraint.op) {
+                if first_constraint {
+                    str.push_str("WHERE ");
+                    first_constraint = false
+                } else {
+                    str.push_str(" AND ");
+                }
+
+                if constraint.op == sqlite::INDEX_CONSTRAINT_ISNOTNULL as u8
+                    || constraint.op == sqlite::INDEX_CONSTRAINT_ISNULL as u8
+                {
+                    str.push_str(&format!("{} {}", col_name, op_string));
+                    constraint_usage[i].argvIndex = 0;
+                    constraint_usage[i].omit = 1;
+                } else {
+                    str.push_str(&format!("{} {} ?", col_name, op_string));
+                    constraint_usage[i].argvIndex = arg_v_index;
+                    constraint_usage[i].omit = 1;
+                    arg_v_index += 1;
+                }
+            }
+        }
+
+        // idx bit mask
+        match col {
+            Some(CrsqlChangesColumn::DbVrsn) => idx_num |= 2,
+            Some(CrsqlChangesColumn::SiteId) => idx_num |= 4,
+            _ => {}
+        }
+    }
+
+    let mut desc = 0;
+    let order_bys = sqlite::args!((*index_info).nOrderBy, (*index_info).aOrderBy);
+    let mut order_by_consumed = true;
+    if order_bys.len() > 0 {
+        str.push_str(" ORDER BY ");
+    } else {
+        // The user didn't provide an ordering? Tack on a default one that will
+        // retrieve changes in-order
+        str.push_str(" ORDER BY db_vrsn, seq ASC");
+    }
+    first_constraint = true;
+    for order_by in order_bys {
+        desc = order_by.desc;
+        let col = CrsqlChangesColumn::from_i32(order_by.iColumn);
+        if let Some(col_name) = get_clock_table_col_name(&col) {
+            if first_constraint {
+                first_constraint = false;
+            } else {
+                str.push_str(", ");
+            }
+            str.push_str(&col_name);
+        } else {
+            // TODO: test we're consuming
+            order_by_consumed = false;
+        }
+    }
+
+    if order_bys.len() > 0 {
+        if desc != 0 {
+            str.push_str(" DESC");
+        } else {
+            str.push_str(" ASC");
+        }
+    }
+
+    // manual null-term since we'll pass to C
+    str.push('\0');
+
+    // TODO: update your order by py test to explain query plans to ensure correct indices are selected
+    // both constraints are present. Also to check that order by is consumed.
+    if idx_num & 6 == 6 {
+        unsafe {
+            (*index_info).estimatedCost = 1.0;
+            (*index_info).estimatedRows = 1;
+        }
+    }
+    // only the version constraint is present
+    else if idx_num & 2 == 2 {
+        unsafe {
+            (*index_info).estimatedCost = 10.0;
+            (*index_info).estimatedRows = 10;
+        }
+    }
+    // only the requestor constraint is present
+    else if idx_num & 4 == 4 {
+        unsafe {
+            (*index_info).estimatedCost = 2147483647.0;
+            (*index_info).estimatedRows = 2147483647;
+        }
+    }
+    // no constraints are present
+    else {
+        unsafe {
+            (*index_info).estimatedCost = 2147483647.0;
+            (*index_info).estimatedRows = 2147483647;
+        }
+    }
+
+    unsafe {
+        (*index_info).idxNum = idx_num;
+        (*index_info).orderByConsumed = if order_by_consumed { 1 } else { 0 };
+        // forget str
+        let (ptr, _, _) = str.into_raw_parts();
+        // pass to c. We've manually null terminated the string.
+        // sqlite will free it for us.
+        (*index_info).idxStr = ptr as *mut c_char;
+        (*index_info).needToFreeIdxStr = 1;
+    }
+
+    Ok(ResultCode::OK)
+}
+
+fn constraint_is_usable(constraint: &sqlite::index_constraint) -> bool {
+    if constraint.usable == 0 {
+        return false;
+    }
+    if let Some(col) = CrsqlChangesColumn::from_i32(constraint.iColumn) {
+        match col {
+            CrsqlChangesColumn::Tbl | CrsqlChangesColumn::Pk | CrsqlChangesColumn::Cval => false,
+            _ => true,
+        }
+    } else {
+        false
+    }
+}
+
+// Note: this is really the col name post-select from the clock table.
+fn get_clock_table_col_name(col: &Option<CrsqlChangesColumn>) -> Option<String> {
+    match col {
+        Some(CrsqlChangesColumn::Tbl) => Some("tbl".to_string()),
+        Some(CrsqlChangesColumn::Pk) => Some("pks".to_string()),
+        Some(CrsqlChangesColumn::Cid) => Some("cid".to_string()),
+        Some(CrsqlChangesColumn::Cval) => None,
+        Some(CrsqlChangesColumn::ColVrsn) => Some("col_vrsn".to_string()),
+        Some(CrsqlChangesColumn::DbVrsn) => Some("db_vrsn".to_string()),
+        Some(CrsqlChangesColumn::SiteId) => Some("site_id".to_string()),
+        Some(CrsqlChangesColumn::Seq) => Some("seq".to_string()),
+        Some(CrsqlChangesColumn::Cl) => Some("cl".to_string()),
+        None => None,
+    }
+}
+
+fn get_operator_string(op: u8) -> Option<String> {
+    // TODO: convert to proper enum
+    match op as u32 {
+        sqlite::INDEX_CONSTRAINT_EQ => Some("=".to_string()),
+        sqlite::INDEX_CONSTRAINT_GT => Some(">".to_string()),
+        sqlite::INDEX_CONSTRAINT_LE => Some("<=".to_string()),
+        sqlite::INDEX_CONSTRAINT_LT => Some("<".to_string()),
+        sqlite::INDEX_CONSTRAINT_GE => Some(">=".to_string()),
+        sqlite::INDEX_CONSTRAINT_MATCH => Some("MATCH".to_string()),
+        sqlite::INDEX_CONSTRAINT_LIKE => Some("LIKE".to_string()),
+        sqlite::INDEX_CONSTRAINT_GLOB => Some("GLOB".to_string()),
+        sqlite::INDEX_CONSTRAINT_REGEXP => Some("REGEXP".to_string()),
+        sqlite::INDEX_CONSTRAINT_NE => Some("!=".to_string()),
+        sqlite::INDEX_CONSTRAINT_ISNOT => Some("IS NOT".to_string()),
+        sqlite::INDEX_CONSTRAINT_ISNOTNULL => Some("IS NOT NULL".to_string()),
+        sqlite::INDEX_CONSTRAINT_ISNULL => Some("IS NULL".to_string()),
+        sqlite::INDEX_CONSTRAINT_IS => Some("IS".to_string()),
+        _ => None,
+    }
+}
+
+// This'll become safe once more code is moved over to Rust
+#[no_mangle]
+pub unsafe extern "C" fn crsql_changes_filter(
+    cursor: *mut sqlite::vtab_cursor,
+    _idx_num: c_int,
+    idx_str: *const c_char,
+    argc: c_int,
+    argv: *mut *mut sqlite::value,
+) -> c_int {
+    let args = sqlite::args!(argc, argv);
+    let cursor = cursor.cast::<crsql_Changes_cursor>();
+    let idx_str = unsafe { CStr::from_ptr(idx_str).to_str() };
+    match idx_str {
+        Ok(idx_str) => match changes_filter(cursor, idx_str, args) {
+            Err(rc) | Ok(rc) => rc as c_int,
+        },
+        Err(_) => ResultCode::FORMAT as c_int,
+    }
+}
+
+unsafe fn changes_filter(
+    cursor: *mut crsql_Changes_cursor,
+    idx_str: &str,
+    args: &[*mut sqlite::value],
+) -> Result<ResultCode, ResultCode> {
+    let tab = (*cursor).pTab;
+    let db = (*tab).db;
+    // This should never happen. pChangesStmt should be finalized
+    // before filter is ever invoked.
+    if !(*cursor).pChangesStmt.is_null() {
+        (*cursor).pChangesStmt.finalize()?;
+        (*cursor).pChangesStmt = null_mut();
+    }
+
+    let c_rc =
+        crsql_ensureTableInfosAreUpToDate(db, (*tab).pExtData, &mut (*tab).base.zErrMsg as *mut _);
+    if c_rc != 0 {
+        if let Some(rc) = ResultCode::from_i32(c_rc) {
+            return Err(rc);
+        } else {
+            return Err(ResultCode::ERROR);
+        }
+    }
+
+    // nothing to fetch, no crrs exist.
+    if (*(*tab).pExtData).tableInfosLen == 0 {
+        return Ok(ResultCode::OK);
+    }
+
+    let table_infos = sqlite::args!(
+        (*(*tab).pExtData).tableInfosLen,
+        (*(*tab).pExtData).zpTableInfos
+    );
+    let sql = changes_union_query(table_infos, idx_str)?;
+
+    let stmt = db.prepare_v2(&sql)?;
+    for (i, arg) in args.iter().enumerate() {
+        stmt.bind_value(i as i32 + 1, *arg)?;
+    }
+    (*cursor).pChangesStmt = stmt.stmt;
+    // forget the stmt. it will be managed by the vtab
+    forget(stmt);
+    changes_next(cursor, (*cursor).pTab.cast::<sqlite::vtab>())
+}
+
+/**
+ * Advances our Changes_cursor to its next row of output.
+ * TODO: this'll get more idiomatic as we move dependencies to Rust
+ */
+#[no_mangle]
+pub unsafe extern "C" fn crsql_changes_next(cursor: *mut sqlite::vtab_cursor) -> c_int {
+    let cursor = cursor.cast::<crsql_Changes_cursor>();
+    let vtab = (*cursor).pTab.cast::<sqlite::vtab>();
+    match changes_next(cursor, vtab) {
+        Ok(rc) => rc as c_int,
+        Err(rc) => {
+            changes_crsr_finalize(cursor);
+            rc as c_int
+        }
+    }
+}
+
+// We'll get more idiomatic once we have more Rust and less C
+unsafe fn changes_next(
+    cursor: *mut crsql_Changes_cursor,
+    vtab: *mut sqlite::vtab,
+) -> Result<ResultCode, ResultCode> {
+    if (*cursor).pChangesStmt.is_null() {
+        let err = CString::new("pChangesStmt is null in changes_next")?;
+        (*vtab).zErrMsg = err.into_raw();
+        return Err(ResultCode::ABORT);
+    }
+
+    if !(*cursor).pRowStmt.is_null() {
+        let rc = reset_cached_stmt((*cursor).pRowStmt);
+        (*cursor).pRowStmt = null_mut();
+        if rc.is_err() {
+            return rc;
+        }
+    }
+
+    let rc = (*cursor).pChangesStmt.step()?;
+    if rc == ResultCode::DONE {
+        let c_rc = changes_crsr_finalize(cursor);
+        if c_rc == 0 {
+            return Ok(ResultCode::OK);
+        } else {
+            return Err(ResultCode::ERROR);
+        }
+    }
+
+    // we had a row... we can do the rest
+    let tbl = (*cursor)
+        .pChangesStmt
+        .column_text(ClockUnionColumn::Tbl as i32);
+    let pks = (*cursor)
+        .pChangesStmt
+        .column_value(ClockUnionColumn::Pks as i32);
+    let cid = (*cursor)
+        .pChangesStmt
+        .column_text(ClockUnionColumn::Cid as i32);
+    let db_version = (*cursor)
+        .pChangesStmt
+        .column_int64(ClockUnionColumn::DbVrsn as i32);
+    let changes_rowid = (*cursor)
+        .pChangesStmt
+        .column_int64(ClockUnionColumn::RowId as i32);
+    (*cursor).dbVersion = db_version;
+
+    let tbl_info_index = crate::c::crsql_indexofTableInfo(
+        (*(*(*cursor).pTab).pExtData).zpTableInfos,
+        (*(*(*cursor).pTab).pExtData).tableInfosLen,
+        // this should be safe since the underlying memory from column_text is null terminated at slice_len + 1.
+        tbl.as_ptr() as *const c_char,
+    );
+
+    if tbl_info_index < 0 {
+        let err = CString::new(format!("could not find schema for table {}", tbl))?;
+        (*vtab).zErrMsg = err.into_raw();
+        return Err(ResultCode::ERROR);
+    }
+
+    let tbl_infos = sqlite::args!(
+        (*(*(*cursor).pTab).pExtData).tableInfosLen,
+        (*(*(*cursor).pTab).pExtData).zpTableInfos
+    );
+    let tbl_info = tbl_infos[tbl_info_index as usize];
+    (*cursor).changesRowid = changes_rowid;
+    (*cursor).tblInfoIdx = tbl_info_index;
+
+    if (*tbl_info).pksLen == 0 {
+        let err = CString::new(format!("crr {} is missing primary keys", tbl))?;
+        (*vtab).zErrMsg = err.into_raw();
+        return Err(ResultCode::ERROR);
+    }
+
+    if cid == crate::c::DELETE_SENTINEL {
+        (*cursor).rowType = ChangeRowType::Delete as c_int;
+        return Ok(ResultCode::OK);
+    } else if cid == crate::c::INSERT_SENTINEL {
+        (*cursor).rowType = ChangeRowType::PkOnly as c_int;
+        return Ok(ResultCode::OK);
+    } else {
+        (*cursor).rowType = ChangeRowType::Update as c_int;
+    }
+
+    let stmt_key = get_cache_key(CachedStmtType::RowPatchData, tbl, Some(cid))?;
+    let mut row_stmt = if let Some(stmt) = get_cached_stmt((*(*cursor).pTab).pExtData, &stmt_key) {
+        stmt
+    } else {
+        null_mut()
+    };
+
+    if row_stmt.is_null() {
+        let sql = row_patch_data_query(tbl_info, cid);
+        if let Some(sql) = sql {
+            let stmt = (*(*cursor).pTab)
+                .db
+                .prepare_v3(&sql, sqlite::PREPARE_PERSISTENT)?;
+            // the cache takes ownership of stmt and stmt_key
+            set_cached_stmt((*(*cursor).pTab).pExtData, stmt_key, stmt.stmt);
+            row_stmt = stmt.stmt;
+            forget(stmt);
+        } else {
+            let err = CString::new(format!(
+                "could not generate row data fetch query for {}",
+                tbl
+            ))?;
+            (*vtab).zErrMsg = err.into_raw();
+            return Err(ResultCode::ERROR);
+        }
+    }
+
+    let packed_pks = pks.blob();
+    let unpacked_pks = unpack_columns(packed_pks)?;
+    bind_package_to_stmt(row_stmt, &unpacked_pks, 0)?;
+
+    match row_stmt.step() {
+        Ok(ResultCode::DONE) => {
+            reset_cached_stmt(row_stmt)?;
+        }
+        Ok(_) => {}
+        Err(rc) => {
+            reset_cached_stmt(row_stmt)?;
+            return Err(rc);
+        }
+    }
+
+    (*cursor).pRowStmt = row_stmt;
+    Ok(ResultCode::OK)
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_changes_eof(cursor: *mut sqlite::vtab_cursor) -> c_int {
+    let cursor = cursor.cast::<crsql_Changes_cursor>();
+    if unsafe { (*cursor).pChangesStmt.is_null() } {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_changes_column(
+    cursor: *mut sqlite::vtab_cursor, /* The cursor */
+    ctx: *mut sqlite::context,        /* First argument to sqlite3_result_...() */
+    i: c_int,                         /* Which column to return */
+) -> c_int {
+    match column_impl(cursor, ctx, i) {
+        Ok(code) | Err(code) => code as c_int,
+    }
+}
+
+fn column_impl(
+    cursor: *mut sqlite::vtab_cursor,
+    ctx: *mut sqlite::context,
+    i: c_int,
+) -> Result<ResultCode, ResultCode> {
+    let cursor = cursor.cast::<crsql_Changes_cursor>();
+    let column = CrsqlChangesColumn::from_i32(i);
+    // TODO: only de-reference where needed?
+    let changes_stmt = unsafe { (*cursor).pChangesStmt };
+    match column {
+        Some(CrsqlChangesColumn::Tbl) => {
+            ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Tbl as i32));
+        }
+        Some(CrsqlChangesColumn::Pk) => {
+            ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Pks as i32));
+        }
+        Some(CrsqlChangesColumn::Cval) => unsafe {
+            if (*cursor).pRowStmt.is_null() {
+                ctx.result_null();
+            } else {
+                ctx.result_value((*cursor).pRowStmt.column_value(0));
+            }
+        },
+        Some(CrsqlChangesColumn::Cid) => unsafe {
+            let row_type = ChangeRowType::from_i32((*cursor).rowType);
+            match row_type {
+                Some(ChangeRowType::PkOnly) => ctx.result_text_static(crate::c::INSERT_SENTINEL),
+                Some(ChangeRowType::Delete) => ctx.result_text_static(crate::c::DELETE_SENTINEL),
+                Some(ChangeRowType::Update) => {
+                    if (*cursor).pRowStmt.is_null() {
+                        ctx.result_text_static(crate::c::DELETE_SENTINEL);
+                    } else {
+                        ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Cid as i32));
+                    }
+                }
+                None => return Err(ResultCode::ABORT),
+            }
+        },
+        Some(CrsqlChangesColumn::ColVrsn) => {
+            ctx.result_value(changes_stmt.column_value(ClockUnionColumn::ColVrsn as i32));
+        }
+        Some(CrsqlChangesColumn::DbVrsn) => {
+            ctx.result_value(changes_stmt.column_value(ClockUnionColumn::DbVrsn as i32));
+        }
+        Some(CrsqlChangesColumn::SiteId) => {
+            // todo: short circuit null? if col type null bind null rather than value?
+            // sholdn't matter..
+            ctx.result_value(changes_stmt.column_value(ClockUnionColumn::SiteId as i32));
+        }
+        Some(CrsqlChangesColumn::Seq) => {
+            ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Seq as i32));
+        }
+        Some(CrsqlChangesColumn::Cl) => {
+            ctx.result_value(changes_stmt.column_value(ClockUnionColumn::Cl as i32))
+        }
+        None => return Err(ResultCode::MISUSE),
+    }
+
+    Ok(ResultCode::OK)
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_changes_rowid(
+    cursor: *mut sqlite::vtab_cursor,
+    rowid: *mut sqlite::int64,
+) -> c_int {
+    let cursor = cursor.cast::<crsql_Changes_cursor>();
+    unsafe {
+        *rowid = crate::util::slab_rowid((*cursor).tblInfoIdx, (*cursor).changesRowid);
+        if *rowid < 0 {
+            return ResultCode::ERROR as c_int;
+        }
+    }
+    return ResultCode::OK as c_int;
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_changes_update(
+    vtab: *mut sqlite::vtab,
+    argc: c_int,
+    argv: *mut *mut sqlite::value,
+    row_id: *mut sqlite::int64,
+) -> c_int {
+    let args = sqlite::args!(argc, argv);
+    let arg = args[0];
+    if args.len() > 1 && arg.value_type() == ColumnType::Null {
+        // insert statement
+        // argv[1] is the rowid.. but why would it ever be filled for us?
+        let mut err_msg = null_mut();
+        let rc = unsafe { crsql_merge_insert(vtab, argc, argv, row_id, &mut err_msg as *mut _) };
+        if rc != ResultCode::OK as c_int {
+            unsafe {
+                (*vtab).zErrMsg = err_msg;
+            }
+        }
+        return rc;
+    } else {
+        if let Ok(err) = CString::new(
+            "Only INSERT and SELECT statements are allowed against the crsql changes table",
+        ) {
+            unsafe {
+                (*vtab).zErrMsg = err.into_raw();
+            }
+            return ResultCode::MISUSE as c_int;
+        } else {
+            return ResultCode::NOMEM as c_int;
+        }
+    }
+}
+
+// If xBegin is not defined xCommit is not called.
+#[no_mangle]
+pub extern "C" fn crsql_changes_begin(_vtab: *mut sqlite::vtab) -> c_int {
+    ResultCode::OK as c_int
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_changes_commit(vtab: *mut sqlite::vtab) -> c_int {
+    let tab = vtab.cast::<crsql_Changes_vtab>();
+    unsafe {
+        (*(*tab).pExtData).rowsImpacted = 0;
+    }
+    ResultCode::OK as c_int
+}

libsql-sqlite3/ext/crr/rs/core/src/changes_vtab_read.rs

@@ -0,0 +1,129 @@
+extern crate alloc;
+use crate::{c::crsql_TableInfo, util};
+use alloc::format;
+use alloc::string::String;
+use alloc::vec;
+use core::{
+    ffi::{c_char, c_int, CStr},
+    ptr::null_mut,
+    slice,
+};
+use sqlite::ResultCode;
+
+use sqlite_nostd as sqlite;
+
+fn crsql_changes_query_for_table(table_info: *mut crsql_TableInfo) -> Result<String, ResultCode> {
+    unsafe {
+        if (*table_info).pksLen == 0 {
+            // no primary keys? We can't get changes for a table w/o primary keys...
+            // this should be an impossible case.
+            return Err(ResultCode::ABORT);
+        }
+    }
+
+    let table_name = unsafe { CStr::from_ptr((*table_info).tblName).to_str()? };
+    let pk_columns =
+        unsafe { slice::from_raw_parts((*table_info).pks, (*table_info).pksLen as usize) };
+    let pk_list = crate::util::as_identifier_list(pk_columns, Some("t1."))?;
+    let self_join = util::map_columns(pk_columns, |c| {
+        format!("t1.\"{c}\" = t2.\"{c}\"", c = crate::util::escape_ident(c))
+    })?
+    .join(" AND ");
+
+    // We LEFT JOIN and COALESCE the causal length
+    // since we incorporated an optimization to not store causal length records
+    // until they're required. I.e., do not store them until a delete
+    // is actually issued. This cuts data weight quite a bit for
+    // rows that never get removed.
+    Ok(format!(
+        "SELECT
+          '{table_name_val}' as tbl,
+          crsql_pack_columns({pk_list}) as pks,
+          t1.__crsql_col_name as cid,
+          t1.__crsql_col_version as col_vrsn,
+          t1.__crsql_db_version as db_vrsn,
+          t3.site_id as site_id,
+          t1._rowid_,
+          t1.__crsql_seq as seq,
+          COALESCE(t2.__crsql_col_version, 1) as cl
+      FROM \"{table_name_ident}__crsql_clock\" AS t1 LEFT JOIN \"{table_name_ident}__crsql_clock\" AS t2 ON
+      {self_join} AND t2.__crsql_col_name = '{sentinel}' LEFT JOIN crsql_site_id as t3 ON t1.__crsql_site_id = t3.ordinal",
+        table_name_val = crate::util::escape_ident_as_value(table_name),
+        pk_list = pk_list,
+        table_name_ident = crate::util::escape_ident(table_name),
+        sentinel = crate::c::INSERT_SENTINEL,
+        self_join = self_join
+    ))
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_changes_union_query(
+    table_infos: *mut *mut crsql_TableInfo,
+    table_infos_len: c_int,
+    idx_str: *const c_char,
+) -> *mut c_char {
+    if let Ok(idx_str) = unsafe { CStr::from_ptr(idx_str).to_str() } {
+        let table_infos = sqlite::args!(table_infos_len, table_infos);
+        let query = changes_union_query(table_infos, idx_str);
+        if let Ok(query) = query {
+            // release ownership of the memory
+            let (ptr, _, _) = query.into_raw_parts();
+            // return to c
+            return ptr as *mut c_char;
+        }
+    }
+    return core::ptr::null_mut() as *mut c_char;
+}
+
+pub fn changes_union_query(
+    table_infos: &[*mut crsql_TableInfo],
+    idx_str: &str,
+) -> Result<String, ResultCode> {
+    let mut sub_queries = vec![];
+
+    for table_info in table_infos {
+        let query_part = crsql_changes_query_for_table(*table_info)?;
+        sub_queries.push(query_part);
+    }
+
+    // Manually null-terminate the string so we don't have to copy it to create a CString.
+    // We can just extract the raw bytes of the Rust string.
+    return Ok(format!(
+      "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id, _rowid_, seq, cl FROM ({unions}) {idx_str}\0",
+      unions = sub_queries.join(" UNION ALL "),
+      idx_str = idx_str,
+    ));
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_row_patch_data_query(
+    table_info: *mut crsql_TableInfo,
+    col_name: *const c_char,
+) -> *mut c_char {
+    if let Ok(col_name) = unsafe { CStr::from_ptr(col_name).to_str() } {
+        if let Some(query) = row_patch_data_query(table_info, col_name) {
+            let (ptr, _, _) = query.into_raw_parts();
+            // release ownership of the memory
+            // return to c
+            return ptr as *mut c_char;
+        }
+    }
+    return null_mut();
+}
+
+pub fn row_patch_data_query(table_info: *mut crsql_TableInfo, col_name: &str) -> Option<String> {
+    let pk_columns =
+        unsafe { slice::from_raw_parts((*table_info).pks, (*table_info).pksLen as usize) };
+    if let Ok(table_name) = unsafe { CStr::from_ptr((*table_info).tblName).to_str() } {
+        if let Ok(where_list) = crate::util::where_list(pk_columns, None) {
+            return Some(format!(
+                "SELECT \"{col_name}\" FROM \"{table_name}\" WHERE {where_list}\0",
+                col_name = crate::util::escape_ident(col_name),
+                table_name = crate::util::escape_ident(table_name),
+                where_list = where_list
+            ));
+        }
+    }
+
+    return None;
+}

libsql-sqlite3/ext/crr/rs/core/src/changes_vtab_write.rs

@@ -0,0 +1,819 @@
+use alloc::ffi::CString;
+use alloc::format;
+use alloc::string::String;
+use alloc::vec::Vec;
+use core::ffi::{c_char, c_int, CStr};
+use core::mem::forget;
+use core::ptr::null_mut;
+use core::slice;
+use sqlite::{Connection, Stmt};
+use sqlite_nostd as sqlite;
+use sqlite_nostd::{sqlite3, ResultCode, Value};
+
+use crate::c::crsql_ExtData;
+use crate::c::{
+    crsql_Changes_vtab, crsql_TableInfo, crsql_ensureTableInfosAreUpToDate, crsql_indexofTableInfo,
+    CrsqlChangesColumn,
+};
+use crate::compare_values::crsql_compare_sqlite_values;
+use crate::pack_columns::bind_package_to_stmt;
+use crate::stmt_cache::{
+    get_cache_key, get_cached_stmt, reset_cached_stmt, set_cached_stmt, CachedStmtType,
+};
+use crate::util::{self, slab_rowid};
+use crate::{unpack_columns, ColumnValue};
+
+fn pk_where_list_from_tbl_info(
+    tbl_info: *mut crsql_TableInfo,
+    prefix: Option<&str>,
+) -> Result<String, core::str::Utf8Error> {
+    let pk_cols = sqlite::args!((*tbl_info).pksLen, (*tbl_info).pks);
+    util::where_list(pk_cols, prefix)
+}
+
+/**
+ * did_cid_win does not take into account the causal length.
+ * The expectation is that all cuasal length concerns have already been handle
+ * via:
+ * - early return because insert_cl < local_cl
+ * - automatic win because insert_cl > local_cl
+ * - come here to did_cid_win iff insert_cl = local_cl
+ */
+fn did_cid_win(
+    db: *mut sqlite3,
+    ext_data: *mut crsql_ExtData,
+    insert_tbl: &str,
+    tbl_info: *mut crsql_TableInfo,
+    unpacked_pks: &Vec<ColumnValue>,
+    col_name: &str,
+    insert_val: *mut sqlite::value,
+    col_version: sqlite::int64,
+    errmsg: *mut *mut c_char,
+) -> Result<bool, ResultCode> {
+    let stmt_key = get_cache_key(CachedStmtType::GetColVersion, insert_tbl, None)?;
+    let col_vrsn_stmt = get_cached_stmt_rt_wt(db, ext_data, stmt_key, || {
+        Ok(format!(
+        "SELECT __crsql_col_version FROM \"{table_name}__crsql_clock\" WHERE {pk_where_list} AND ? = __crsql_col_name",
+        table_name = crate::util::escape_ident(insert_tbl),
+        pk_where_list = pk_where_list_from_tbl_info(tbl_info, None)?,
+      ))
+    })?;
+
+    let bind_result = bind_package_to_stmt(col_vrsn_stmt, &unpacked_pks, 0);
+    if let Err(rc) = bind_result {
+        reset_cached_stmt(col_vrsn_stmt)?;
+        return Err(rc);
+    }
+    if let Err(rc) = col_vrsn_stmt.bind_text(
+        unpacked_pks.len() as i32 + 1,
+        col_name,
+        sqlite::Destructor::STATIC,
+    ) {
+        reset_cached_stmt(col_vrsn_stmt)?;
+        return Err(rc);
+    }
+
+    match col_vrsn_stmt.step() {
+        Ok(ResultCode::ROW) => {
+            let local_version = col_vrsn_stmt.column_int64(0);
+            reset_cached_stmt(col_vrsn_stmt)?;
+            // causal lengths are the same. Fall back to original algorithm.
+            if col_version > local_version {
+                return Ok(true);
+            } else if col_version < local_version {
+                return Ok(false);
+            }
+        }
+        Ok(ResultCode::DONE) => {
+            reset_cached_stmt(col_vrsn_stmt)?;
+            // no rows returned
+            // of course the incoming change wins if there's nothing there locally.
+            return Ok(true);
+        }
+        Ok(rc) | Err(rc) => {
+            reset_cached_stmt(col_vrsn_stmt)?;
+            let err = CString::new("Bad return code when selecting local column version")?;
+            unsafe { *errmsg = err.into_raw() };
+            return Err(rc);
+        }
+    }
+
+    // versions are equal
+    // need to pull the current value and compare
+    // we could compare on site_id if we can guarantee site_id is always provided.
+    // would be slightly more performant..
+    let stmt_key = get_cache_key(CachedStmtType::GetCurrValue, insert_tbl, Some(col_name))?;
+    let col_val_stmt = get_cached_stmt_rt_wt(db, ext_data, stmt_key, || {
+        Ok(format!(
+            "SELECT \"{col_name}\" FROM \"{table_name}\" WHERE {pk_where_list}",
+            col_name = crate::util::escape_ident(col_name),
+            table_name = crate::util::escape_ident(insert_tbl),
+            pk_where_list = pk_where_list_from_tbl_info(tbl_info, None)?,
+        ))
+    })?;
+
+    let bind_result = bind_package_to_stmt(col_val_stmt, &unpacked_pks, 0);
+    if let Err(rc) = bind_result {
+        reset_cached_stmt(col_val_stmt)?;
+        return Err(rc);
+    }
+
+    let step_result = col_val_stmt.step();
+    match step_result {
+        Ok(ResultCode::ROW) => {
+            let local_value = col_val_stmt.column_value(0);
+            let ret = crsql_compare_sqlite_values(insert_val, local_value);
+            reset_cached_stmt(col_val_stmt)?;
+            return Ok(ret > 0);
+        }
+        _ => {
+            // ResultCode::DONE would happen if clock values exist but actual values are missing.
+            // should we just allow the insert anyway?
+            reset_cached_stmt(col_val_stmt)?;
+            let err = CString::new(format!(
+                "could not find row to merge with for tbl {}",
+                insert_tbl
+            ))?;
+            unsafe { *errmsg = err.into_raw() };
+            return Err(ResultCode::ERROR);
+        }
+    }
+}
+
+fn get_cached_stmt_rt_wt<F>(
+    db: *mut sqlite::sqlite3,
+    ext_data: *mut crsql_ExtData,
+    key: String,
+    query_builder: F,
+) -> Result<*mut sqlite::stmt, ResultCode>
+where
+    F: Fn() -> Result<String, core::str::Utf8Error>,
+{
+    let mut ret = if let Some(stmt) = get_cached_stmt(ext_data, &key) {
+        stmt
+    } else {
+        null_mut()
+    };
+    if ret.is_null() {
+        let sql = query_builder()?;
+        if let Ok(stmt) = db.prepare_v3(&sql, sqlite::PREPARE_PERSISTENT) {
+            set_cached_stmt(ext_data, key, stmt.stmt);
+            ret = stmt.stmt;
+            forget(stmt);
+        } else {
+            return Err(ResultCode::ERROR);
+        }
+    }
+
+    Ok(ret)
+}
+
+fn set_winner_clock(
+    db: *mut sqlite3,
+    ext_data: *mut crsql_ExtData,
+    tbl_info: *mut crsql_TableInfo,
+    unpacked_pks: &Vec<ColumnValue>,
+    insert_col_name: &str,
+    insert_col_vrsn: sqlite::int64,
+    insert_db_vrsn: sqlite::int64,
+    insert_site_id: &[u8],
+    insert_seq: sqlite::int64,
+) -> Result<sqlite::int64, ResultCode> {
+    let tbl_name_str = unsafe { CStr::from_ptr((*tbl_info).tblName).to_str()? };
+
+    // set the site_id ordinal
+    // get the returned ordinal
+    // use that in place of insert_site_id in the metadata table(s)
+
+    // on changes read, join to gather the proper site id.
+    let ordinal = unsafe {
+        if insert_site_id.is_empty() {
+            None
+        } else {
+            (*ext_data).pSelectSiteIdOrdinalStmt.bind_blob(
+                1,
+                insert_site_id,
+                sqlite::Destructor::STATIC,
+            )?;
+            let rc = (*ext_data).pSelectSiteIdOrdinalStmt.step()?;
+            if rc == ResultCode::ROW {
+                let ordinal = (*ext_data).pSelectSiteIdOrdinalStmt.column_int64(0);
+                (*ext_data).pSelectSiteIdOrdinalStmt.clear_bindings()?;
+                (*ext_data).pSelectSiteIdOrdinalStmt.reset()?;
+
+                Some(ordinal)
+            } else {
+                (*ext_data).pSelectSiteIdOrdinalStmt.clear_bindings()?;
+                (*ext_data).pSelectSiteIdOrdinalStmt.reset()?;
+                // site id had no ordinal yet.
+                // set one and return the ordinal.
+                (*ext_data).pSetSiteIdOrdinalStmt.bind_blob(
+                    1,
+                    insert_site_id,
+                    sqlite::Destructor::STATIC,
+                )?;
+                let rc = (*ext_data).pSetSiteIdOrdinalStmt.step()?;
+                if rc == ResultCode::DONE {
+                    (*ext_data).pSetSiteIdOrdinalStmt.clear_bindings()?;
+                    (*ext_data).pSetSiteIdOrdinalStmt.reset()?;
+                    return Err(ResultCode::ABORT);
+                }
+                let ordinal = (*ext_data).pSetSiteIdOrdinalStmt.column_int64(0);
+                (*ext_data).pSetSiteIdOrdinalStmt.clear_bindings()?;
+                (*ext_data).pSetSiteIdOrdinalStmt.reset()?;
+                Some(ordinal)
+            }
+        }
+    };
+
+    let stmt_key = get_cache_key(CachedStmtType::SetWinnerClock, tbl_name_str, None)?;
+
+    let set_stmt = get_cached_stmt_rt_wt(db, ext_data, stmt_key, || {
+        let pk_cols = sqlite::args!((*tbl_info).pksLen, (*tbl_info).pks);
+        Ok(format!(
+          "INSERT OR REPLACE INTO \"{table_name}__crsql_clock\"
+            ({pk_ident_list}, __crsql_col_name, __crsql_col_version, __crsql_db_version, __crsql_seq, __crsql_site_id)
+            VALUES (
+              {pk_bind_list},
+              ?,
+              ?,
+              crsql_next_db_version(?),
+              ?,
+              ?
+            ) RETURNING _rowid_",
+          table_name = crate::util::escape_ident(tbl_name_str),
+          pk_ident_list = crate::util::as_identifier_list(pk_cols, None)?,
+          pk_bind_list = crate::util::binding_list(pk_cols.len()),
+        ))
+    })?;
+
+    let bind_result = bind_package_to_stmt(set_stmt, unpacked_pks, 0);
+    if let Err(rc) = bind_result {
+        reset_cached_stmt(set_stmt)?;
+        return Err(rc);
+    }
+    let bind_result = set_stmt
+        .bind_text(
+            unpacked_pks.len() as i32 + 1,
+            insert_col_name,
+            sqlite::Destructor::STATIC,
+        )
+        .and_then(|_| set_stmt.bind_int64(unpacked_pks.len() as i32 + 2, insert_col_vrsn))
+        .and_then(|_| set_stmt.bind_int64(unpacked_pks.len() as i32 + 3, insert_db_vrsn))
+        .and_then(|_| set_stmt.bind_int64(unpacked_pks.len() as i32 + 4, insert_seq))
+        .and_then(|_| match ordinal {
+            Some(ordinal) => set_stmt.bind_int64(unpacked_pks.len() as i32 + 5, ordinal),
+            None => set_stmt.bind_null(unpacked_pks.len() as i32 + 5),
+        });
+
+    if let Err(rc) = bind_result {
+        reset_cached_stmt(set_stmt)?;
+        return Err(rc);
+    }
+
+    match set_stmt.step() {
+        Ok(ResultCode::ROW) => {
+            let rowid = set_stmt.column_int64(0);
+            reset_cached_stmt(set_stmt)?;
+            Ok(rowid)
+        }
+        _ => {
+            reset_cached_stmt(set_stmt)?;
+            Err(ResultCode::ERROR)
+        }
+    }
+}
+
+fn merge_sentinel_only_insert(
+    db: *mut sqlite3,
+    ext_data: *mut crsql_ExtData,
+    tbl_info: *mut crsql_TableInfo,
+    unpacked_pks: &Vec<ColumnValue>,
+    remote_col_vrsn: sqlite::int64,
+    remote_db_vsn: sqlite::int64,
+    remote_site_id: &[u8],
+    remote_seq: sqlite::int64,
+) -> Result<sqlite::int64, ResultCode> {
+    let tbl_name_str = unsafe { CStr::from_ptr((*tbl_info).tblName).to_str()? };
+
+    let stmt_key = get_cache_key(CachedStmtType::MergePkOnlyInsert, tbl_name_str, None)?;
+    let merge_stmt = get_cached_stmt_rt_wt(db, ext_data, stmt_key, || {
+        let pk_cols = sqlite::args!((*tbl_info).pksLen, (*tbl_info).pks);
+        Ok(format!(
+            "INSERT OR IGNORE INTO \"{table_name}\" ({pk_idents}) VALUES ({pk_bindings})",
+            table_name = crate::util::escape_ident(tbl_name_str),
+            pk_idents = crate::util::as_identifier_list(pk_cols, None)?,
+            pk_bindings = crate::util::binding_list(pk_cols.len()),
+        ))
+    })?;
+
+    let rc = bind_package_to_stmt(merge_stmt, unpacked_pks, 0);
+    if let Err(rc) = rc {
+        reset_cached_stmt(merge_stmt)?;
+        return Err(rc);
+    }
+    let rc = unsafe {
+        (*ext_data)
+            .pSetSyncBitStmt
+            .step()
+            .and_then(|_| (*ext_data).pSetSyncBitStmt.reset())
+            .and_then(|_| merge_stmt.step())
+    };
+
+    // TODO: report err?
+    let _ = reset_cached_stmt(merge_stmt);
+
+    let sync_rc = unsafe {
+        (*ext_data)
+            .pClearSyncBitStmt
+            .step()
+            .and_then(|_| (*ext_data).pClearSyncBitStmt.reset())
+    };
+
+    if let Err(sync_rc) = sync_rc {
+        return Err(sync_rc);
+    }
+    if let Err(rc) = rc {
+        return Err(rc);
+    }
+
+    if let Ok(_) = rc {
+        zero_clocks_on_resurrect(
+            db,
+            ext_data,
+            tbl_name_str,
+            tbl_info,
+            unpacked_pks,
+            remote_db_vsn,
+        )?;
+        return set_winner_clock(
+            db,
+            ext_data,
+            tbl_info,
+            unpacked_pks,
+            crate::c::INSERT_SENTINEL,
+            remote_col_vrsn,
+            remote_db_vsn,
+            remote_site_id,
+            remote_seq,
+        );
+    }
+
+    Ok(-1)
+}
+
+fn zero_clocks_on_resurrect(
+    db: *mut sqlite3,
+    ext_data: *mut crsql_ExtData,
+    table_name: &str,
+    tbl_info: *mut crsql_TableInfo,
+    unpacked_pks: &Vec<ColumnValue>,
+    insert_db_vrsn: sqlite::int64,
+) -> Result<ResultCode, ResultCode> {
+    let stmt_key = get_cache_key(CachedStmtType::ZeroClocksOnResurrect, table_name, None)?;
+    let zero_stmt = get_cached_stmt_rt_wt(db, ext_data, stmt_key, || {
+        Ok(format!(
+            "UPDATE \"{table_name}__crsql_clock\" SET __crsql_col_version = 0, __crsql_db_version = crsql_next_db_version(?) WHERE {pk_where_list} AND __crsql_col_name IS NOT '{sentinel}'",
+            table_name = crate::util::escape_ident(table_name),
+            pk_where_list = pk_where_list_from_tbl_info(tbl_info, None)?,
+            sentinel = crate::c::INSERT_SENTINEL
+        ))
+    })?;
+
+    if let Err(rc) = zero_stmt.bind_int64(1, insert_db_vrsn) {
+        reset_cached_stmt(zero_stmt)?;
+        return Err(rc);
+    }
+
+    if let Err(rc) = bind_package_to_stmt(zero_stmt, unpacked_pks, 1) {
+        reset_cached_stmt(zero_stmt)?;
+        return Err(rc);
+    }
+
+    if let Err(rc) = zero_stmt.step() {
+        reset_cached_stmt(zero_stmt)?;
+        return Err(rc);
+    }
+
+    reset_cached_stmt(zero_stmt)
+}
+
+unsafe fn merge_delete(
+    db: *mut sqlite3,
+    ext_data: *mut crsql_ExtData,
+    tbl_info: *mut crsql_TableInfo,
+    unpacked_pks: &Vec<ColumnValue>,
+    remote_col_vrsn: sqlite::int64,
+    remote_db_vrsn: sqlite::int64,
+    remote_site_id: &[u8],
+    remote_seq: sqlite::int64,
+) -> Result<sqlite::int64, ResultCode> {
+    let tbl_name_str = CStr::from_ptr((*tbl_info).tblName).to_str()?;
+    let stmt_key = get_cache_key(CachedStmtType::MergeDelete, tbl_name_str, None)?;
+    let delete_stmt = get_cached_stmt_rt_wt(db, ext_data, stmt_key, || {
+        Ok(format!(
+            "DELETE FROM \"{table_name}\" WHERE {pk_where_list}",
+            table_name = crate::util::escape_ident(tbl_name_str),
+            pk_where_list = pk_where_list_from_tbl_info(tbl_info, None)?
+        ))
+    })?;
+
+    if let Err(rc) = bind_package_to_stmt(delete_stmt, unpacked_pks, 0) {
+        reset_cached_stmt(delete_stmt)?;
+        return Err(rc);
+    }
+    let rc = (*ext_data)
+        .pSetSyncBitStmt
+        .step()
+        .and_then(|_| (*ext_data).pSetSyncBitStmt.reset())
+        .and_then(|_| delete_stmt.step());
+
+    reset_cached_stmt(delete_stmt)?;
+
+    let sync_rc = (*ext_data)
+        .pClearSyncBitStmt
+        .step()
+        .and_then(|_| (*ext_data).pClearSyncBitStmt.reset());
+
+    if let Err(sync_rc) = sync_rc {
+        return Err(sync_rc);
+    }
+    if let Err(rc) = rc {
+        return Err(rc);
+    }
+
+    let ret = set_winner_clock(
+        db,
+        ext_data,
+        tbl_info,
+        unpacked_pks,
+        crate::c::DELETE_SENTINEL,
+        remote_col_vrsn,
+        remote_db_vrsn,
+        remote_site_id,
+        remote_seq,
+    )?;
+
+    // Drop clocks _after_ setting the winner clock so we don't lose track of the max db_version!!
+    // This must never come before `set_winner_clock`
+    let stmt_key = get_cache_key(CachedStmtType::MergeDeleteDropClocks, tbl_name_str, None)?;
+    let drop_clocks_stmt = get_cached_stmt_rt_wt(db, ext_data, stmt_key, || {
+        Ok(format!(
+            "DELETE FROM \"{table_name}__crsql_clock\" WHERE {pk_where_list} AND __crsql_col_name IS NOT '{sentinel}'",
+            table_name = crate::util::escape_ident(tbl_name_str),
+            pk_where_list = pk_where_list_from_tbl_info(tbl_info, None)?,
+            sentinel = crate::c::DELETE_SENTINEL
+        ))
+    })?;
+
+    if let Err(rc) = bind_package_to_stmt(drop_clocks_stmt, unpacked_pks, 0) {
+        reset_cached_stmt(drop_clocks_stmt)?;
+        return Err(rc);
+    }
+
+    if let Err(rc) = drop_clocks_stmt.step() {
+        reset_cached_stmt(drop_clocks_stmt)?;
+        return Err(rc);
+    }
+
+    reset_cached_stmt(drop_clocks_stmt)?;
+
+    return Ok(ret);
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn crsql_merge_insert(
+    vtab: *mut sqlite::vtab,
+    argc: c_int,
+    argv: *mut *mut sqlite::value,
+    rowid: *mut sqlite::int64,
+    errmsg: *mut *mut c_char,
+) -> c_int {
+    match merge_insert(vtab, argc, argv, rowid, errmsg) {
+        Err(rc) | Ok(rc) => rc as c_int,
+    }
+}
+
+fn get_local_cl(
+    db: *mut sqlite::sqlite3,
+    ext_data: *mut crsql_ExtData,
+    tbl_name: &str,
+    tbl_info: *mut crsql_TableInfo,
+    unpacked_pks: &Vec<ColumnValue>,
+) -> Result<sqlite::int64, ResultCode> {
+    let stmt_key = get_cache_key(CachedStmtType::GetLocalCl, tbl_name, None)?;
+
+    let local_cl_stmt = get_cached_stmt_rt_wt(db, ext_data, stmt_key, || {
+        // We do an optimization to not store unnecessary create records.
+        // If a create record for the rows does not exist, see if any record does
+        // if a record does, the causal length is implicitly 1
+        Ok(format!(
+        "SELECT COALESCE(
+          (SELECT __crsql_col_version FROM \"{table_name}__crsql_clock\" WHERE {pk_where_list} AND __crsql_col_name = '{delete_sentinel}'),
+          (SELECT 1 FROM \"{table_name}__crsql_clock\" WHERE {pk_where_list})
+        )",
+        table_name = crate::util::escape_ident(tbl_name),
+        pk_where_list = pk_where_list_from_tbl_info(tbl_info, None)?,
+        delete_sentinel = crate::c::DELETE_SENTINEL,
+      ))
+    })?;
+
+    let rc = bind_package_to_stmt(local_cl_stmt, unpacked_pks, 0);
+    if let Err(rc) = rc {
+        reset_cached_stmt(local_cl_stmt)?;
+        return Err(rc);
+    }
+    let rc = bind_package_to_stmt(local_cl_stmt, unpacked_pks, unpacked_pks.len());
+    if let Err(rc) = rc {
+        reset_cached_stmt(local_cl_stmt)?;
+        return Err(rc);
+    }
+
+    let step_result = local_cl_stmt.step();
+    match step_result {
+        Ok(ResultCode::ROW) => {
+            let ret = local_cl_stmt.column_int64(0);
+            reset_cached_stmt(local_cl_stmt)?;
+            Ok(ret)
+        }
+        Ok(ResultCode::DONE) => {
+            reset_cached_stmt(local_cl_stmt)?;
+            Ok(0)
+        }
+        Ok(rc) | Err(rc) => {
+            reset_cached_stmt(local_cl_stmt)?;
+            Err(rc)
+        }
+    }
+}
+unsafe fn merge_insert(
+    vtab: *mut sqlite::vtab,
+    argc: c_int,
+    argv: *mut *mut sqlite::value,
+    rowid: *mut sqlite::int64,
+    errmsg: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    let tab = vtab.cast::<crsql_Changes_vtab>();
+    let db = (*tab).db;
+
+    let rc = crsql_ensureTableInfosAreUpToDate(db, (*tab).pExtData, errmsg);
+    if rc != ResultCode::OK as i32 {
+        let err = CString::new("Failed to update CRR table information")?;
+        *errmsg = err.into_raw();
+        return Err(ResultCode::ERROR);
+    }
+
+    let args = sqlite::args!(argc, argv);
+    let insert_tbl = args[2 + CrsqlChangesColumn::Tbl as usize];
+    if insert_tbl.bytes() > crate::consts::MAX_TBL_NAME_LEN {
+        let err = CString::new("crsql - table name exceeded max length")?;
+        *errmsg = err.into_raw();
+        return Err(ResultCode::ERROR);
+    }
+
+    let insert_tbl = insert_tbl.text();
+    let insert_pks = args[2 + CrsqlChangesColumn::Pk as usize];
+    let insert_col = args[2 + CrsqlChangesColumn::Cid as usize];
+    if insert_col.bytes() > crate::consts::MAX_TBL_NAME_LEN {
+        let err = CString::new("crsql - column name exceeded max length")?;
+        *errmsg = err.into_raw();
+        return Err(ResultCode::ERROR);
+    }
+
+    let insert_col = insert_col.text();
+    let insert_val = args[2 + CrsqlChangesColumn::Cval as usize];
+    let insert_col_vrsn = args[2 + CrsqlChangesColumn::ColVrsn as usize].int64();
+    let insert_db_vrsn = args[2 + CrsqlChangesColumn::DbVrsn as usize].int64();
+    let insert_site_id = args[2 + CrsqlChangesColumn::SiteId as usize];
+    let insert_cl = args[2 + CrsqlChangesColumn::Cl as usize].int64();
+    let insert_seq = args[2 + CrsqlChangesColumn::Seq as usize].int64();
+
+    if insert_site_id.bytes() > crate::consts::SITE_ID_LEN {
+        let err = CString::new("crsql - site id exceeded max length")?;
+        *errmsg = err.into_raw();
+        return Err(ResultCode::ERROR);
+    }
+
+    let insert_site_id = insert_site_id.blob();
+    let tbl_info_index = crsql_indexofTableInfo(
+        (*(*tab).pExtData).zpTableInfos,
+        (*(*tab).pExtData).tableInfosLen,
+        insert_tbl.as_ptr() as *const c_char,
+    );
+
+    let tbl_infos = sqlite::args!(
+        (*(*tab).pExtData).tableInfosLen,
+        (*(*tab).pExtData).zpTableInfos
+    );
+    if tbl_info_index == -1 {
+        let err = CString::new(format!(
+            "crsql - could not find the schema information for table {}",
+            insert_tbl
+        ))?;
+        *errmsg = err.into_raw();
+        return Err(ResultCode::ERROR);
+    }
+
+    let tbl_info = tbl_infos[tbl_info_index as usize];
+    let unpacked_pks = unpack_columns(insert_pks.blob())?;
+    let local_cl = get_local_cl(db, (*tab).pExtData, insert_tbl, tbl_info, &unpacked_pks)?;
+
+    // We can ignore all updates from older causal lengths.
+    // They won't win at anything.
+    if insert_cl < local_cl {
+        return Ok(ResultCode::OK);
+    }
+
+    let is_delete = insert_cl % 2 == 0;
+    // Resurrect or update to latest cl.
+    // The current node might have missed the delete preceeding this causal length
+    // in out-of-order delivery setups but we still call it a resurrect as special
+    // handling needs to happen in the "alive -> missed_delete -> alive" case.
+    let needs_resurrect = insert_cl > local_cl && insert_cl % 2 == 1;
+    let row_exists_locally = local_cl != 0;
+    let is_sentinel_only = crate::c::INSERT_SENTINEL == insert_col;
+
+    if is_delete {
+        // We got a delete event but we've already processed a delete at that version.
+        // Just bail.
+        if insert_cl == local_cl {
+            return Ok(ResultCode::OK);
+        }
+        // else, it is a delete and the cl is > than ours. Drop the row.
+        let merge_result = merge_delete(
+            db,
+            (*tab).pExtData,
+            tbl_info,
+            &unpacked_pks,
+            insert_col_vrsn,
+            insert_db_vrsn,
+            insert_site_id,
+            insert_seq,
+        );
+        match merge_result {
+            Err(rc) => {
+                return Err(rc);
+            }
+            Ok(inner_rowid) => {
+                (*(*tab).pExtData).rowsImpacted += 1;
+                *rowid = slab_rowid(tbl_info_index, inner_rowid);
+                return Ok(ResultCode::OK);
+            }
+        }
+    }
+
+    /*
+    || crsql_columnExists(
+            // TODO: only safe because we _know_ this is actually a cstr
+            insert_col.as_ptr() as *const c_char,
+            (*tbl_info).nonPks,
+            (*tbl_info).nonPksLen,
+        ) == 0
+     */
+    if is_sentinel_only {
+        // If it is a sentinel but the local_cl already matches, nothing to do
+        // as the local sentinel already has the same data!
+        if insert_cl == local_cl {
+            return Ok(ResultCode::OK);
+        }
+        let merge_result = merge_sentinel_only_insert(
+            db,
+            (*tab).pExtData,
+            tbl_info,
+            &unpacked_pks,
+            insert_col_vrsn,
+            insert_db_vrsn,
+            insert_site_id,
+            insert_seq,
+        );
+        match merge_result {
+            Err(rc) => {
+                return Err(rc);
+            }
+            Ok(inner_rowid) => {
+                // a success & rowid of -1 means the merge was a no-op
+                if inner_rowid != -1 {
+                    (*(*tab).pExtData).rowsImpacted += 1;
+                    *rowid = slab_rowid(tbl_info_index, inner_rowid);
+                    return Ok(ResultCode::OK);
+                } else {
+                    return Ok(ResultCode::OK);
+                }
+            }
+        }
+    }
+
+    // we got a causal length which would resurrect the row.
+    // In an in-order delivery situation then `sentinel_only` would have already resurrected the row
+    // In out-of-order delivery, we need to resurrect the row as soon as we get a value
+    // which should resurrect the row. I.e., don't wait on the sentinel value to resurrect the row!
+    if needs_resurrect && row_exists_locally {
+        // this should work -- same as `merge_sentinel_only_insert` except we're not done once we do it
+        // and the version to set to is the cl not col_vrsn of current insert
+        merge_sentinel_only_insert(
+            db,
+            (*tab).pExtData,
+            tbl_info,
+            &unpacked_pks,
+            insert_cl,
+            insert_db_vrsn,
+            insert_site_id,
+            insert_seq,
+        )?;
+        (*(*tab).pExtData).rowsImpacted += 1;
+    }
+
+    // we can short-circuit via needs_resurrect
+    // given the greater cl automatically means a win.
+    // or if we realize that the row does not exist locally at all.
+    let does_cid_win = needs_resurrect
+        || !row_exists_locally
+        || did_cid_win(
+            db,
+            (*tab).pExtData,
+            insert_tbl,
+            tbl_info,
+            &unpacked_pks,
+            insert_col,
+            insert_val,
+            insert_col_vrsn,
+            errmsg,
+        )?;
+
+    if does_cid_win == false {
+        // doesCidWin == 0? compared against our clocks, nothing wins. OK and
+        // Done.
+        return Ok(ResultCode::OK);
+    }
+
+    // TODO: this is all almost identical between all three merge cases!
+    let stmt_key = get_cache_key(
+        CachedStmtType::MergeInsert,
+        // This is currently safe since these are c strings under the hood
+        insert_tbl,
+        Some(insert_col),
+    )?;
+    let merge_stmt = get_cached_stmt_rt_wt(db, (*tab).pExtData, stmt_key, || {
+        let pk_cols = sqlite::args!((*tbl_info).pksLen, (*tbl_info).pks);
+        Ok(format!(
+            "INSERT INTO \"{table_name}\" ({pk_list}, \"{col_name}\")
+            VALUES ({pk_bind_list}, ?)
+            ON CONFLICT DO UPDATE
+            SET \"{col_name}\" = ?",
+            table_name = crate::util::escape_ident(insert_tbl),
+            pk_list = crate::util::as_identifier_list(pk_cols, None)?,
+            col_name = crate::util::escape_ident(insert_col),
+            pk_bind_list = crate::util::binding_list(pk_cols.len()),
+        ))
+    })?;
+
+    let bind_result = bind_package_to_stmt(merge_stmt, &unpacked_pks, 0)
+        .and_then(|_| merge_stmt.bind_value(unpacked_pks.len() as i32 + 1, insert_val))
+        .and_then(|_| merge_stmt.bind_value(unpacked_pks.len() as i32 + 2, insert_val));
+    if let Err(rc) = bind_result {
+        reset_cached_stmt(merge_stmt)?;
+        return Err(rc);
+    }
+
+    let rc = (*(*tab).pExtData)
+        .pSetSyncBitStmt
+        .step()
+        .and_then(|_| (*(*tab).pExtData).pSetSyncBitStmt.reset())
+        .and_then(|_| merge_stmt.step());
+
+    reset_cached_stmt(merge_stmt)?;
+
+    let sync_rc = (*(*tab).pExtData)
+        .pClearSyncBitStmt
+        .step()
+        .and_then(|_| (*(*tab).pExtData).pClearSyncBitStmt.reset());
+
+    if let Err(rc) = rc {
+        return Err(rc);
+    }
+    if let Err(sync_rc) = sync_rc {
+        return Err(sync_rc);
+    }
+
+    let merge_result = set_winner_clock(
+        db,
+        (*tab).pExtData,
+        tbl_info,
+        &unpacked_pks,
+        insert_col,
+        insert_col_vrsn,
+        insert_db_vrsn,
+        insert_site_id,
+        insert_seq,
+    );
+    match merge_result {
+        Err(rc) => {
+            return Err(rc);
+        }
+        Ok(inner_rowid) => {
+            (*(*tab).pExtData).rowsImpacted += 1;
+            *rowid = slab_rowid(tbl_info_index, inner_rowid);
+            return Ok(ResultCode::OK);
+        }
+    }
+}

libsql-sqlite3/ext/crr/rs/core/src/compare_values.rs

@@ -0,0 +1,45 @@
+use core::ffi::c_int;
+use sqlite::Value;
+use sqlite_nostd as sqlite;
+
+#[no_mangle]
+pub extern "C" fn crsql_compare_sqlite_values(
+    l: *mut sqlite::value,
+    r: *mut sqlite::value,
+) -> c_int {
+    let l_type = l.value_type();
+    let r_type = r.value_type();
+
+    if l_type != r_type {
+        // We swap the compare since we want null to be _less than_ all things
+        // and null is assigned to ordinal 5 (greatest thing).
+        return (r_type as i32) - (l_type as i32);
+    }
+
+    match l_type {
+        sqlite::ColumnType::Blob => l.blob().cmp(r.blob()) as c_int,
+        sqlite::ColumnType::Float => {
+            let l_double = l.double();
+            let r_double = r.double();
+            if l_double < r_double {
+                return -1;
+            } else if l_double > r_double {
+                return 1;
+            }
+            return 0;
+        }
+        sqlite::ColumnType::Integer => {
+            let l_int = l.int64();
+            let r_int = r.int64();
+            // no subtraction since that could overflow the c_int return type
+            if l_int < r_int {
+                return -1;
+            } else if l_int > r_int {
+                return 1;
+            }
+            return 0;
+        }
+        sqlite::ColumnType::Null => 0,
+        sqlite::ColumnType::Text => l.text().cmp(r.text()) as c_int,
+    }
+}

libsql-sqlite3/ext/crr/rs/core/src/consts.rs

@@ -0,0 +1,18 @@
+pub const TBL_SITE_ID: &'static str = "crsql_site_id";
+pub const TBL_SCHEMA: &'static str = "crsql_master";
+// pub const CRSQLITE_VERSION_0_15_0: i32 = 15_00_00;
+// pub const CRSQLITE_VERSION_0_13_0: i32 = 13_00_00;
+// MM_mm_pp_xx
+// so a 1.0.0 release is:
+// 01_00_00_00 -> 1000000
+// a 0.5 release is:
+// 00_05_00_00 ->   50000
+// a 0.5.1 is:
+// 00_05_01_00
+// and, if we ever need it, we can track individual builds of a patch release
+// 00_05_01_01
+pub const CRSQLITE_VERSION: i32 = 15_00_00;
+pub const SITE_ID_LEN: i32 = 16;
+pub const ROWID_SLAB_SIZE: i64 = 10000000000000;
+pub const MIN_POSSIBLE_DB_VERSION: i64 = 0;
+pub const MAX_TBL_NAME_LEN: i32 = 2048;

libsql-sqlite3/ext/crr/rs/core/src/create_cl_set_vtab.rs

@@ -0,0 +1,271 @@
+extern crate alloc;
+
+use core::ffi::{c_char, c_int, c_void};
+
+use crate::alloc::borrow::ToOwned;
+use crate::c::crsql_createCrr;
+use alloc::boxed::Box;
+use alloc::ffi::CString;
+use alloc::format;
+use alloc::string::String;
+use sqlite::{convert_rc, sqlite3, Connection, CursorRef, StrRef, VTabArgs, VTabRef};
+use sqlite_nostd as sqlite;
+use sqlite_nostd::ResultCode;
+
+// Virtual table definition to create a causal length set backed table.
+
+#[repr(C)]
+struct CLSetTab {
+    base: sqlite::vtab,
+    base_table_name: String,
+    db_name: String,
+    db: *mut sqlite3,
+}
+
+// used in response to `create virtual table ... using clset`
+extern "C" fn create(
+    db: *mut sqlite::sqlite3,
+    _aux: *mut c_void,
+    argc: c_int,
+    argv: *const *const c_char,
+    vtab: *mut *mut sqlite::vtab,
+    err: *mut *mut c_char,
+) -> c_int {
+    match create_impl(db, argc, argv, vtab, err) {
+        Ok(rc) => rc as c_int,
+        Err(rc) => {
+            // deallocate the vtab on error.
+            unsafe {
+                if *vtab != core::ptr::null_mut() {
+                    let tab = Box::from_raw((*vtab).cast::<CLSetTab>());
+                    drop(tab);
+                    *vtab = core::ptr::null_mut();
+                }
+            }
+            rc as c_int
+        }
+    }
+}
+
+fn create_impl(
+    db: *mut sqlite::sqlite3,
+    argc: c_int,
+    argv: *const *const c_char,
+    vtab: *mut *mut sqlite::vtab,
+    err: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    // This is the schema component
+    let vtab_args = sqlite::parse_vtab_args(argc, argv)?;
+    if !vtab_args.table_name.ends_with("_schema") {
+        err.set(&format!(
+          "{tbl_name} MUST end with _schema. E.g., {tbl_name}_schema. Two tables will be created: {tbl_name}_schema for managing the CRDT schemas and {tbl_name} for storing the data.",
+          tbl_name = vtab_args.table_name
+        ));
+        return Err(ResultCode::ERROR);
+    }
+    connect_create_shared(db, vtab, &vtab_args)?;
+
+    // We can't wrap this in a savepoint for some reason. I guess because the `CREATE VIRTUAL TABLE..`
+    // statement is processing? 🤷‍♂️
+    create_clset_storage(db, &vtab_args, err)?;
+    let db_name_c = CString::new(vtab_args.database_name)?;
+    let table_name_c = CString::new(base_name_from_virtual_name(vtab_args.table_name))?;
+
+    // TODO: move `createCrr` to Rust
+    let rc = unsafe { crsql_createCrr(db, db_name_c.as_ptr(), table_name_c.as_ptr(), 0, 1, err) };
+    convert_rc(rc)
+}
+
+fn create_clset_storage(
+    db: *mut sqlite::sqlite3,
+    args: &VTabArgs,
+    err: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    // Is the _last_ arg all the args? Or is it comma separated in some way?
+    // What about index definitions...
+    // Let the user later create them against the base table? Or via insertions into our vtab schema?
+    let table_def = args.arguments.join(",");
+    if !args.table_name.ends_with("_schema") {
+        err.set("CLSet virtual table names must end with `_schema`");
+        return Err(ResultCode::MISUSE);
+    }
+
+    db.exec_safe(&format!(
+        "CREATE TABLE \"{db_name}\".\"{table_name}\" ({table_def})",
+        db_name = crate::util::escape_ident(args.database_name),
+        table_name = crate::util::escape_ident(base_name_from_virtual_name(args.table_name)),
+        table_def = table_def
+    ))
+}
+
+fn base_name_from_virtual_name(virtual_name: &str) -> &str {
+    &virtual_name[0..(virtual_name.len() - "_schema".len())]
+}
+
+// connect to an existing virtual table previously created by `create virtual table`
+extern "C" fn connect(
+    db: *mut sqlite::sqlite3,
+    _aux: *mut c_void,
+    argc: c_int,
+    argv: *const *const c_char,
+    vtab: *mut *mut sqlite::vtab,
+    _err: *mut *mut c_char,
+) -> c_int {
+    let vtab_args = sqlite::parse_vtab_args(argc, argv);
+    match vtab_args {
+        Ok(vtab_args) => match connect_create_shared(db, vtab, &vtab_args) {
+            Ok(rc) | Err(rc) => rc as c_int,
+        },
+        Err(_e) => {
+            // free the tab if it was allocated
+            unsafe {
+                if *vtab != core::ptr::null_mut() {
+                    let tab = Box::from_raw((*vtab).cast::<CLSetTab>());
+                    drop(tab);
+                    *vtab = core::ptr::null_mut();
+                }
+            };
+            ResultCode::FORMAT as c_int
+        }
+    }
+}
+
+fn connect_create_shared(
+    db: *mut sqlite::sqlite3,
+    vtab: *mut *mut sqlite::vtab,
+    args: &VTabArgs,
+) -> Result<ResultCode, ResultCode> {
+    sqlite::declare_vtab(
+        db,
+        "CREATE TABLE x(alteration TEXT HIDDEN, schema TEXT HIDDEN);",
+    )?;
+    let tab = Box::new(CLSetTab {
+        base: sqlite::vtab {
+            nRef: 0,
+            pModule: core::ptr::null(),
+            zErrMsg: core::ptr::null_mut(),
+        },
+        base_table_name: base_name_from_virtual_name(args.table_name).to_owned(),
+        db_name: args.database_name.to_owned(),
+        db: db,
+    });
+    vtab.set(tab);
+    Ok(ResultCode::OK)
+}
+
+extern "C" fn best_index(_vtab: *mut sqlite::vtab, _index_info: *mut sqlite::index_info) -> c_int {
+    ResultCode::OK as c_int
+}
+
+extern "C" fn disconnect(vtab: *mut sqlite::vtab) -> c_int {
+    if vtab != core::ptr::null_mut() {
+        let tab = unsafe { Box::from_raw(vtab.cast::<CLSetTab>()) };
+        drop(tab);
+    }
+    ResultCode::OK as c_int
+}
+
+extern "C" fn destroy(vtab: *mut sqlite::vtab) -> c_int {
+    let tab = unsafe { Box::from_raw(vtab.cast::<CLSetTab>()) };
+    let ret = tab.db.exec_safe(&format!(
+        "DROP TABLE \"{db_name}\".\"{table_name}\";
+        DROP TABLE \"{db_name}\".\"{table_name}__crsql_clock\";",
+        table_name = crate::util::escape_ident(&tab.base_table_name),
+        db_name = crate::util::escape_ident(&tab.db_name)
+    ));
+    match ret {
+        Err(rc) | Ok(rc) => rc as c_int,
+    }
+}
+
+extern "C" fn open(_vtab: *mut sqlite::vtab, cursor: *mut *mut sqlite::vtab_cursor) -> c_int {
+    cursor.set(Box::new(sqlite::vtab_cursor {
+        pVtab: core::ptr::null_mut(),
+    }));
+    ResultCode::OK as c_int
+}
+
+extern "C" fn close(cursor: *mut sqlite::vtab_cursor) -> c_int {
+    unsafe {
+        drop(Box::from_raw(cursor));
+    }
+    ResultCode::OK as c_int
+}
+
+extern "C" fn filter(
+    _cursor: *mut sqlite::vtab_cursor,
+    _idx_num: c_int,
+    _idx_str: *const c_char,
+    _argc: c_int,
+    _argv: *mut *mut sqlite::value,
+) -> c_int {
+    ResultCode::OK as c_int
+}
+
+extern "C" fn next(_cursor: *mut sqlite::vtab_cursor) -> c_int {
+    ResultCode::OK as c_int
+}
+
+extern "C" fn eof(_cursor: *mut sqlite::vtab_cursor) -> c_int {
+    ResultCode::OK as c_int
+}
+
+extern "C" fn column(
+    _cursor: *mut sqlite::vtab_cursor,
+    _ctx: *mut sqlite::context,
+    _col_num: c_int,
+) -> c_int {
+    ResultCode::OK as c_int
+}
+
+extern "C" fn rowid(_cursor: *mut sqlite::vtab_cursor, _row_id: *mut sqlite::int64) -> c_int {
+    ResultCode::OK as c_int
+}
+
+extern "C" fn begin(_vtab: *mut sqlite::vtab) -> c_int {
+    ResultCode::OK as c_int
+}
+
+extern "C" fn commit(_vtab: *mut sqlite::vtab) -> c_int {
+    ResultCode::OK as c_int
+}
+
+extern "C" fn rollback(_vtab: *mut sqlite::vtab) -> c_int {
+    ResultCode::OK as c_int
+}
+
+static MODULE: sqlite_nostd::module = sqlite_nostd::module {
+    iVersion: 0,
+    xCreate: Some(create),
+    xConnect: Some(connect),
+    xBestIndex: Some(best_index),
+    xDisconnect: Some(disconnect),
+    xDestroy: Some(destroy),
+    xOpen: Some(open),
+    xClose: Some(close),
+    xFilter: Some(filter),
+    xNext: Some(next),
+    xEof: Some(eof),
+    xColumn: Some(column),
+    xRowid: Some(rowid),
+    xUpdate: None,
+    xBegin: Some(begin),
+    xSync: None,
+    xCommit: Some(commit),
+    xRollback: Some(rollback),
+    xFindFunction: None,
+    xRename: None,
+    xSavepoint: None,
+    xRelease: None,
+    xRollbackTo: None,
+    xShadowName: None,
+    xPreparedSql: None,
+};
+
+pub fn create_module(db: *mut sqlite::sqlite3) -> Result<ResultCode, ResultCode> {
+    db.create_module_v2("clset", &MODULE, None, None)?;
+
+    // xCreate(|x| 0);
+
+    Ok(ResultCode::OK)
+}

libsql-sqlite3/ext/crr/rs/core/src/lib.rs

@@ -1,27 +1,40 @@
 #![cfg_attr(not(test), no_std)]
+#![feature(vec_into_raw_parts)]
 
+mod alter;
 mod automigrate;
 mod backfill;
+mod bootstrap;
+mod c;
+mod changes_vtab;
+mod changes_vtab_read;
+mod changes_vtab_write;
+mod compare_values;
+mod consts;
+mod create_cl_set_vtab;
 mod is_crr;
+mod pack_columns;
+mod stmt_cache;
 mod teardown;
+mod triggers;
+mod unpack_columns_vtab;
+mod util;
 
 use core::{ffi::c_char, slice};
 extern crate alloc;
-use alloc::string::String;
 use alloc::vec::Vec;
 pub use automigrate::*;
 pub use backfill::*;
 use core::ffi::{c_int, CStr};
 pub use is_crr::*;
+use pack_columns::crsql_pack_columns;
+pub use pack_columns::unpack_columns;
+pub use pack_columns::ColumnValue;
 use sqlite::ResultCode;
 use sqlite_nostd as sqlite;
-use sqlite_nostd::{context, Connection, Context, Value};
+use sqlite_nostd::{Connection, Context, Value};
 pub use teardown::*;
 
-fn escape_ident(ident: &str) -> String {
-    return ident.replace("\"", "\"\"");
-}
-
 pub extern "C" fn crsql_as_table(
     ctx: *mut sqlite::context,
     argc: i32,
@@ -65,7 +78,7 @@ pub extern "C" fn sqlite3_crsqlcore_init(
     let rc = db
         .create_function_v2(
             "crsql_automigrate",
-            1,
+            -1,
             sqlite::UTF8,
             None,
             Some(crsql_automigrate),
@@ -78,27 +91,56 @@ pub extern "C" fn sqlite3_crsqlcore_init(
         return rc as c_int;
     }
 
-    db.create_function_v2(
-        "crsql_as_table",
-        1,
-        sqlite::UTF8,
-        None,
-        Some(crsql_as_table),
-        None,
-        None,
-        None,
-    )
-    .unwrap_or(sqlite::ResultCode::ERROR) as c_int
+    let rc = db
+        .create_function_v2(
+            "crsql_pack_columns",
+            -1,
+            sqlite::UTF8,
+            None,
+            Some(crsql_pack_columns),
+            None,
+            None,
+            None,
+        )
+        .unwrap_or(sqlite::ResultCode::ERROR);
+    if rc != ResultCode::OK {
+        return rc as c_int;
+    }
+
+    let rc = db
+        .create_function_v2(
+            "crsql_as_table",
+            1,
+            sqlite::UTF8,
+            None,
+            Some(crsql_as_table),
+            None,
+            None,
+            None,
+        )
+        .unwrap_or(sqlite::ResultCode::ERROR);
+    if rc != ResultCode::OK {
+        return rc as c_int;
+    }
+
+    let rc = unpack_columns_vtab::create_module(db).unwrap_or(sqlite::ResultCode::ERROR);
+    if rc != ResultCode::OK {
+        return rc as c_int;
+    }
+    let rc = create_cl_set_vtab::create_module(db).unwrap_or(ResultCode::ERROR);
+    return rc as c_int;
 }
 
 #[no_mangle]
 pub extern "C" fn crsql_backfill_table(
-    context: *mut context,
+    db: *mut sqlite::sqlite3,
     table: *const c_char,
     pk_cols: *const *const c_char,
     pk_cols_len: c_int,
     non_pk_cols: *const *const c_char,
     non_pk_cols_len: c_int,
+    is_commit_alter: c_int,
+    no_tx: c_int,
 ) -> c_int {
     let table = unsafe { CStr::from_ptr(table).to_str() };
     let pk_cols = unsafe {
@@ -117,10 +159,14 @@ pub extern "C" fn crsql_backfill_table(
     };
 
     let result = match (table, pk_cols, non_pk_cols) {
-        (Ok(table), Ok(pk_cols), Ok(non_pk_cols)) => {
-            let db = context.db_handle();
-            backfill_table(db, table, pk_cols, non_pk_cols)
-        }
+        (Ok(table), Ok(pk_cols), Ok(non_pk_cols)) => backfill_table(
+            db,
+            table,
+            pk_cols,
+            non_pk_cols,
+            is_commit_alter != 0,
+            no_tx != 0,
+        ),
         _ => Err(ResultCode::ERROR),
     };
 

libsql-sqlite3/ext/crr/rs/core/src/pack_columns.rs

@@ -0,0 +1,209 @@
+extern crate alloc;
+
+use alloc::string::String;
+use alloc::vec;
+use alloc::vec::Vec;
+use bytes::{Buf, BufMut};
+use core::slice;
+#[cfg(not(feature = "std"))]
+use num_traits::FromPrimitive;
+use sqlite_nostd as sqlite;
+use sqlite_nostd::{ColumnType, Context, ResultCode, Stmt, Value};
+
+pub extern "C" fn crsql_pack_columns(
+    ctx: *mut sqlite::context,
+    argc: i32,
+    argv: *mut *mut sqlite::value,
+) {
+    let args = sqlite::args!(argc, argv);
+
+    match pack_columns(args) {
+        Err(code) => {
+            ctx.result_error("Failed to pack columns");
+            ctx.result_error_code(code);
+        }
+        Ok(blob) => {
+            // TODO: pass a destructor so we don't have to copy the blob
+            ctx.result_blob_owned(blob);
+        }
+    }
+}
+
+fn pack_columns(args: &[*mut sqlite::value]) -> Result<Vec<u8>, ResultCode> {
+    let mut buf = vec![];
+    /*
+     * Format:
+     * [num_columns:u8,...[(type(0-3),num_bytes?(3-7)):u8, length?:i32, ...bytes:u8[]]]
+     *
+     * The byte used for column type also encodes the number of bytes used for the integer.
+     * e.g.: (type(0-3),num_bytes?(3-7)):u8
+     * first 3 bits are type
+     * last 5 encode how long the following integer, if there is a following integer, is. 1, 2, 3, ... 8 bytes.
+     *
+     * Not packing an integer into the minimal number of bytes required is rather wasteful.
+     * E.g., the number `0` would take 8 bytes rather than 1 byte.
+     */
+    let len_result: Result<u8, _> = args.len().try_into();
+    if let Ok(len) = len_result {
+        buf.put_u8(len);
+        for value in args {
+            match value.value_type() {
+                ColumnType::Blob => {
+                    let len = value.bytes();
+                    let num_bytes_for_len = num_bytes_needed_i32(len);
+                    let type_byte = num_bytes_for_len << 3 | (ColumnType::Blob as u8);
+                    buf.put_u8(type_byte);
+                    buf.put_int(len as i64, num_bytes_for_len as usize);
+                    buf.put_slice(value.blob());
+                }
+                ColumnType::Null => {
+                    buf.put_u8(ColumnType::Null as u8);
+                }
+                ColumnType::Float => {
+                    buf.put_u8(ColumnType::Float as u8);
+                    buf.put_f64(value.double());
+                }
+                ColumnType::Integer => {
+                    let val = value.int64();
+                    let num_bytes_for_int = num_bytes_needed_i64(val);
+                    let type_byte = num_bytes_for_int << 3 | (ColumnType::Integer as u8);
+                    buf.put_u8(type_byte);
+                    buf.put_int(val, num_bytes_for_int as usize);
+                }
+                ColumnType::Text => {
+                    let len = value.bytes();
+                    let num_bytes_for_len = num_bytes_needed_i32(len);
+                    let type_byte = num_bytes_for_len << 3 | (ColumnType::Text as u8);
+                    buf.put_u8(type_byte);
+                    buf.put_int(len as i64, num_bytes_for_len as usize);
+                    buf.put_slice(value.blob());
+                }
+            }
+        }
+        Ok(buf)
+    } else {
+        Err(ResultCode::ABORT)
+    }
+}
+
+fn num_bytes_needed_i32(val: i32) -> u8 {
+    if val & 0xFF000000u32 as i32 != 0 {
+        return 4;
+    } else if val & 0x00FF0000 != 0 {
+        return 3;
+    } else if val & 0x0000FF00 != 0 {
+        return 2;
+    } else if val * 0x000000FF != 0 {
+        return 1;
+    } else {
+        return 0;
+    }
+}
+
+fn num_bytes_needed_i64(val: i64) -> u8 {
+    if val & 0xFF00000000000000u64 as i64 != 0 {
+        return 8;
+    } else if val & 0x00FF000000000000 != 0 {
+        return 7;
+    } else if val & 0x0000FF0000000000 != 0 {
+        return 6;
+    } else if val & 0x000000FF00000000 != 0 {
+        return 5;
+    } else {
+        return num_bytes_needed_i32(val as i32);
+    }
+}
+
+pub enum ColumnValue {
+    Blob(Vec<u8>),
+    Float(f64),
+    Integer(i64),
+    Null,
+    Text(String),
+}
+
+// TODO: make a table valued function that can be used to extract a row per packed column?
+pub fn unpack_columns(data: &[u8]) -> Result<Vec<ColumnValue>, ResultCode> {
+    let mut ret = vec![];
+    let mut buf = data;
+    let num_columns = buf.get_u8();
+
+    for _i in 0..num_columns {
+        if !buf.has_remaining() {
+            return Err(ResultCode::ABORT);
+        }
+        let column_type_and_maybe_intlen = buf.get_u8();
+        let column_type = ColumnType::from_u8(column_type_and_maybe_intlen & 0x07);
+        let intlen = (column_type_and_maybe_intlen >> 3 & 0xFF) as usize;
+
+        match column_type {
+            Some(ColumnType::Blob) => {
+                if buf.remaining() < intlen {
+                    return Err(ResultCode::ABORT);
+                }
+                let len = buf.get_int(intlen) as usize;
+                if buf.remaining() < len {
+                    return Err(ResultCode::ABORT);
+                }
+                let bytes = buf.copy_to_bytes(len);
+                ret.push(ColumnValue::Blob(bytes.to_vec()));
+            }
+            Some(ColumnType::Float) => {
+                if buf.remaining() < 8 {
+                    return Err(ResultCode::ABORT);
+                }
+                ret.push(ColumnValue::Float(buf.get_f64()));
+            }
+            Some(ColumnType::Integer) => {
+                if buf.remaining() < intlen {
+                    return Err(ResultCode::ABORT);
+                }
+                ret.push(ColumnValue::Integer(buf.get_int(intlen)));
+            }
+            Some(ColumnType::Null) => {
+                ret.push(ColumnValue::Null);
+            }
+            Some(ColumnType::Text) => {
+                if buf.remaining() < intlen {
+                    return Err(ResultCode::ABORT);
+                }
+                let len = buf.get_int(intlen) as usize;
+                if buf.remaining() < len {
+                    return Err(ResultCode::ABORT);
+                }
+                let bytes = buf.copy_to_bytes(len);
+                ret.push(ColumnValue::Text(unsafe {
+                    String::from_utf8_unchecked(bytes.to_vec())
+                }))
+            }
+            None => return Err(ResultCode::MISUSE),
+        }
+    }
+
+    Ok(ret)
+}
+
+pub fn bind_package_to_stmt(
+    stmt: *mut sqlite::stmt,
+    values: &Vec<crate::ColumnValue>,
+    offset: usize,
+) -> Result<ResultCode, ResultCode> {
+    for (i, val) in values.iter().enumerate() {
+        bind_slot(i + 1 + offset, val, stmt)?;
+    }
+    Ok(ResultCode::OK)
+}
+
+fn bind_slot(
+    slot_num: usize,
+    val: &ColumnValue,
+    stmt: *mut sqlite::stmt,
+) -> Result<ResultCode, ResultCode> {
+    match val {
+        ColumnValue::Blob(b) => stmt.bind_blob(slot_num as i32, b, sqlite::Destructor::STATIC),
+        ColumnValue::Float(f) => stmt.bind_double(slot_num as i32, *f),
+        ColumnValue::Integer(i) => stmt.bind_int64(slot_num as i32, *i),
+        ColumnValue::Null => stmt.bind_null(slot_num as i32),
+        ColumnValue::Text(t) => stmt.bind_text(slot_num as i32, t, sqlite::Destructor::STATIC),
+    }
+}

libsql-sqlite3/ext/crr/rs/core/src/stmt_cache.rs

@@ -0,0 +1,137 @@
+extern crate alloc;
+use core::ffi::c_void;
+use core::mem::forget;
+use core::ptr::null_mut;
+
+use alloc::boxed::Box;
+use alloc::collections::BTreeMap;
+use alloc::format;
+use alloc::string::String;
+use alloc::string::ToString;
+use sqlite::Stmt;
+use sqlite_nostd as sqlite;
+use sqlite_nostd::ResultCode;
+
+use crate::c::crsql_ExtData;
+
+// port the stmt cache so we can
+// - start removing some unsafe code
+// - remove uthash and just use rust btreemap
+pub enum CachedStmtType {
+    SetWinnerClock = 0,
+    GetLocalCl = 1,
+    GetColVersion = 2,
+    // can we one day delete this and use site id for ties?
+    // if we do, how does that impact the backup and restore story?
+    // e.g., restoring a database snapshot on a new machine with a new siteid but
+    // bootstrapped from a backup?
+    // If we track that "we've seen this restored node since the backup point with the old site_id"
+    // then site_id comparisons could change merge results after restore for nodes that
+    // have different "seen since" records for the old site_id.
+    GetCurrValue = 3,
+    MergePkOnlyInsert = 4,
+    MergeDelete = 5,
+    MergeInsert = 6,
+    RowPatchData = 7,
+    // We zero clocks, rather than going to 1, because
+    // the current values should be totally ignored at all sites.
+    // This is because the current values would not exist had the current node
+    // processed the intervening delete.
+    // This also means that col_version is not always >= 1. A resurrected column,
+    // which missed a delete event, will have a 0 version.
+    ZeroClocksOnResurrect = 8,
+    MergeDeleteDropClocks = 9,
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_init_stmt_cache(ext_data: *mut crsql_ExtData) {
+    let map: BTreeMap<String, *mut sqlite::stmt> = BTreeMap::new();
+    unsafe {
+        (*ext_data).pStmtCache = Box::into_raw(Box::new(map)) as *mut c_void;
+    }
+}
+
+#[no_mangle]
+pub extern "C" fn crsql_clear_stmt_cache(ext_data: *mut crsql_ExtData) {
+    if unsafe { (*ext_data).pStmtCache.is_null() } {
+        return;
+    }
+    let map: Box<BTreeMap<String, *mut sqlite::stmt>> = unsafe {
+        Box::from_raw((*ext_data).pStmtCache as *mut BTreeMap<String, *mut sqlite::stmt>)
+    };
+    for (_key, stmt) in map.iter() {
+        let _ = stmt.finalize();
+    }
+    unsafe {
+        (*ext_data).pStmtCache = null_mut();
+    }
+}
+
+pub fn get_cache_key(
+    stmt_type: CachedStmtType,
+    tbl_name: &str,
+    col_name: Option<&str>,
+) -> Result<String, ResultCode> {
+    match stmt_type {
+        CachedStmtType::SetWinnerClock
+        | CachedStmtType::GetLocalCl
+        | CachedStmtType::GetColVersion
+        | CachedStmtType::MergePkOnlyInsert
+        | CachedStmtType::MergeDelete
+        | CachedStmtType::ZeroClocksOnResurrect
+        | CachedStmtType::MergeDeleteDropClocks => {
+            if col_name.is_some() {
+                // col name should not be specified for these cases
+                return Err(ResultCode::MISUSE);
+            }
+            Ok(format!(
+                "{stmt_type}_{tbl_name}",
+                stmt_type = (stmt_type as i32).to_string(),
+                tbl_name = tbl_name
+            ))
+        }
+        CachedStmtType::GetCurrValue
+        | CachedStmtType::MergeInsert
+        | CachedStmtType::RowPatchData => {
+            if let Some(col_name) = col_name {
+                Ok(format!(
+                    "{stmt_type}_{tbl_name}_{col_name}",
+                    stmt_type = (stmt_type as i32).to_string(),
+                    tbl_name = tbl_name,
+                    col_name = col_name
+                ))
+            } else {
+                // col_name must be specified in this case
+                Err(ResultCode::MISUSE)
+            }
+        }
+    }
+}
+
+pub fn set_cached_stmt(ext_data: *mut crsql_ExtData, key: String, stmt: *mut sqlite::stmt) {
+    // give ownership of the key to C
+    let mut map: Box<BTreeMap<String, *mut sqlite::stmt>> = unsafe {
+        Box::from_raw((*ext_data).pStmtCache as *mut BTreeMap<String, *mut sqlite::stmt>)
+    };
+    map.insert(key, stmt);
+    // C owns this memory.
+    forget(map);
+}
+
+pub fn get_cached_stmt(ext_data: *mut crsql_ExtData, key: &String) -> Option<*mut sqlite::stmt> {
+    let map: Box<BTreeMap<String, *mut sqlite::stmt>> = unsafe {
+        Box::from_raw((*ext_data).pStmtCache as *mut BTreeMap<String, *mut sqlite::stmt>)
+    };
+    let ret = map.get(key).copied();
+    // C owns this memory
+    forget(map);
+    return ret;
+}
+
+pub fn reset_cached_stmt(stmt: *mut sqlite::stmt) -> Result<ResultCode, ResultCode> {
+    if stmt.is_null() {
+        return Ok(ResultCode::OK);
+    }
+    stmt.clear_bindings()?;
+    stmt.reset()
+}

libsql-sqlite3/ext/crr/rs/core/src/teardown.rs

@@ -7,7 +7,7 @@ pub fn remove_crr_clock_table_if_exists(
     db: *mut sqlite::sqlite3,
     table: &str,
 ) -> Result<ResultCode, ResultCode> {
-    let escaped_table = crate::escape_ident(table);
+    let escaped_table = crate::util::escape_ident(table);
     db.exec_safe(&format!(
         "DROP TABLE IF EXISTS \"{table}__crsql_clock\"",
         table = escaped_table
@@ -18,7 +18,7 @@ pub fn remove_crr_triggers_if_exist(
     db: *mut sqlite::sqlite3,
     table: &str,
 ) -> Result<ResultCode, ResultCode> {
-    let escaped_table = crate::escape_ident(table);
+    let escaped_table = crate::util::escape_ident(table);
 
     db.exec_safe(&format!(
         "DROP TRIGGER IF EXISTS \"{table}__crsql_itrig\"",
@@ -30,6 +30,20 @@ pub fn remove_crr_triggers_if_exist(
         table = escaped_table
     ))?;
 
+    // get all columns of table
+    // iterate pk cols
+    // drop triggers against those pk cols
+    let stmt = db.prepare_v2("SELECT name FROM pragma_table_info(?) WHERE pk > 0")?;
+    stmt.bind_text(1, table, sqlite::Destructor::STATIC)?;
+    while stmt.step()? == ResultCode::ROW {
+        let col_name = stmt.column_text(0)?;
+        db.exec_safe(&format!(
+            "DROP TRIGGER IF EXISTS \"{tbl_name}_{col_name}__crsql_utrig\"",
+            tbl_name = crate::util::escape_ident(table),
+            col_name = crate::util::escape_ident(col_name),
+        ))?;
+    }
+
     db.exec_safe(&format!(
         "DROP TRIGGER IF EXISTS \"{table}__crsql_dtrig\"",
         table = escaped_table

libsql-sqlite3/ext/crr/rs/core/src/triggers.rs

@@ -0,0 +1,366 @@
+extern crate alloc;
+use alloc::format;
+use alloc::string::String;
+use alloc::vec;
+use sqlite::Connection;
+
+use core::{
+    ffi::{c_char, c_int, CStr},
+    slice,
+    str::Utf8Error,
+};
+
+use crate::c::crsql_TableInfo;
+use sqlite::{sqlite3, ResultCode};
+use sqlite_nostd as sqlite;
+
+#[no_mangle]
+pub extern "C" fn crsql_create_crr_triggers(
+    db: *mut sqlite3,
+    table_info: *mut crsql_TableInfo,
+    err: *mut *mut c_char,
+) -> c_int {
+    match create_triggers(db, table_info, err) {
+        Ok(code) => code as c_int,
+        Err(code) => code as c_int,
+    }
+}
+
+fn create_triggers(
+    db: *mut sqlite3,
+    table_info: *mut crsql_TableInfo,
+    err: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    create_insert_trigger(db, table_info, err)?;
+    create_update_trigger(db, table_info, err)?;
+    create_delete_trigger(db, table_info, err)
+}
+
+fn create_insert_trigger(
+    db: *mut sqlite3,
+    table_info: *mut crsql_TableInfo,
+    _err: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    let table_name = unsafe { CStr::from_ptr((*table_info).tblName).to_str()? };
+    let pk_columns =
+        unsafe { slice::from_raw_parts((*table_info).pks, (*table_info).pksLen as usize) };
+    let pk_list = crate::util::as_identifier_list(pk_columns, None)?;
+    let pk_new_list = crate::util::as_identifier_list(pk_columns, Some("NEW."))?;
+    let pk_where_list = crate::util::pk_where_list(pk_columns, Some("NEW."))?;
+    let trigger_body =
+        insert_trigger_body(table_info, table_name, pk_list, pk_new_list, pk_where_list)?;
+
+    let create_trigger_sql = format!(
+        "CREATE TRIGGER IF NOT EXISTS \"{table_name}__crsql_itrig\"
+      AFTER INSERT ON \"{table_name}\" WHEN crsql_internal_sync_bit() = 0
+      BEGIN
+        {trigger_body}
+      END;",
+        table_name = crate::util::escape_ident(table_name),
+        trigger_body = trigger_body
+    );
+
+    db.exec_safe(&create_trigger_sql)
+}
+
+fn insert_trigger_body(
+    table_info: *mut crsql_TableInfo,
+    table_name: &str,
+    pk_list: String,
+    pk_new_list: String,
+    pk_where_list: String,
+) -> Result<String, Utf8Error> {
+    let non_pk_columns =
+        unsafe { slice::from_raw_parts((*table_info).nonPks, (*table_info).nonPksLen as usize) };
+    let mut trigger_components = vec![];
+
+    if non_pk_columns.len() == 0 {
+        // a table that only has primary keys.
+        // we'll need to record a create record in this case.
+        trigger_components.push(format!(
+          "INSERT INTO \"{table_name}__crsql_clock\" (
+            {pk_list},
+            __crsql_col_name,
+            __crsql_col_version,
+            __crsql_db_version,
+            __crsql_seq,
+            __crsql_site_id
+            ) SELECT
+            {pk_new_list},
+            '{col_name}',
+            1,
+            crsql_next_db_version(),
+            crsql_increment_and_get_seq(),
+            NULL
+          ON CONFLICT DO UPDATE SET
+            __crsql_col_version = CASE __crsql_col_version % 2 WHEN 0 THEN __crsql_col_version + 1 ELSE __crsql_col_version + 2 END,
+            __crsql_db_version = crsql_next_db_version(),
+            __crsql_seq = crsql_get_seq() - 1,
+            __crsql_site_id = NULL;",
+          table_name = crate::util::escape_ident(table_name),
+          pk_list = pk_list,
+          pk_new_list = pk_new_list,
+          col_name = crate::c::INSERT_SENTINEL
+      ));
+    } else {
+        // only update the create record if it exists.
+        // this is an optimization so as not to create create records
+        // for things that don't strictly need them.
+        trigger_components.push(format!(
+          "UPDATE \"{table_name}__crsql_clock\" SET
+            __crsql_col_version = CASE __crsql_col_version % 2 WHEN 0 THEN __crsql_col_version + 1 ELSE __crsql_col_version + 2 END,
+            __crsql_db_version = crsql_next_db_version(),
+            __crsql_seq = crsql_increment_and_get_seq(),
+            __crsql_site_id = NULL
+          WHERE {pk_where_list} AND __crsql_col_name = '{col_name}';",
+          table_name = crate::util::escape_ident(table_name),
+          pk_where_list = pk_where_list,
+          col_name = crate::c::INSERT_SENTINEL
+        ));
+    }
+
+    for col in non_pk_columns {
+        let col_name = unsafe { CStr::from_ptr(col.name).to_str()? };
+        trigger_components.push(format_insert_trigger_component(
+            table_name,
+            &pk_list,
+            &pk_new_list,
+            col_name,
+        ))
+    }
+
+    Ok(trigger_components.join("\n"))
+}
+
+fn format_insert_trigger_component(
+    table_name: &str,
+    pk_list: &str,
+    pk_new_list: &str,
+    col_name: &str,
+) -> String {
+    format!(
+        "INSERT INTO \"{table_name}__crsql_clock\" (
+          {pk_list},
+          __crsql_col_name,
+          __crsql_col_version,
+          __crsql_db_version,
+          __crsql_seq,
+          __crsql_site_id
+        ) SELECT
+          {pk_new_list},
+          '{col_name}',
+          1,
+          crsql_next_db_version(),
+          crsql_increment_and_get_seq(),
+          NULL
+        ON CONFLICT DO UPDATE SET
+          __crsql_col_version = __crsql_col_version + 1,
+          __crsql_db_version = crsql_next_db_version(),
+          __crsql_seq = crsql_get_seq() - 1,
+          __crsql_site_id = NULL;",
+        table_name = crate::util::escape_ident(table_name),
+        pk_list = pk_list,
+        pk_new_list = pk_new_list,
+        col_name = crate::util::escape_ident_as_value(col_name)
+    )
+}
+
+fn create_update_trigger(
+    db: *mut sqlite3,
+    table_info: *mut crsql_TableInfo,
+    _err: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    let table_name = unsafe { CStr::from_ptr((*table_info).tblName).to_str()? };
+    let pk_columns =
+        unsafe { slice::from_raw_parts((*table_info).pks, (*table_info).pksLen as usize) };
+    let pk_list = crate::util::as_identifier_list(pk_columns, None)?;
+    let pk_new_list = crate::util::as_identifier_list(pk_columns, Some("NEW."))?;
+    let pk_old_list = crate::util::as_identifier_list(pk_columns, Some("OLD."))?;
+    let pk_where_list = crate::util::pk_where_list(pk_columns, Some("OLD."))?;
+    let mut any_pk_differs = vec![];
+    for c in pk_columns {
+        let name = unsafe { CStr::from_ptr(c.name).to_str()? };
+        any_pk_differs.push(format!(
+            "NEW.\"{col_name}\" IS NOT OLD.\"{col_name}\"",
+            col_name = crate::util::escape_ident(name)
+        ));
+    }
+    let any_pk_differs = any_pk_differs.join(" OR ");
+
+    // Changing a primary key to a new value is the same as deleting the row previously
+    // identified by that primary key. TODO: share this code with `create_delete_trigger`
+    for col in pk_columns {
+        let col_name = unsafe { CStr::from_ptr(col.name).to_str()? };
+        db.exec_safe(&format!(
+            "CREATE TRIGGER IF NOT EXISTS \"{tbl_name}_{col_name}__crsql_utrig\"
+          AFTER UPDATE OF \"{col_name}\" ON \"{tbl_name}\"
+          WHEN crsql_internal_sync_bit() = 0 AND NEW.\"{col_name}\" IS NOT OLD.\"{col_name}\"
+          BEGIN
+            INSERT INTO \"{table_name}__crsql_clock\" (
+              {pk_list},
+              __crsql_col_name,
+              __crsql_col_version,
+              __crsql_db_version,
+              __crsql_seq,
+              __crsql_site_id
+            ) SELECT
+              {pk_old_list},
+              '{sentinel}',
+              2,
+              crsql_next_db_version(),
+              crsql_increment_and_get_seq(),
+              NULL WHERE true
+            ON CONFLICT DO UPDATE SET
+              __crsql_col_version = 1 + __crsql_col_version,
+              __crsql_db_version = crsql_next_db_version(),
+              __crsql_seq = crsql_get_seq() - 1,
+              __crsql_site_id = NULL;
+            DELETE FROM \"{table_name}__crsql_clock\"
+              WHERE {pk_where_list} AND __crsql_col_name != '{sentinel}';
+          END;",
+            tbl_name = crate::util::escape_ident(table_name),
+            col_name = crate::util::escape_ident(col_name),
+            pk_list = pk_list,
+            pk_old_list = pk_old_list,
+            sentinel = crate::c::DELETE_SENTINEL,
+        ))?;
+    }
+
+    let trigger_body =
+        update_trigger_body(table_info, table_name, pk_list, pk_new_list, any_pk_differs)?;
+
+    let create_trigger_sql = format!(
+        "CREATE TRIGGER IF NOT EXISTS \"{table_name}__crsql_utrig\"
+      AFTER UPDATE ON \"{table_name}\" WHEN crsql_internal_sync_bit() = 0
+      BEGIN
+        {trigger_body}
+      END;",
+        table_name = crate::util::escape_ident(table_name),
+        trigger_body = trigger_body
+    );
+
+    db.exec_safe(&create_trigger_sql)
+}
+
+fn update_trigger_body(
+    table_info: *mut crsql_TableInfo,
+    table_name: &str,
+    pk_list: String,
+    pk_new_list: String,
+    any_pk_differs: String,
+) -> Result<String, Utf8Error> {
+    let non_pk_columns =
+        unsafe { slice::from_raw_parts((*table_info).nonPks, (*table_info).nonPksLen as usize) };
+    let mut trigger_components = vec![];
+
+    // If any PK is different, record a create for the row
+    // as setting a PK to a _new value_ is like insertion or creating a row.
+    // If we have CL and we conflict.. and pk is not _dead_, ignore?
+    trigger_components.push(format!(
+        "INSERT INTO \"{table_name}__crsql_clock\" (
+          {pk_list},
+          __crsql_col_name,
+          __crsql_col_version,
+          __crsql_db_version,
+          __crsql_seq,
+          __crsql_site_id
+        ) SELECT
+          {pk_new_list},
+          '{sentinel}',
+          1,
+          crsql_next_db_version(),
+          crsql_increment_and_get_seq(),
+          NULL
+        WHERE {any_pk_differs}
+        ON CONFLICT DO UPDATE SET
+          __crsql_col_version = CASE __crsql_col_version % 2 WHEN 0 THEN __crsql_col_version + 1 ELSE __crsql_col_version + 2 END,
+          __crsql_db_version = crsql_next_db_version(),
+          __crsql_seq = crsql_get_seq() - 1,
+          __crsql_site_id = NULL;",
+        table_name = crate::util::escape_ident(table_name),
+        pk_list = pk_list,
+        pk_new_list = pk_new_list,
+        sentinel = crate::c::INSERT_SENTINEL,
+        any_pk_differs = any_pk_differs
+    ));
+
+    for col in non_pk_columns {
+        let col_name = unsafe { CStr::from_ptr(col.name).to_str()? };
+        trigger_components.push(format!(
+            "INSERT INTO \"{table_name}__crsql_clock\" (
+          {pk_list},
+          __crsql_col_name,
+          __crsql_col_version,
+          __crsql_db_version,
+          __crsql_seq,
+          __crsql_site_id
+        ) SELECT
+          {pk_new_list},
+          '{col_name_val}',
+          1,
+          crsql_next_db_version(),
+          crsql_increment_and_get_seq(),
+          NULL
+        WHERE NEW.\"{col_name_ident}\" IS NOT OLD.\"{col_name_ident}\"
+        ON CONFLICT DO UPDATE SET
+          __crsql_col_version = __crsql_col_version + 1,
+          __crsql_db_version = crsql_next_db_version(),
+          __crsql_seq = crsql_get_seq() - 1,
+          __crsql_site_id = NULL;",
+            table_name = crate::util::escape_ident(table_name),
+            pk_list = pk_list,
+            pk_new_list = pk_new_list,
+            col_name_val = crate::util::escape_ident_as_value(col_name),
+            col_name_ident = crate::util::escape_ident(col_name)
+        ))
+    }
+
+    Ok(trigger_components.join("\n"))
+}
+
+fn create_delete_trigger(
+    db: *mut sqlite3,
+    table_info: *mut crsql_TableInfo,
+    _err: *mut *mut c_char,
+) -> Result<ResultCode, ResultCode> {
+    let table_name = unsafe { CStr::from_ptr((*table_info).tblName).to_str()? };
+    let pk_columns =
+        unsafe { slice::from_raw_parts((*table_info).pks, (*table_info).pksLen as usize) };
+    let pk_list = crate::util::as_identifier_list(pk_columns, None)?;
+    let pk_old_list = crate::util::as_identifier_list(pk_columns, Some("OLD."))?;
+    let pk_where_list = crate::util::pk_where_list(pk_columns, Some("OLD."))?;
+
+    let create_trigger_sql = format!(
+        "CREATE TRIGGER IF NOT EXISTS \"{table_name}__crsql_dtrig\"
+    AFTER DELETE ON \"{table_name}\" WHEN crsql_internal_sync_bit() = 0
+    BEGIN
+      INSERT INTO \"{table_name}__crsql_clock\" (
+        {pk_list},
+        __crsql_col_name,
+        __crsql_col_version,
+        __crsql_db_version,
+        __crsql_seq,
+        __crsql_site_id
+      ) SELECT
+        {pk_old_list},
+        '{sentinel}',
+        2,
+        crsql_next_db_version(),
+        crsql_increment_and_get_seq(),
+        NULL WHERE true
+      ON CONFLICT DO UPDATE SET
+        __crsql_col_version = 1 + __crsql_col_version,
+        __crsql_db_version = crsql_next_db_version(),
+        __crsql_seq = crsql_get_seq() - 1,
+        __crsql_site_id = NULL;
+      DELETE FROM \"{table_name}__crsql_clock\"
+        WHERE {pk_where_list} AND __crsql_col_name != '{sentinel}';
+    END;",
+        table_name = crate::util::escape_ident(table_name),
+        sentinel = crate::c::DELETE_SENTINEL,
+        pk_where_list = pk_where_list,
+        pk_old_list = pk_old_list
+    );
+
+    db.exec_safe(&create_trigger_sql)
+}

libsql-sqlite3/ext/crr/rs/core/src/unpack_columns_vtab.rs

@@ -0,0 +1,267 @@
+extern crate alloc;
+
+use core::ffi::{c_char, c_int, c_void};
+use core::slice;
+
+use alloc::boxed::Box;
+use alloc::ffi::CString;
+use alloc::format;
+use alloc::vec::Vec;
+use sqlite::{Connection, Context, Value};
+use sqlite_nostd as sqlite;
+use sqlite_nostd::ResultCode;
+
+use crate::{unpack_columns, ColumnValue};
+
+#[derive(Debug)]
+enum Columns {
+    CELL = 0,
+    PACKAGE = 1,
+}
+
+extern "C" fn connect(
+    db: *mut sqlite::sqlite3,
+    _aux: *mut c_void,
+    _argc: c_int,
+    _argv: *const *const c_char,
+    vtab: *mut *mut sqlite::vtab,
+    _err: *mut *mut c_char,
+) -> c_int {
+    // TODO: more ergonomic rust binding for this
+    if let Err(rc) = sqlite::declare_vtab(db, "CREATE TABLE x(cell ANY, package BLOB hidden);") {
+        return rc as c_int;
+    }
+
+    unsafe {
+        // TODO: more ergonomic rust bindings
+        *vtab = Box::into_raw(Box::new(sqlite::vtab {
+            nRef: 0,
+            pModule: core::ptr::null(),
+            zErrMsg: core::ptr::null_mut(),
+        }));
+        let _ = sqlite::vtab_config(db, sqlite::INNOCUOUS);
+    }
+    ResultCode::OK as c_int
+}
+
+extern "C" fn disconnect(vtab: *mut sqlite::vtab) -> c_int {
+    unsafe {
+        drop(Box::from_raw(vtab));
+    }
+    ResultCode::OK as c_int
+}
+
+extern "C" fn best_index(vtab: *mut sqlite::vtab, index_info: *mut sqlite::index_info) -> c_int {
+    // TODO: better bindings to create this slice for the user
+    let constraints = unsafe {
+        slice::from_raw_parts_mut(
+            (*index_info).aConstraint,
+            (*index_info).nConstraint as usize,
+        )
+    };
+    let constraint_usage = unsafe {
+        slice::from_raw_parts_mut(
+            (*index_info).aConstraintUsage,
+            (*index_info).nConstraint as usize,
+        )
+    };
+
+    for (i, constraint) in constraints.iter().enumerate() {
+        if constraint.usable == 0 {
+            continue;
+        }
+        if constraint.iColumn != Columns::PACKAGE as i32 {
+            unsafe {
+                (*vtab).zErrMsg = CString::new(format!(
+                    "no package column specified. Got {:?} instead",
+                    Columns::PACKAGE
+                ))
+                .map_or(core::ptr::null_mut(), |f| f.into_raw());
+            }
+            return ResultCode::MISUSE as c_int;
+        } else {
+            constraint_usage[i].argvIndex = 1;
+            constraint_usage[i].omit = 1;
+        }
+    }
+
+    ResultCode::OK as c_int
+}
+
+#[repr(C)]
+struct Cursor {
+    base: sqlite::vtab_cursor,
+    crsr: usize,
+    unpacked: Option<Vec<ColumnValue>>,
+}
+
+extern "C" fn open(_vtab: *mut sqlite::vtab, cursor: *mut *mut sqlite::vtab_cursor) -> c_int {
+    unsafe {
+        let boxed = Box::new(Cursor {
+            base: sqlite::vtab_cursor {
+                pVtab: core::ptr::null_mut(),
+            },
+            crsr: 0,
+            unpacked: None,
+        });
+        let raw_cursor = Box::into_raw(boxed);
+        *cursor = raw_cursor.cast::<sqlite::vtab_cursor>();
+    }
+
+    ResultCode::OK as c_int
+}
+
+extern "C" fn close(cursor: *mut sqlite::vtab_cursor) -> c_int {
+    let crsr = cursor.cast::<Cursor>();
+    unsafe {
+        drop(Box::from_raw(crsr));
+    }
+    ResultCode::OK as c_int
+}
+
+extern "C" fn filter(
+    cursor: *mut sqlite::vtab_cursor,
+    _idx_num: c_int,
+    _idx_str: *const c_char,
+    argc: c_int,
+    argv: *mut *mut sqlite::value,
+) -> c_int {
+    // pull out package arg as set up by xBestIndex (should always be argv0)
+    // stick into cursor
+    let args = sqlite::args!(argc, argv);
+    if args.len() < 1 {
+        unsafe {
+            (*(*cursor).pVtab).zErrMsg = CString::new("Zero args passed to filter")
+                .map_or(core::ptr::null_mut(), |f| f.into_raw());
+        }
+        return ResultCode::MISUSE as c_int;
+    }
+
+    let crsr = cursor.cast::<Cursor>();
+    unsafe {
+        if let Ok(cols) = unpack_columns(args[0].blob()) {
+            (*crsr).unpacked = Some(cols);
+            (*crsr).crsr = 0;
+        } else {
+            return ResultCode::ERROR as c_int;
+        }
+    }
+
+    ResultCode::OK as c_int
+}
+
+extern "C" fn next(cursor: *mut sqlite::vtab_cursor) -> c_int {
+    // go so long as crsr < unpacked.len
+    // if crsr == unpacked.len continue
+    // else, return done
+    let crsr = cursor.cast::<Cursor>();
+    unsafe {
+        (*crsr).crsr += 1;
+    }
+    ResultCode::OK as c_int
+}
+
+extern "C" fn eof(cursor: *mut sqlite::vtab_cursor) -> c_int {
+    // crsr >= unpacked.len
+    let crsr = cursor.cast::<Cursor>();
+    unsafe {
+        match &(*crsr).unpacked {
+            Some(cols) => {
+                if (*crsr).crsr >= cols.len() {
+                    1
+                } else {
+                    0
+                }
+            }
+            None => 1,
+        }
+    }
+}
+
+extern "C" fn column(
+    cursor: *mut sqlite::vtab_cursor,
+    ctx: *mut sqlite::context,
+    col_num: c_int,
+) -> c_int {
+    let crsr = cursor.cast::<Cursor>();
+    if col_num == Columns::CELL as i32 {
+        unsafe {
+            if let Some(cols) = &(*crsr).unpacked {
+                let col_value = &cols[(*crsr).crsr];
+                match col_value {
+                    ColumnValue::Blob(b) => {
+                        ctx.result_blob_static(b);
+                    }
+                    ColumnValue::Float(f) => {
+                        ctx.result_double(*f);
+                    }
+                    ColumnValue::Integer(i) => {
+                        ctx.result_int64(*i);
+                    }
+                    ColumnValue::Null => {
+                        ctx.result_null();
+                    }
+                    ColumnValue::Text(t) => {
+                        ctx.result_text_static(t);
+                    }
+                }
+                ResultCode::OK as c_int
+            } else {
+                (*(*cursor).pVtab).zErrMsg = CString::new("No columns to unpack!")
+                    .map_or(core::ptr::null_mut(), |f| f.into_raw());
+                ResultCode::ABORT as c_int
+            }
+        }
+    } else {
+        unsafe {
+            (*(*cursor).pVtab).zErrMsg =
+                CString::new(format!("Selected a column besides cell! {}", col_num))
+                    .map_or(core::ptr::null_mut(), |f| f.into_raw());
+        }
+        ResultCode::MISUSE as c_int
+    }
+}
+
+extern "C" fn rowid(cursor: *mut sqlite::vtab_cursor, row_id: *mut sqlite::int64) -> c_int {
+    let crsr = cursor.cast::<Cursor>();
+    unsafe { *row_id = (*crsr).crsr as i64 }
+    ResultCode::OK as c_int
+}
+
+static MODULE: sqlite_nostd::module = sqlite_nostd::module {
+    iVersion: 0,
+    xCreate: None,
+    xConnect: Some(connect),
+    xBestIndex: Some(best_index),
+    xDisconnect: Some(disconnect),
+    xDestroy: None,
+    xOpen: Some(open),
+    xClose: Some(close),
+    xFilter: Some(filter),
+    xNext: Some(next),
+    xEof: Some(eof),
+    xColumn: Some(column),
+    xRowid: Some(rowid),
+    xUpdate: None,
+    xBegin: None,
+    xSync: None,
+    xCommit: None,
+    xRollback: None,
+    xFindFunction: None,
+    xRename: None,
+    xSavepoint: None,
+    xRelease: None,
+    xRollbackTo: None,
+    xShadowName: None,
+    xPreparedSql: None,
+};
+
+/**
+ * CREATE TABLE [x] (cell, package HIDDEN);
+ * SELECT cell FROM crsql_unpack_columns WHERE package = ___;
+ */
+pub fn create_module(db: *mut sqlite::sqlite3) -> Result<ResultCode, ResultCode> {
+    db.create_module_v2("crsql_unpack_columns", &MODULE, None, None)?;
+
+    Ok(ResultCode::OK)
+}

libsql-sqlite3/ext/crr/rs/core/src/util.rs

@@ -0,0 +1,146 @@
+extern crate alloc;
+
+use crate::alloc::string::ToString;
+use crate::c::crsql_ColumnInfo;
+use alloc::format;
+use alloc::string::String;
+use alloc::vec;
+use alloc::vec::Vec;
+use core::{ffi::CStr, str::Utf8Error};
+use sqlite::{sqlite3, ColumnType, Connection, ResultCode};
+use sqlite_nostd as sqlite;
+
+pub fn get_dflt_value(
+    db: *mut sqlite3,
+    table: &str,
+    col: &str,
+) -> Result<Option<String>, ResultCode> {
+    let sql = "SELECT [dflt_value], [notnull] FROM pragma_table_info(?) WHERE name = ?";
+    let stmt = db.prepare_v2(sql)?;
+    stmt.bind_text(1, table, sqlite_nostd::Destructor::STATIC)?;
+    stmt.bind_text(2, col, sqlite_nostd::Destructor::STATIC)?;
+    let rc = stmt.step()?;
+    if rc == ResultCode::DONE {
+        // There should always be a row for a column in pragma_table_info
+        return Err(ResultCode::DONE);
+    }
+
+    let notnull = stmt.column_int(1)?;
+    let dflt_column_type = stmt.column_type(0)?;
+
+    // if the column is nullable and no default value is specified
+    // then the default value is null.
+    if notnull == 0 && dflt_column_type == ColumnType::Null {
+        return Ok(Some(String::from("NULL")));
+    }
+
+    if dflt_column_type == ColumnType::Null {
+        // no default value specified
+        // and the column is not nullable
+        return Ok(None);
+    }
+
+    return Ok(Some(String::from(stmt.column_text(0)?)));
+}
+
+pub fn slab_rowid(idx: i32, rowid: sqlite::int64) -> sqlite::int64 {
+    if idx < 0 {
+        return -1;
+    }
+
+    let modulo = rowid % crate::consts::ROWID_SLAB_SIZE;
+    return (idx as i64) * crate::consts::ROWID_SLAB_SIZE + modulo;
+}
+
+pub fn pk_where_list(
+    columns: &[crsql_ColumnInfo],
+    rhs_prefix: Option<&str>,
+) -> Result<String, Utf8Error> {
+    let mut result = vec![];
+    for c in columns {
+        let name = unsafe { CStr::from_ptr(c.name) };
+        result.push(if let Some(prefix) = rhs_prefix {
+            format!(
+                "\"{col_name}\" IS {prefix}\"{col_name}\"",
+                prefix = prefix,
+                col_name = crate::util::escape_ident(name.to_str()?)
+            )
+        } else {
+            format!(
+                "\"{col_name}\" = \"{col_name}\"",
+                col_name = crate::util::escape_ident(name.to_str()?)
+            )
+        })
+    }
+    Ok(result.join(" AND "))
+}
+
+pub fn where_list(columns: &[crsql_ColumnInfo], prefix: Option<&str>) -> Result<String, Utf8Error> {
+    let mut result = vec![];
+    for c in columns {
+        let name = unsafe { CStr::from_ptr(c.name) };
+        if let Some(prefix) = prefix {
+            result.push(format!(
+                "{prefix}\"{col_name}\" IS ?",
+                prefix = prefix,
+                col_name = crate::util::escape_ident(name.to_str()?)
+            ));
+        } else {
+            result.push(format!(
+                "\"{col_name}\" IS ?",
+                col_name = crate::util::escape_ident(name.to_str()?)
+            ));
+        }
+    }
+
+    Ok(result.join(" AND "))
+}
+
+pub fn binding_list(num_slots: usize) -> String {
+    core::iter::repeat('?')
+        .take(num_slots)
+        .map(|c| c.to_string())
+        .collect::<Vec<_>>()
+        .join(", ")
+}
+
+pub fn as_identifier_list(
+    columns: &[crsql_ColumnInfo],
+    prefix: Option<&str>,
+) -> Result<String, Utf8Error> {
+    let mut result = vec![];
+    for c in columns {
+        let name = unsafe { CStr::from_ptr(c.name) };
+        result.push(if let Some(prefix) = prefix {
+            format!(
+                "{}\"{}\"",
+                prefix,
+                crate::util::escape_ident(name.to_str()?)
+            )
+        } else {
+            format!("\"{}\"", crate::util::escape_ident(name.to_str()?))
+        })
+    }
+    Ok(result.join(","))
+}
+
+pub fn map_columns<F>(columns: &[crsql_ColumnInfo], map: F) -> Result<Vec<String>, Utf8Error>
+where
+    F: Fn(&str) -> String,
+{
+    let mut result = vec![];
+    for c in columns {
+        let name = unsafe { CStr::from_ptr(c.name) };
+        result.push(map(name.to_str()?))
+    }
+
+    return Ok(result);
+}
+
+pub fn escape_ident(ident: &str) -> String {
+    return ident.replace("\"", "\"\"");
+}
+
+pub fn escape_ident_as_value(ident: &str) -> String {
+    return ident.replace("'", "''");
+}

libsql-sqlite3/ext/crr/rs/fractindex-core/rust-toolchain.toml

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "nightly-2023-06-17"

libsql-sqlite3/ext/crr/rs/fractindex-core/src/as_ordered.rs

@@ -118,8 +118,8 @@ fn create_pend_trigger(
 ) -> Result<ResultCode, ResultCode> {
     let trigger = format!(
         "CREATE TRIGGER IF NOT EXISTS \"__crsql_{table}_fractindex_pend_trig\" AFTER INSERT ON \"{table}\"
-        WHEN NEW.\"{order_by_column}\" = -1 OR NEW.\"{order_by_column}\" = 1 BEGIN
-            UPDATE \"{table}\" SET \"{order_by_column}\" = CASE NEW.\"{order_by_column}\"
+        WHEN CAST(NEW.\"{order_by_column}\" AS INTEGER) = -1 OR CAST(NEW.\"{order_by_column}\" AS INTEGER) = 1 BEGIN
+            UPDATE \"{table}\" SET \"{order_by_column}\" = CASE CAST(NEW.\"{order_by_column}\" AS INTEGER)
             WHEN -1 THEN crsql_fract_key_between(NULL, ({min_select}))
             WHEN 1 THEN crsql_fract_key_between(({max_select}), NULL)
             END

libsql-sqlite3/ext/crr/rs/fractindex-core/src/fractindex_view.rs

@@ -270,17 +270,22 @@ pub fn fix_conflict_return_old_key(
         "UPDATE \"{table}\" SET \"{order_col}\" = crsql_fract_key_between(
         (
           SELECT \"{order_col}\" FROM \"{table}\"
-          JOIN (SELECT {list_columns} FROM \"{table}\" WHERE {pk_predicates}) as t
-          ON {list_join_predicates} WHERE \"{order_col}\" < ?{target_order_slot} ORDER BY \"{order_col}\" DESC LIMIT 1
+          {maybe_join} WHERE \"{order_col}\" < ?{target_order_slot} ORDER BY \"{order_col}\" DESC LIMIT 1
         ),
         ?{target_order_slot}
       ) WHERE {pk_predicates} RETURNING \"{order_col}\"",
         table = escape_ident(table),
         order_col = escape_ident(order_col.text()),
         pk_predicates = pk_predicates,
-        list_join_predicates = list_join_predicates,
-        list_columns = list_columns,
-        target_order_slot = pk_values.len() + 1
+        target_order_slot = pk_values.len() + 1,
+        maybe_join = if list_columns.len() > 0 {
+          format!(
+            "JOIN (SELECT {list_columns} FROM \"{table}\" WHERE {pk_predicates}) as t
+            ON {list_join_predicates}",
+            list_columns = list_columns, pk_predicates = pk_predicates, table = escape_ident(table), list_join_predicates = list_join_predicates)
+        } else {
+          format!("")
+        }
     );
 
     let stmt = db.prepare_v2(&sql)?;

libsql-sqlite3/ext/crr/rs/fractindex-core/src/lib.rs

@@ -169,7 +169,7 @@ pub extern "C" fn sqlite3_crsqlfractionalindex_init(
     if let Err(rc) = db.create_function_v2(
         "crsql_fract_fix_conflict_return_old_key",
         -1,
-        sqlite::UTF8,
+        sqlite::UTF8 | sqlite::INNOCUOUS,
         None,
         Some(crsql_fract_fix_conflict_return_old_key),
         None,

libsql-sqlite3/ext/crr/rs/integration-check/Cargo.lock

@@ -67,6 +67,12 @@ version = "1.3.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
 
+[[package]]
+name = "bytes"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be"
+
 [[package]]
 name = "bytesize"
 version = "1.1.0"
@@ -129,6 +135,16 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "crsql_core"
+version = "0.1.0"
+dependencies = [
+ "bytes",
+ "num-derive",
+ "num-traits",
+ "sqlite_nostd",
+]
+
 [[package]]
 name = "either"
 version = "1.8.1"
@@ -176,6 +192,7 @@ version = "0.0.1"
 dependencies = [
  "cargo-valgrind",
  "cc",
+ "crsql_core",
  "integration_utils",
  "sqlite_nostd",
 ]

libsql-sqlite3/ext/crr/rs/integration-check/Cargo.toml

@@ -9,6 +9,7 @@ license = "Apache-2.0"
 
 [dependencies]
 sqlite_nostd = { path="../sqlite-rs-embedded/sqlite_nostd", features=["static"] }
+crsql_core = { path="../core" }
 integration_utils = { path="../integration-utils" }
 cargo-valgrind = "2.1.0"
 

libsql-sqlite3/ext/crr/rs/integration-check/build.rs

@@ -19,8 +19,8 @@ fn main() {
         .expect("failed to make loadable extension");
 
     cc::Build::new()
-        .file("../../src/sqlite/sqlite3.c")
-        .include("../../src/sqlite/")
+        .file("../../../../sqlite3.c")
+        .include("../../../../")
         .flag("-DSQLITE_CORE")
         .compile("sqlite3");
 }

libsql-sqlite3/ext/crr/rs/integration-check/rust-toolchain.toml

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "nightly-2023-06-17"

libsql-sqlite3/ext/crr/rs/integration-check/tests/automigrate.rs

@@ -2,7 +2,7 @@ use sqlite::{Connection, ManagedConnection, ResultCode};
 use sqlite_nostd as sqlite;
 
 // TODO: auto-calculate starting number
-integration_utils::counter_setup!(25);
+integration_utils::counter_setup!(26);
 
 #[test]
 fn empty_schema() {
@@ -141,6 +141,303 @@ fn no_default_value() {
     decrement_counter();
 }
 
+#[test]
+fn strut_schema() {
+    strut_schema_impl().unwrap();
+    decrement_counter();
+}
+
+fn strut_schema_impl() -> Result<(), ResultCode> {
+    let db = integration_utils::opendb()?;
+    let stmt = db.db.prepare_v2(
+        r#"
+SELECT crsql_automigrate(?)"#,
+    )?;
+    stmt.bind_text(
+        1,
+        r#"
+CREATE TABLE IF NOT EXISTS "deck" (
+  "id" INTEGER primary key,
+  "title",
+  "created",
+  "modified",
+  "theme_id",
+  "chosen_presenter"
+);
+
+CREATE TABLE IF NOT EXISTS "slide" (
+  "id" INTEGER primary key,
+  "deck_id",
+  "order",
+  "created",
+  "modified",
+  "x",
+  "y",
+  "z"
+);
+
+CREATE INDEX IF NOT EXISTS "slide_deck_id" ON "slide" ("deck_id", "order");
+
+CREATE TABLE IF NOT EXISTS "text_component" (
+  "id" INTEGER primary key,
+  "slide_id",
+  "text",
+  "styles",
+  "x",
+  "y"
+);
+
+CREATE TABLE IF NOT EXISTS "embed_component" ("id" primary key, "slide_id", "src", "x", "y");
+
+CREATE INDEX IF NOT EXISTS "embed_component_slide_id" ON "embed_component" ("slide_id");
+
+CREATE TABLE IF NOT EXISTS "shape_component" (
+  "id" INTEGER primary key,
+  "slide_id",
+  "type",
+  "props",
+  "x",
+  "y"
+);
+
+CREATE INDEX IF NOT EXISTS "shape_component_slide_id" ON "shape_component" ("slide_id");
+
+CREATE TABLE IF NOT EXISTS "line_component" ("id" primary key, "slide_id", "props");
+
+CREATE INDEX IF NOT EXISTS "line_component_slide_id" ON "line_component" ("slide_id");
+
+CREATE TABLE IF NOT EXISTS "line_point" ("id" primary key, "line_id", "x", "y");
+
+CREATE INDEX IF NOT EXISTS "line_point_line_id" ON "line_point" ("line_id");
+
+CREATE INDEX IF NOT EXISTS "text_component_slide_id" ON "text_component" ("slide_id");
+
+CREATE TABLE IF NOT EXISTS "theme" (
+  "id" INTEGER primary key,
+  "name",
+  "bg_colorset",
+  "fg_colorset",
+  "fontset",
+  "surface_color",
+  "font_color"
+);
+
+CREATE TABLE IF NOT EXISTS "recent_color" (
+  "color" INTEGER primary key,
+  "last_used",
+  "first_used",
+  "theme_id"
+);
+
+CREATE TABLE IF NOT EXISTS "presenter" (
+  "name" primary key,
+  "available_transitions",
+  "picked_transition"
+);
+
+SELECT crsql_as_crr('deck');
+
+SELECT crsql_as_crr('slide');
+
+SELECT crsql_fract_as_ordered('slide', 'order', 'deck_id');
+
+SELECT crsql_as_crr('text_component');
+
+SELECT crsql_as_crr('embed_component');
+
+SELECT crsql_as_crr('shape_component');
+
+SELECT crsql_as_crr('line_component');
+
+SELECT crsql_as_crr('line_point');
+
+SELECT crsql_as_crr('theme');
+
+SELECT crsql_as_crr('recent_color');
+
+SELECT crsql_as_crr('presenter');
+
+CREATE TABLE IF NOT EXISTS "selected_slide" (
+  "deck_id",
+  "slide_id",
+  primary key ("deck_id", "slide_id")
+);
+
+CREATE TABLE IF NOT EXISTS "selected_component" (
+  "slide_id",
+  "component_id",
+  "component_type",
+  primary key ("slide_id", "component_id")
+);
+
+CREATE TABLE IF NOT EXISTS "undo_stack" (
+  "deck_id",
+  "operation",
+  "order",
+  primary key ("deck_id", "order")
+);
+
+CREATE TABLE IF NOT EXISTS "redo_stack" (
+  "deck_id",
+  "operation",
+  "order",
+  primary key ("deck_id", "order")
+);"#,
+        sqlite::Destructor::STATIC,
+    )?;
+    stmt.step()?;
+    assert_eq!(stmt.column_text(0)?, "migration complete");
+    stmt.reset()?;
+    stmt.step()?;
+    assert_eq!(stmt.column_text(0)?, "migration complete");
+
+    // Now lets make change
+    let stmt = db.db.prepare_v2(
+        r#"
+SELECT crsql_automigrate(?)"#,
+    )?;
+    stmt.bind_text(
+        1,
+        r#"
+CREATE TABLE IF NOT EXISTS "deck" (
+"id" INTEGER primary key,
+"title",
+"created",
+"modified",
+"theme_id",
+"chosen_presenter"
+);
+
+CREATE TABLE IF NOT EXISTS "slide" (
+"id" INTEGER primary key,
+"deck_id",
+"order",
+"created",
+"modified",
+"x",
+"y",
+"z"
+);
+
+CREATE INDEX IF NOT EXISTS "slide_deck_id" ON "slide" ("deck_id", "order");
+
+CREATE TABLE IF NOT EXISTS "text_component" (
+"id" INTEGER primary key,
+"slide_id",
+"text",
+"styles",
+"x",
+"y",
+"width",
+"height"
+);
+
+CREATE TABLE IF NOT EXISTS "embed_component" ("id" primary key, "slide_id", "src", "x", "y", "width", "height");
+
+CREATE INDEX IF NOT EXISTS "embed_component_slide_id" ON "embed_component" ("slide_id");
+
+CREATE TABLE IF NOT EXISTS "shape_component" (
+"id" INTEGER primary key,
+"slide_id",
+"type",
+"props",
+"x",
+"y",
+"width",
+"height"
+);
+
+CREATE INDEX IF NOT EXISTS "shape_component_slide_id" ON "shape_component" ("slide_id");
+
+CREATE TABLE IF NOT EXISTS "line_component" ("id" primary key, "slide_id", "props");
+
+CREATE INDEX IF NOT EXISTS "line_component_slide_id" ON "line_component" ("slide_id");
+
+CREATE TABLE IF NOT EXISTS "line_point" ("id" primary key, "line_id", "x", "y");
+
+CREATE INDEX IF NOT EXISTS "line_point_line_id" ON "line_point" ("line_id");
+
+CREATE INDEX IF NOT EXISTS "text_component_slide_id" ON "text_component" ("slide_id");
+
+CREATE TABLE IF NOT EXISTS "theme" (
+"id" INTEGER primary key,
+"name",
+"bg_colorset",
+"fg_colorset",
+"fontset",
+"surface_color",
+"font_color"
+);
+
+CREATE TABLE IF NOT EXISTS "recent_color" (
+"color" INTEGER primary key,
+"last_used",
+"first_used",
+"theme_id"
+);
+
+CREATE TABLE IF NOT EXISTS "presenter" (
+"name" primary key,
+"available_transitions",
+"picked_transition"
+);
+
+SELECT crsql_as_crr('deck');
+
+SELECT crsql_as_crr('slide');
+
+SELECT crsql_fract_as_ordered('slide', 'order', 'deck_id');
+
+SELECT crsql_as_crr('text_component');
+
+SELECT crsql_as_crr('embed_component');
+
+SELECT crsql_as_crr('shape_component');
+
+SELECT crsql_as_crr('line_component');
+
+SELECT crsql_as_crr('line_point');
+
+SELECT crsql_as_crr('theme');
+
+SELECT crsql_as_crr('recent_color');
+
+SELECT crsql_as_crr('presenter');
+
+CREATE TABLE IF NOT EXISTS "selected_slide" (
+"deck_id",
+"slide_id",
+primary key ("deck_id", "slide_id")
+);
+
+CREATE TABLE IF NOT EXISTS "selected_component" (
+"slide_id",
+"component_id",
+"component_type",
+primary key ("slide_id", "component_id")
+);
+
+CREATE TABLE IF NOT EXISTS "undo_stack" (
+"deck_id",
+"operation",
+"order",
+primary key ("deck_id", "order")
+);
+
+CREATE TABLE IF NOT EXISTS "redo_stack" (
+"deck_id",
+"operation",
+"order",
+primary key ("deck_id", "order")
+);"#,
+        sqlite::Destructor::STATIC,
+    )?;
+    stmt.step()?;
+    assert_eq!(stmt.column_text(0)?, "migration complete");
+
+    Ok(())
+}
+
 fn empty_schema_impl() -> Result<(), ResultCode> {
     let db = integration_utils::opendb()?;
     let stmt = db.db.prepare_v2("SELECT crsql_automigrate('')")?;
@@ -256,6 +553,28 @@ fn remove_col_impl() -> Result<(), ResultCode> {
     Ok(())
 }
 
+#[test]
+fn remove_col_fract_table() {
+    let db = integration_utils::opendb().expect("db opened");
+    db.db
+        .exec_safe("CREATE TABLE todo (id primary key, content text, position, thing)")
+        .expect("table made");
+    db.db
+        .exec_safe("SELECT crsql_fract_as_ordered('todo', 'position');")
+        .expect("as ordered");
+
+    let schema = "
+      CREATE TABLE IF NOT EXISTS todo (
+          id primary key,
+          content text,
+          position
+      );
+  ";
+    invoke_automigrate(&db.db, schema).expect("migrated");
+
+    assert!(expect_columns(&db.db, "todo", vec!["id", "content", "position"]).expect("matched"));
+}
+
 fn remove_index_impl() -> Result<(), ResultCode> {
     let db = integration_utils::opendb()?;
     db.db.exec_safe(

libsql-sqlite3/ext/crr/rs/integration-check/tests/backfill.rs

@@ -68,14 +68,14 @@ fn new_nonempty_table_impl(apply_twice: bool) -> Result<(), ResultCode> {
         "SELECT [table], [pk], [cid], [val], [col_version], [db_version] FROM crsql_changes;",
     )?;
     let mut cnt = 0;
-    while stmt.step()? == ResultCode::ROW {
+    while stmt.step().unwrap() == ResultCode::ROW {
         cnt = cnt + 1;
         if cnt == 1 {
-            assert_eq!(stmt.column_text(1)?, "1"); // pk
-            assert_eq!(stmt.column_text(3)?, "'one'"); // col value
+            assert_eq!(stmt.column_blob(1)?, [1, 9, 1]); // pk
+            assert_eq!(stmt.column_text(3)?, "one"); // col value
         } else {
-            assert_eq!(stmt.column_text(1)?, "2"); // pk
-            assert_eq!(stmt.column_text(3)?, "'two'"); // col value
+            assert_eq!(stmt.column_blob(1)?, [1, 9, 2]); // pk
+            assert_eq!(stmt.column_text(3)?, "two"); // col value
         }
         assert_eq!(stmt.column_text(0)?, "foo"); // table name
         assert_eq!(stmt.column_text(2)?, "name"); // col name

libsql-sqlite3/ext/crr/rs/integration-check/tests/fract.rs

@@ -0,0 +1,19 @@
+use sqlite::{Connection, ResultCode};
+use sqlite_nostd as sqlite;
+
+#[test]
+fn sort_no_list_col() {
+    let w = integration_utils::opendb().expect("db opened");
+    let db = &w.db;
+
+    db.exec_safe("CREATE TABLE todo (id primary key, position)")
+        .expect("table created");
+    db.exec_safe("SELECT crsql_fract_as_ordered('todo', 'position')")
+        .expect("as ordered");
+    db.exec_safe(
+        "INSERT INTO todo VALUES (1, 'Zm'), (2, 'ZmG'), (3, 'ZmG'), (4, 'ZmV'), (5, 'Zn')",
+    )
+    .expect("inserted initial values");
+    db.exec_safe("UPDATE todo_fractindex SET after_id = 2 WHERE id = 5")
+        .expect("repositioned id 5");
+}

libsql-sqlite3/ext/crr/rs/integration-check/tests/pack_columns.rs

@@ -0,0 +1,149 @@
+use crsql_core::unpack_columns;
+use crsql_core::ColumnValue;
+use sqlite::{Connection, ResultCode};
+use sqlite_nostd as sqlite;
+
+#[test]
+fn pack_columns_test() {
+    pack_columns_impl().unwrap();
+}
+
+#[test]
+fn unpack_columns_test() {
+    unpack_columns_impl().unwrap();
+}
+
+// The rust test is mainly to check with valgrind and ensure we're correctly
+// freeing data as we do some passing of destructors from rust to SQLite.
+// Complete property based tests for encode & decode exist in python.
+fn pack_columns_impl() -> Result<(), ResultCode> {
+    let db = integration_utils::opendb()?;
+    db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY, x, y)")?;
+    let insert_stmt = db.db.prepare_v2("INSERT INTO foo VALUES (?, ?, ?)")?;
+    let blob: [u8; 3] = [1, 2, 3];
+
+    insert_stmt.bind_int(1, 12)?;
+    insert_stmt.bind_text(2, "str", sqlite::Destructor::STATIC)?;
+    insert_stmt.bind_blob(3, &blob, sqlite::Destructor::STATIC)?;
+    insert_stmt.step()?;
+
+    let select_stmt = db
+        .db
+        .prepare_v2("SELECT quote(crsql_pack_columns(id, x, y)) FROM foo")?;
+    select_stmt.step()?;
+    let result = select_stmt.column_text(0)?;
+    assert!(result == "X'03090C0B037374720C03010203'");
+    // 03 09 0C 0B 03 73 74 72 0C 03 01 02 03
+    // cols: 03
+    // type & intlen: 09 -> 0b00001001 -> 01 type & 01 intlen
+    // value: 0C -> 12
+    // type & intlen: 0B -> 0b00001011 -> 03 type & 01 intlen
+    // 03 -> len
+    // 73 74 72 -> str
+    // type & intlen: 0C ->  0b00001100 -> 04 type & 01 intlen
+    // len: 03
+    // bytes: 01 02 3
+    // voila, done in 13 bytes! < 18 byte string < 26 byte binary w/o varints
+
+    // Before variable length encoding:
+    // 03 01 00 00 00 00 00 00 00 0C 03 00 00 00 03 73 74 72 04 00 00 00 03 01 02 03
+    // cols:03
+    // type: 01 (integer)
+    // value: 00 00 00 00 00 00 00 0C (12) TODO: encode as variable length integers to save space?
+    // type: 03 (text)
+    // len: 00 00 00 03 (3)
+    // byes: 73 (s) 74 (t) 72 (r)
+    // type: 04 (blob)
+    // len: 00 00 00 03 (3)
+    // bytes: 01 02 03
+    // vs string:
+    // 12|'str'|x'010203'
+    // ^ 18 bytes via string
+    // vs
+    // 26 bytes via binary
+    // 13 bytes are wasted due to not using variable length encoding for integers
+    // So.. do variable length ints?
+
+    let select_stmt = db
+        .db
+        .prepare_v2("SELECT crsql_pack_columns(id, x, y) FROM foo")?;
+    select_stmt.step()?;
+    let result = select_stmt.column_blob(0)?;
+    assert!(result.len() == 13);
+    let unpacked = unpack_columns(result)?;
+    assert!(unpacked.len() == 3);
+
+    if let ColumnValue::Integer(i) = unpacked[0] {
+        assert!(i == 12);
+    } else {
+        assert!("unexpected type" == "");
+    }
+    if let ColumnValue::Text(s) = &unpacked[1] {
+        assert!(s == "str")
+    } else {
+        assert!("unexpected type" == "");
+    }
+    if let ColumnValue::Blob(b) = &unpacked[2] {
+        assert!(b[..] == blob);
+    } else {
+        assert!("unexpected type" == "");
+    }
+
+    db.db.exec_safe("DELETE FROM foo")?;
+    let insert_stmt = db.db.prepare_v2("INSERT INTO foo VALUES (?, ?, ?)")?;
+
+    insert_stmt.bind_int(1, 0)?;
+    insert_stmt.bind_int(2, 10000000)?;
+    insert_stmt.bind_int(3, -2500000)?;
+    insert_stmt.step()?;
+
+    let select_stmt = db
+        .db
+        .prepare_v2("SELECT crsql_pack_columns(id, x, y) FROM foo")?;
+    select_stmt.step()?;
+    let result = select_stmt.column_blob(0)?;
+    let unpacked = unpack_columns(result)?;
+    assert!(unpacked.len() == 3);
+
+    if let ColumnValue::Integer(i) = unpacked[0] {
+        assert!(i == 0);
+    } else {
+        assert!("unexpected type" == "");
+    }
+    if let ColumnValue::Integer(i) = unpacked[1] {
+        assert!(i == 10000000)
+    } else {
+        assert!("unexpected type" == "");
+    }
+    if let ColumnValue::Integer(i) = unpacked[2] {
+        assert!(i == -2500000);
+    } else {
+        assert!("unexpected type" == "");
+    }
+
+    Ok(())
+}
+
+fn unpack_columns_impl() -> Result<(), ResultCode> {
+    let db = integration_utils::opendb().unwrap();
+    db.db.exec_safe("CREATE TABLE foo (id PRIMARY KEY, x, y)")?;
+    let insert_stmt = db.db.prepare_v2("INSERT INTO foo VALUES (?, ?, ?)")?;
+    let blob: [u8; 3] = [1, 2, 3];
+
+    insert_stmt.bind_int(1, 12)?;
+    insert_stmt.bind_text(2, "str", sqlite::Destructor::STATIC)?;
+    insert_stmt.bind_blob(3, &blob, sqlite::Destructor::STATIC)?;
+    insert_stmt.step()?;
+
+    let select_stmt = db
+        .db
+        .prepare_v2("SELECT cell FROM crsql_unpack_columns WHERE package = (SELECT crsql_pack_columns(id, x, y) FROM foo)")?;
+    select_stmt.step()?;
+    assert!(select_stmt.column_int(0)? == 12);
+    select_stmt.step()?;
+    assert!(select_stmt.column_text(0)? == "str");
+    select_stmt.step()?;
+    assert!(select_stmt.column_blob(0)? == blob);
+
+    Ok(())
+}

libsql-sqlite3/ext/crr/rs/integration-check/tests/pk_only_tables.rs

@@ -37,7 +37,7 @@ fn junction_table() {
 
 // https://discord.com/channels/989870439897653248/989870440585494530/1081084118680485938
 #[test]
-fn dicord_report_1() {
+fn discord_report_1() {
     discord_report_1_impl().unwrap();
 }
 
@@ -46,7 +46,7 @@ fn sync_left_to_right(
     r: &dyn Connection,
     since: sqlite::int64,
 ) -> Result<ResultCode, ResultCode> {
-    let siteid_stmt = r.prepare_v2("SELECT crsql_siteid()")?;
+    let siteid_stmt = r.prepare_v2("SELECT crsql_site_id()")?;
     siteid_stmt.step()?;
     let siteid = siteid_stmt.column_blob(0)?;
 
@@ -56,8 +56,9 @@ fn sync_left_to_right(
     stmt_l.bind_blob(2, siteid, Destructor::STATIC)?;
 
     while stmt_l.step()? == ResultCode::ROW {
-        let stmt_r = r.prepare_v2("INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?)")?;
-        for x in 0..7 {
+        let stmt_r =
+            r.prepare_v2("INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)")?;
+        for x in 0..9 {
             stmt_r.bind_value(x + 1, stmt_l.column_value(x)?)?;
         }
         stmt_r.step()?;
@@ -70,7 +71,7 @@ fn sync_left_to_right(
 //     for_db: Option<&dyn Connection>,
 // ) -> Result<ResultCode, ResultCode> {
 //     let stmt = if let Some(for_db) = for_db {
-//         let siteid_stmt = for_db.prepare_v2("SELECT crsql_siteid()")?;
+//         let siteid_stmt = for_db.prepare_v2("SELECT crsql_site_id()")?;
 //         siteid_stmt.step()?;
 //         let siteid = siteid_stmt.column_blob(0)?;
 //         let stmt = db.prepare_v2(
@@ -260,10 +261,10 @@ fn discord_report_1_impl() -> Result<(), ResultCode> {
 
     let table = stmt.column_text(0)?;
     assert_eq!(table, "data");
-    let pk_val = stmt.column_text(1)?;
-    assert_eq!(pk_val, "42");
+    let pk_val = stmt.column_blob(1)?;
+    assert_eq!(pk_val, [0x01, 0x09, 0x2A]);
     let cid = stmt.column_text(2)?;
-    assert_eq!(cid, "__crsql_pko");
+    assert_eq!(cid, "-1");
     let val_type = stmt.column_type(3)?;
     assert_eq!(val_type, ColumnType::Null);
     let col_version = stmt.column_int64(4)?;

libsql-sqlite3/ext/crr/rs/integration-check/tests/sync_bit_honored.rs

@@ -0,0 +1,26 @@
+use sqlite::{Connection, ResultCode};
+use sqlite_nostd as sqlite;
+
+#[test]
+fn sync_bit_honored() {
+    sync_bit_honored_impl().unwrap();
+}
+
+// If sync bit is on, nothing gets written to clock tables for that connection.
+fn sync_bit_honored_impl() -> Result<(), ResultCode> {
+    let db = integration_utils::opendb()?;
+    let conn = &db.db;
+    conn.exec_safe("CREATE TABLE foo (a primary key, b);")?;
+    conn.exec_safe("SELECT crsql_as_crr('foo');")?;
+    conn.exec_safe("SELECT crsql_internal_sync_bit(1)")?;
+    conn.exec_safe("INSERT INTO foo VALUES (1, 2);")?;
+    conn.exec_safe("UPDATE foo SET b = 5 WHERE a = 1;")?;
+    conn.exec_safe("INSERT INTO foo VALUES (2, 2);")?;
+    conn.exec_safe("DELETE FROM foo WHERE a = 2;")?;
+
+    let stmt = conn.prepare_v2("SELECT 1 FROM foo__crsql_clock")?;
+    let result = stmt.step()?;
+    assert!(result == ResultCode::DONE);
+
+    Ok(())
+}

libsql-sqlite3/ext/crr/rs/integration-check/tests/teardown.rs

@@ -1,15 +1,16 @@
+use core::mem::forget;
 use sqlite::{Connection, ResultCode};
 use sqlite_nostd as sqlite;
 
-integration_utils::counter_setup!(1);
+integration_utils::counter_setup!(2);
 
 #[test]
-fn tear_down() {
-    tear_down_impl().unwrap();
+fn crr_to_table() {
+    crr_to_table_impl().unwrap();
     decrement_counter();
 }
 
-fn tear_down_impl() -> Result<(), ResultCode> {
+fn crr_to_table_impl() -> Result<(), ResultCode> {
     let db = integration_utils::opendb()?;
     db.db.exec_safe("CREATE TABLE foo (a primary key, b);")?;
     db.db.exec_safe("SELECT crsql_as_crr('foo');")?;
@@ -22,3 +23,22 @@ fn tear_down_impl() -> Result<(), ResultCode> {
     assert!(count == 0);
     Ok(())
 }
+
+#[test]
+fn statements_finalized_on_connection_close() {
+    let db_wrapped = integration_utils::opendb().expect("opened db");
+    let db = &db_wrapped.db;
+    db.exec_safe("CREATE TABLE foo (a primary key not null, b);")
+        .expect("created foo");
+    db.exec_safe("SELECT crsql_as_crr('foo');")
+        .expect("created crr");
+
+    let db_ptr = db.db;
+    // forget the db. We'll close it ourself
+    forget(db);
+    // forget so we don't run the `crsql_finalize` routine -- the close hook should now do that for us.
+    forget(db_wrapped);
+    let rc = sqlite::close(db_ptr);
+    assert!(rc == 0);
+    decrement_counter();
+}

libsql-sqlite3/ext/crr/rs/integration-check/tests/test_cl_set_vtab.rs

@@ -0,0 +1,102 @@
+use sqlite::{Connection, ManagedConnection, ResultCode};
+use sqlite_nostd as sqlite;
+
+/*
+Test:
+- create crr
+- destroy crr
+- use crr that was created
+- create if not exist vtab
+-
+*/
+
+#[test]
+fn create_crr_via_vtab() {
+    create_crr_via_vtab_impl().unwrap();
+}
+
+fn create_crr_via_vtab_impl() -> Result<(), ResultCode> {
+    let db = integration_utils::opendb()?;
+    let conn = &db.db;
+
+    conn.exec_safe("CREATE VIRTUAL TABLE foo_schema USING CLSet (a primary key, b);")?;
+    conn.exec_safe("INSERT INTO foo VALUES (1, 2);")?;
+    let stmt = conn.prepare_v2("SELECT count(*) FROM crsql_changes")?;
+    stmt.step()?;
+    let count = stmt.column_int(0)?;
+    assert_eq!(count, 1);
+    Ok(())
+}
+
+#[test]
+fn destroy_crr_via_vtab() {
+    destroy_crr_via_vtab_impl().unwrap();
+}
+
+fn destroy_crr_via_vtab_impl() -> Result<(), ResultCode> {
+    let db = integration_utils::opendb()?;
+    let conn = &db.db;
+
+    conn.exec_safe("CREATE VIRTUAL TABLE foo_schema USING CLSet (a primary key, b);")?;
+    conn.exec_safe("DROP TABLE foo_schema")?;
+    let stmt = conn.prepare_v2("SELECT count(*) FROM sqlite_master WHERE name LIKE '%foo%'")?;
+    stmt.step()?;
+    let count = stmt.column_int(0)?;
+    assert_eq!(count, 0);
+    Ok(())
+}
+
+#[test]
+fn create_invalid_crr() {
+    create_invalid_crr_impl().unwrap();
+}
+
+fn create_invalid_crr_impl() -> Result<(), ResultCode> {
+    let db = integration_utils::opendb()?;
+    let conn = &db.db;
+
+    let result = conn.exec_safe("CREATE VIRTUAL TABLE foo_schema USING CLSet (a, b);");
+    assert_eq!(result, Err(ResultCode::ERROR));
+    let msg = conn.errmsg().unwrap();
+    assert_eq!(
+        msg,
+        "Table foo has no primary key. CRRs must have a primary key"
+    );
+    Ok(())
+}
+
+#[test]
+fn create_if_not_exists() {
+    create_if_not_exists_impl().unwrap();
+}
+
+fn create_if_not_exists_impl() -> Result<(), ResultCode> {
+    let db = integration_utils::opendb()?;
+    let conn = &db.db;
+
+    conn.exec_safe(
+        "CREATE VIRTUAL TABLE IF NOT EXISTS foo_schema USING CLSet (a primary key, b);",
+    )?;
+    conn.exec_safe("INSERT INTO foo VALUES (1, 2);")?;
+    let stmt = conn.prepare_v2("SELECT count(*) FROM crsql_changes")?;
+    stmt.step()?;
+    let count = stmt.column_int(0)?;
+    assert_eq!(count, 1);
+    drop(stmt);
+    // second create is a no-op
+    conn.exec_safe(
+        "CREATE VIRTUAL TABLE IF NOT EXISTS foo_schema USING CLSet (a primary key, b);",
+    )?;
+    let stmt = conn.prepare_v2("SELECT count(*) FROM crsql_changes")?;
+    stmt.step()?;
+    let count = stmt.column_int(0)?;
+    assert_eq!(count, 1);
+    Ok(())
+}
+
+// and later migration tests
+// UPDATE foo SET schema = '...';
+// INSERT INTO foo (alter) VALUES ('...');
+// and auto-migrate tests for whole schema.
+// auto-migrate would...
+// - re-write `create vtab` things as `update foo set schema = ...` where those vtabs did not exist.

libsql-sqlite3/ext/crr/rs/integration-utils/rust-toolchain.toml

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "nightly-2023-06-17"

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/rust-toolchain.toml

@@ -0,0 +1,2 @@
+[toolchain]
+channel = "nightly-2023-06-17"

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite3_allocator/src/allocator.rs

@@ -7,6 +7,6 @@ unsafe impl GlobalAlloc for SQLite3Allocator {
     }
 
     unsafe fn dealloc(&self, ptr: *mut u8, _layout: Layout) {
-        sqlite3_capi::free(ptr);
+        sqlite3_capi::free(ptr as *mut core::ffi::c_void);
     }
 }

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite3_capi/deps/sqlite3.h

@@ -0,0 +1 @@
+../../../../../../sqlite3.h

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite3_capi/deps/sqlite3ext.h

@@ -0,0 +1 @@
+../../../../../../sqlite3ext.h

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite3_capi/src/capi.rs

@@ -1,6 +1,6 @@
 extern crate alloc;
 
-use core::ffi::{c_char, c_int, c_void, CStr};
+use core::ffi::{c_char, c_uchar, c_int, c_uint, c_void, CStr};
 use core::ptr;
 
 use alloc::borrow::ToOwned;
@@ -8,10 +8,28 @@ use alloc::ffi::CString;
 
 pub use crate::bindings::{
     sqlite3, sqlite3_api_routines as api_routines, sqlite3_context as context,
-    sqlite3_index_info as index_info, sqlite3_module as module, sqlite3_stmt as stmt,
-    sqlite3_uint64 as uint64, sqlite3_value as value, sqlite_int64 as int64,
-    SQLITE_DETERMINISTIC as DETERMINISTIC, SQLITE_DIRECTONLY as DIRECTONLY,
-    SQLITE_INNOCUOUS as INNOCUOUS, SQLITE_UTF8 as UTF8,
+    sqlite3_index_info as index_info,
+    sqlite3_index_info_sqlite3_index_constraint as index_constraint,
+    sqlite3_index_info_sqlite3_index_constraint_usage as index_constraint_usage,
+    sqlite3_module as module, sqlite3_stmt as stmt, sqlite3_uint64 as uint64,
+    sqlite3_value as value, sqlite3_vtab as vtab, sqlite3_vtab_cursor as vtab_cursor,
+    sqlite_int64 as int64, SQLITE_DETERMINISTIC as DETERMINISTIC, SQLITE_DIRECTONLY as DIRECTONLY,
+    SQLITE_INDEX_CONSTRAINT_EQ as INDEX_CONSTRAINT_EQ,
+    SQLITE_INDEX_CONSTRAINT_GE as INDEX_CONSTRAINT_GE,
+    SQLITE_INDEX_CONSTRAINT_GLOB as INDEX_CONSTRAINT_GLOB,
+    SQLITE_INDEX_CONSTRAINT_GT as INDEX_CONSTRAINT_GT,
+    SQLITE_INDEX_CONSTRAINT_IS as INDEX_CONSTRAINT_IS,
+    SQLITE_INDEX_CONSTRAINT_ISNOT as INDEX_CONSTRAINT_ISNOT,
+    SQLITE_INDEX_CONSTRAINT_ISNOTNULL as INDEX_CONSTRAINT_ISNOTNULL,
+    SQLITE_INDEX_CONSTRAINT_ISNULL as INDEX_CONSTRAINT_ISNULL,
+    SQLITE_INDEX_CONSTRAINT_LE as INDEX_CONSTRAINT_LE,
+    SQLITE_INDEX_CONSTRAINT_LIKE as INDEX_CONSTRAINT_LIKE,
+    SQLITE_INDEX_CONSTRAINT_LT as INDEX_CONSTRAINT_LT,
+    SQLITE_INDEX_CONSTRAINT_MATCH as INDEX_CONSTRAINT_MATCH,
+    SQLITE_INDEX_CONSTRAINT_NE as INDEX_CONSTRAINT_NE,
+    SQLITE_INDEX_CONSTRAINT_REGEXP as INDEX_CONSTRAINT_REGEXP, SQLITE_INNOCUOUS as INNOCUOUS,
+    SQLITE_PREPARE_NORMALIZE as PREPARE_NORMALIZE, SQLITE_PREPARE_NO_VTAB as PREPARE_NO_VTAB,
+    SQLITE_PREPARE_PERSISTENT as PREPARE_PERSISTENT, SQLITE_UTF8 as UTF8,
 };
 
 mod aliased {
@@ -23,12 +41,12 @@ mod aliased {
         sqlite3_bind_parameter_index as bind_parameter_index,
         sqlite3_bind_parameter_name as bind_parameter_name, sqlite3_bind_pointer as bind_pointer,
         sqlite3_bind_text as bind_text, sqlite3_bind_value as bind_value,
-        sqlite3_bind_zeroblob as bind_zeroblob, sqlite3_close as close,
-        sqlite3_column_blob as column_blob, sqlite3_column_bytes as column_bytes,
-        sqlite3_column_count as column_count, sqlite3_column_decltype as column_decltype,
-        sqlite3_column_double as column_double, sqlite3_column_int as column_int,
-        sqlite3_column_int64 as column_int64, sqlite3_column_name as column_name,
-        sqlite3_column_origin_name as column_origin_name,
+        sqlite3_bind_zeroblob as bind_zeroblob, sqlite3_clear_bindings as clear_bindings,
+        sqlite3_close as close, sqlite3_column_blob as column_blob,
+        sqlite3_column_bytes as column_bytes, sqlite3_column_count as column_count,
+        sqlite3_column_decltype as column_decltype, sqlite3_column_double as column_double,
+        sqlite3_column_int as column_int, sqlite3_column_int64 as column_int64,
+        sqlite3_column_name as column_name, sqlite3_column_origin_name as column_origin_name,
         sqlite3_column_table_name as column_table_name, sqlite3_column_text as column_text,
         sqlite3_column_type as column_type, sqlite3_column_value as column_value,
         sqlite3_commit_hook as commit_hook, sqlite3_context_db_handle as context_db_handle,
@@ -37,21 +55,23 @@ mod aliased {
         sqlite3_errcode as errcode, sqlite3_errmsg as errmsg, sqlite3_exec as exec,
         sqlite3_finalize as finalize, sqlite3_free as free, sqlite3_get_auxdata as get_auxdata,
         sqlite3_malloc as malloc, sqlite3_malloc64 as malloc64, sqlite3_next_stmt as next_stmt,
-        sqlite3_open as open, sqlite3_prepare_v2 as prepare_v2, sqlite3_reset as reset,
+        sqlite3_open as open, sqlite3_prepare_v2 as prepare_v2, sqlite3_prepare_v3 as prepare_v3,
+        sqlite3_randomness as randomness, sqlite3_reset as reset,
         sqlite3_result_blob as result_blob, sqlite3_result_double as result_double,
         sqlite3_result_error as result_error, sqlite3_result_error_code as result_error_code,
         sqlite3_result_int as result_int, sqlite3_result_int64 as result_int64,
         sqlite3_result_null as result_null, sqlite3_result_pointer as result_pointer,
         sqlite3_result_subtype as result_subtype, sqlite3_result_text as result_text,
-        sqlite3_result_value as result_value, sqlite3_set_auxdata as set_auxdata,
-        sqlite3_shutdown as shutdown, sqlite3_sql as sql, sqlite3_step as step,
-        sqlite3_value_blob as value_blob, sqlite3_value_bytes as value_bytes,
-        sqlite3_value_double as value_double, sqlite3_value_int as value_int,
-        sqlite3_value_int64 as value_int64, sqlite3_value_pointer as value_pointer,
-        sqlite3_value_subtype as value_subtype, sqlite3_value_text as value_text,
-        sqlite3_value_type as value_type, sqlite3_vtab_collation as vtab_collation,
-        sqlite3_vtab_config as vtab_config, sqlite3_vtab_distinct as vtab_distinct,
-        sqlite3_vtab_nochange as vtab_nochange, sqlite3_vtab_on_conflict as vtab_on_conflict,
+        sqlite3_result_value as result_value, sqlite3_set_authorizer as set_authorizer,
+        sqlite3_set_auxdata as set_auxdata, sqlite3_shutdown as shutdown, sqlite3_sql as sql,
+        sqlite3_step as step, sqlite3_user_data as user_data, sqlite3_value_blob as value_blob,
+        sqlite3_value_bytes as value_bytes, sqlite3_value_double as value_double,
+        sqlite3_value_int as value_int, sqlite3_value_int64 as value_int64,
+        sqlite3_value_pointer as value_pointer, sqlite3_value_subtype as value_subtype,
+        sqlite3_value_text as value_text, sqlite3_value_type as value_type,
+        sqlite3_vtab_collation as vtab_collation, sqlite3_vtab_config as vtab_config,
+        sqlite3_vtab_distinct as vtab_distinct, sqlite3_vtab_nochange as vtab_nochange,
+        sqlite3_vtab_on_conflict as vtab_on_conflict, sqlite3_get_autocommit as get_autocommit
     };
 }
 
@@ -83,9 +103,7 @@ macro_rules! invoke_sqlite {
 }
 
 pub extern "C" fn droprust(ptr: *mut c_void) {
-    unsafe {
-        ptr.drop_in_place();
-    }
+    unsafe { invoke_sqlite!(free, ptr as *mut c_void) }
 }
 
 #[macro_export]
@@ -95,6 +113,13 @@ macro_rules! args {
     };
 }
 
+#[macro_export]
+macro_rules! args_mut {
+    ($argc:expr, $argv:expr) => {
+        unsafe { slice::from_raw_parts_mut($argv, $argc as usize) }
+    };
+}
+
 static mut SQLITE3_API: *mut api_routines = ptr::null_mut();
 
 pub fn EXTENSION_INIT2(api: *mut api_routines) {
@@ -144,6 +169,18 @@ pub fn bind_int64(stmt: *mut stmt, c: c_int, i: int64) -> c_int {
     unsafe { invoke_sqlite!(bind_int64, stmt, c, i) }
 }
 
+pub fn bind_double(stmt: *mut stmt, c: c_int, f: f64) -> c_int {
+    unsafe { invoke_sqlite!(bind_double, stmt, c, f) }
+}
+
+pub fn bind_null(stmt: *mut stmt, c: c_int) -> c_int {
+    unsafe { invoke_sqlite!(bind_null, stmt, c) }
+}
+
+pub fn clear_bindings(stmt: *mut stmt) -> c_int {
+    unsafe { invoke_sqlite!(clear_bindings, stmt) }
+}
+
 pub fn bind_text(
     stmt: *mut stmt,
     c: c_int,
@@ -167,8 +204,8 @@ pub fn bind_text(
     }
 }
 
-pub fn bind_pointer(db: *mut stmt, i: c_int, p: *mut c_void, t: *const c_char) -> c_int {
-    unsafe { invoke_sqlite!(bind_pointer, db, i, p, t, None) }
+pub fn bind_pointer(stmt: *mut stmt, i: c_int, p: *mut c_void, t: *const c_char) -> c_int {
+    unsafe { invoke_sqlite!(bind_pointer, stmt, i, p, t, None) }
 }
 
 pub fn bind_value(stmt: *mut stmt, c: c_int, v: *mut value) -> c_int {
@@ -179,6 +216,10 @@ pub fn close(db: *mut sqlite3) -> c_int {
     unsafe { invoke_sqlite!(close, db) }
 }
 
+pub fn vtab_config(db: *mut sqlite3, options: u32) -> c_int {
+    unsafe { invoke_sqlite!(vtab_config, db, options as i32) }
+}
+
 pub type xCommitHook = unsafe extern "C" fn(*mut c_void) -> c_int;
 pub fn commit_hook(
     db: *mut sqlite3,
@@ -213,6 +254,10 @@ pub fn column_text<'a>(stmt: *mut stmt, c: c_int) -> &'a str {
     }
 }
 
+pub fn column_text_ptr(stmt: *mut stmt, c: c_int) -> *const c_uchar {
+    unsafe { invoke_sqlite!(column_text, stmt, c) }
+}
+
 pub fn column_blob(stmt: *mut stmt, c: c_int) -> *const c_void {
     unsafe { invoke_sqlite!(column_blob, stmt, c) }
 }
@@ -312,8 +357,8 @@ pub fn finalize(stmt: *mut stmt) -> c_int {
 }
 
 #[inline]
-pub fn free(ptr: *mut u8) {
-    unsafe { invoke_sqlite!(free, ptr as *mut c_void) }
+pub fn free(ptr: *mut c_void) {
+    unsafe { invoke_sqlite!(free, ptr) }
 }
 
 pub fn get_auxdata(context: *mut context, n: c_int) -> *mut c_void {
@@ -359,6 +404,21 @@ pub fn prepare_v2(
     unsafe { invoke_sqlite!(prepare_v2, db, sql, n, stmt, leftover) }
 }
 
+pub fn prepare_v3(
+    db: *mut sqlite3,
+    sql: *const c_char,
+    n: c_int,
+    flags: c_uint,
+    stmt: *mut *mut stmt,
+    leftover: *mut *const c_char,
+) -> c_int {
+    unsafe { invoke_sqlite!(prepare_v3, db, sql, n, flags, stmt, leftover) }
+}
+
+pub fn randomness(len: c_int, blob: *mut c_void) {
+    unsafe { invoke_sqlite!(randomness, len, blob) }
+}
+
 pub fn result_int(context: *mut context, v: c_int) {
     unsafe { invoke_sqlite!(result_int, context, v) }
 }
@@ -414,6 +474,10 @@ pub fn result_error_code(context: *mut context, code: c_int) {
     unsafe { invoke_sqlite!(result_error_code, context, code) }
 }
 
+pub fn result_value(ctx: *mut context, v: *mut value) {
+    unsafe { invoke_sqlite!(result_value, ctx, v) }
+}
+
 // d is our destructor function.
 // -- https://dev.to/kgrech/7-ways-to-pass-a-string-between-rust-and-c-4ieb
 pub fn result_text(context: *mut context, s: *const c_char, n: c_int, d: Destructor) {
@@ -436,6 +500,23 @@ pub fn result_subtype(context: *mut context, subtype: u32) {
     unsafe { invoke_sqlite!(result_subtype, context, subtype) }
 }
 
+pub type XAuthorizer = unsafe extern "C" fn(
+    user_data: *mut c_void,
+    action_code: c_int,
+    item_name: *const c_char,
+    sub_item_name: *const c_char,
+    db_name: *const c_char,
+    trigger_view_or_null: *const c_char,
+) -> c_int;
+
+pub fn set_authorizer(
+    db: *mut sqlite3,
+    xAuth: ::core::option::Option<XAuthorizer>,
+    user_data: *mut c_void,
+) -> c_int {
+    unsafe { invoke_sqlite!(set_authorizer, db, xAuth, user_data) }
+}
+
 pub fn set_auxdata(
     context: *mut context,
     n: c_int,
@@ -453,10 +534,16 @@ pub fn reset(stmt: *mut stmt) -> c_int {
     unsafe { invoke_sqlite!(reset, stmt) }
 }
 
+#[inline]
 pub fn step(stmt: *mut stmt) -> c_int {
     unsafe { invoke_sqlite!(step, stmt) }
 }
 
+#[inline]
+pub fn user_data(ctx: *mut context) -> *mut c_void {
+    unsafe { invoke_sqlite!(user_data, ctx) }
+}
+
 pub fn value_text<'a>(arg1: *mut value) -> &'a str {
     unsafe {
         let len = value_bytes(arg1);
@@ -501,3 +588,7 @@ pub fn value_pointer(arg1: *mut value, p: *mut c_char) -> *mut c_void {
 pub fn vtab_distinct(index_info: *mut index_info) -> c_int {
     unsafe { invoke_sqlite!(vtab_distinct, index_info) }
 }
+
+pub fn get_autocommit(db: *mut sqlite3) -> c_int {
+    unsafe { invoke_sqlite!(get_autocommit, db) }
+}

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite3_capi/wrapper.h

@@ -1 +1 @@
-#include "../../../../../sqlite3ext.h"
+#include "deps/sqlite3ext.h"

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite_nostd/Cargo.toml

@@ -10,7 +10,7 @@ crate-type = ["rlib"]
 [dependencies]
 sqlite3_capi = { path="../sqlite3_capi"}
 sqlite3_allocator = { path="../sqlite3_allocator" }
-num-traits = { version = ">=0.2.0", default-features = false }
+num-traits = { version = "0.2.15", default-features = false }
 num-derive = "0.3"
 
 [features]

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite_nostd/src/lib.rs

@@ -1,5 +1,6 @@
 #![no_std]
 #![feature(vec_into_raw_parts)]
+#![feature(error_in_core)]
 #![allow(non_camel_case_types)]
 
 mod nostd;

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite_nostd/src/nostd.rs

@@ -1,9 +1,13 @@
 extern crate alloc;
 
-use alloc::ffi::IntoStringError;
+use alloc::boxed::Box;
+use alloc::ffi::{IntoStringError, NulError};
 use alloc::vec::Vec;
 use alloc::{ffi::CString, string::String};
-use core::ffi::{c_char, c_int, c_void};
+use core::array::TryFromSliceError;
+use core::ffi::{c_char, c_int, c_void, CStr};
+use core::ptr::null_mut;
+use core::{error::Error, slice, str::Utf8Error};
 
 #[cfg(not(feature = "std"))]
 use num_derive::FromPrimitive;
@@ -13,6 +17,45 @@ use num_traits::FromPrimitive;
 pub use sqlite3_allocator::*;
 pub use sqlite3_capi::*;
 
+// https://www.sqlite.org/c3ref/c_alter_table.html
+#[derive(FromPrimitive, PartialEq, Debug)]
+pub enum ActionCode {
+    COPY = 0,
+    CREATE_INDEX = 1,
+    CREATE_TABLE = 2,
+    CREATE_TEMP_INDEX = 3,
+    CREATE_TEMP_TABLE = 4,
+    CREATE_TEMP_TRIGGER = 5,
+    CREATE_TEMP_VIEW = 6,
+    CREATE_TRIGGER = 7,
+    CREATE_VIEW = 8,
+    DELETE = 9,
+    DROP_INDEX = 10,
+    DROP_TABLE = 11,
+    DROP_TEMP_INDEX = 12,
+    DROP_TEMP_TABLE = 13,
+    DROP_TEMP_TRIGGER = 14,
+    DROP_TEMP_VIEW = 15,
+    DROP_TRIGGER = 16,
+    DROP_VIEW = 17,
+    INSERT = 18,
+    PRAGMA = 19,
+    READ = 20,
+    SELECT = 21,
+    TRANSACTION = 22,
+    UPDATE = 23,
+    ATTACH = 24,
+    DETACH = 25,
+    ALTER_TABLE = 26,
+    REINDEX = 27,
+    ANALYZE = 28,
+    CREATE_VTABLE = 29,
+    DROP_VTABLE = 30,
+    FUNCTION = 31,
+    SAVEPOINT = 32,
+    RECURSIVE = 33,
+}
+
 #[derive(FromPrimitive, PartialEq, Debug)]
 pub enum ResultCode {
     OK = 0,
@@ -125,6 +168,38 @@ pub enum ResultCode {
     NULL = 5000,
 }
 
+impl core::fmt::Display for ResultCode {
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        write!(f, "{:?}", self)
+    }
+}
+
+impl Error for ResultCode {}
+
+impl From<Utf8Error> for ResultCode {
+    fn from(_error: Utf8Error) -> Self {
+        ResultCode::FORMAT
+    }
+}
+
+impl From<TryFromSliceError> for ResultCode {
+    fn from(_error: TryFromSliceError) -> Self {
+        ResultCode::RANGE
+    }
+}
+
+impl From<NulError> for ResultCode {
+    fn from(_error: NulError) -> Self {
+        ResultCode::NOMEM
+    }
+}
+
+impl From<IntoStringError> for ResultCode {
+    fn from(_error: IntoStringError) -> Self {
+        ResultCode::FORMAT
+    }
+}
+
 #[derive(FromPrimitive, PartialEq, Debug)]
 pub enum ColumnType {
     Integer = 1,
@@ -145,8 +220,12 @@ pub fn open(filename: *const c_char) -> Result<ManagedConnection, ResultCode> {
     }
 }
 
+pub fn randomness(blob: &mut [u8]) {
+    sqlite3_capi::randomness(blob.len() as c_int, blob.as_mut_ptr() as *mut c_void)
+}
+
 pub struct ManagedConnection {
-    db: *mut sqlite3,
+    pub db: *mut sqlite3,
 }
 
 pub trait Connection {
@@ -168,6 +247,14 @@ pub trait Connection {
         destroy: Option<xDestroy>,
     ) -> Result<ResultCode, ResultCode>;
 
+    fn create_module_v2(
+        &self,
+        name: &str,
+        module: *const module,
+        user_data: Option<*mut c_void>,
+        destroy: Option<xDestroy>,
+    ) -> Result<ResultCode, ResultCode>;
+
     #[cfg(all(feature = "static", not(feature = "omit_load_extension")))]
     fn enable_load_extension(&self, enable: bool) -> Result<ResultCode, ResultCode>;
 
@@ -192,6 +279,16 @@ pub trait Connection {
     fn next_stmt(&self, s: Option<*mut stmt>) -> Option<*mut stmt>;
 
     fn prepare_v2(&self, sql: &str) -> Result<ManagedStmt, ResultCode>;
+
+    fn prepare_v3(&self, sql: &str, flags: u32) -> Result<ManagedStmt, ResultCode>;
+
+    fn set_authorizer(
+        &self,
+        x_auth: Option<XAuthorizer>,
+        user_data: *mut c_void,
+    ) -> Result<ResultCode, ResultCode>;
+
+    fn get_autocommit(&self) -> bool;
 }
 
 impl Connection for ManagedConnection {
@@ -221,6 +318,24 @@ impl Connection for ManagedConnection {
         )
     }
 
+    fn set_authorizer(
+        &self,
+        x_auth: Option<XAuthorizer>,
+        user_data: *mut c_void,
+    ) -> Result<ResultCode, ResultCode> {
+        self.db.set_authorizer(x_auth, user_data)
+    }
+
+    fn create_module_v2(
+        &self,
+        name: &str,
+        module: *const module,
+        user_data: Option<*mut c_void>,
+        destroy: Option<xDestroy>,
+    ) -> Result<ResultCode, ResultCode> {
+        self.db.create_module_v2(name, module, user_data, destroy)
+    }
+
     #[inline]
     fn next_stmt(&self, s: Option<*mut stmt>) -> Option<*mut stmt> {
         self.db.next_stmt(s)
@@ -231,6 +346,11 @@ impl Connection for ManagedConnection {
         self.db.prepare_v2(sql)
     }
 
+    #[inline]
+    fn prepare_v3(&self, sql: &str, flags: u32) -> Result<ManagedStmt, ResultCode> {
+        self.db.prepare_v3(sql, flags)
+    }
+
     #[inline]
     unsafe fn exec(&self, sql: *const c_char) -> Result<ResultCode, ResultCode> {
         self.db.exec(sql)
@@ -256,6 +376,11 @@ impl Connection for ManagedConnection {
         self.db.errcode()
     }
 
+    #[inline]
+    fn get_autocommit(&self) -> bool {
+        self.db.get_autocommit()
+    }
+
     #[cfg(all(feature = "static", not(feature = "omit_load_extension")))]
     fn load_extension(
         &self,
@@ -322,6 +447,26 @@ impl Connection for *mut sqlite3 {
         }
     }
 
+    fn create_module_v2(
+        &self,
+        name: &str,
+        module: *const module,
+        user_data: Option<*mut c_void>,
+        destroy: Option<xDestroy>,
+    ) -> Result<ResultCode, ResultCode> {
+        if let Ok(name) = CString::new(name) {
+            convert_rc(create_module_v2(
+                *self,
+                name.as_ptr(),
+                module,
+                user_data.unwrap_or(core::ptr::null_mut()),
+                destroy,
+            ))
+        } else {
+            Err(ResultCode::NOMEM)
+        }
+    }
+
     #[inline]
     fn prepare_v2(&self, sql: &str) -> Result<ManagedStmt, ResultCode> {
         let mut stmt = core::ptr::null_mut();
@@ -341,6 +486,26 @@ impl Connection for *mut sqlite3 {
         }
     }
 
+    #[inline]
+    fn prepare_v3(&self, sql: &str, flags: u32) -> Result<ManagedStmt, ResultCode> {
+        let mut stmt = core::ptr::null_mut();
+        let mut tail = core::ptr::null();
+        let rc = ResultCode::from_i32(prepare_v3(
+            *self,
+            sql.as_ptr() as *const c_char,
+            sql.len() as i32,
+            flags,
+            &mut stmt as *mut *mut stmt,
+            &mut tail as *mut *const c_char,
+        ))
+        .unwrap();
+        if rc == ResultCode::OK {
+            Ok(ManagedStmt { stmt: stmt })
+        } else {
+            Err(rc)
+        }
+    }
+
     #[inline]
     unsafe fn exec(&self, sql: *const c_char) -> Result<ResultCode, ResultCode> {
         convert_rc(exec(*self, sql))
@@ -407,6 +572,14 @@ impl Connection for *mut sqlite3 {
         }
     }
 
+    fn set_authorizer(
+        &self,
+        x_auth: Option<XAuthorizer>,
+        user_data: *mut c_void,
+    ) -> Result<ResultCode, ResultCode> {
+        convert_rc(set_authorizer(*self, x_auth, user_data))
+    }
+
     fn errmsg(&self) -> Result<String, IntoStringError> {
         errmsg(*self).into_string()
     }
@@ -414,10 +587,14 @@ impl Connection for *mut sqlite3 {
     fn errcode(&self) -> ResultCode {
         ResultCode::from_i32(errcode(*self)).unwrap()
     }
+
+    fn get_autocommit(&self) -> bool {
+        get_autocommit(*self) != 0
+    }
 }
 
-fn convert_rc(rc: i32) -> Result<ResultCode, ResultCode> {
-    let rc = ResultCode::from_i32(rc).unwrap();
+pub fn convert_rc(rc: i32) -> Result<ResultCode, ResultCode> {
+    let rc = ResultCode::from_i32(rc).unwrap_or(ResultCode::ABORT);
     if rc == ResultCode::OK {
         Ok(rc)
     } else {
@@ -426,7 +603,7 @@ fn convert_rc(rc: i32) -> Result<ResultCode, ResultCode> {
 }
 
 pub struct ManagedStmt {
-    stmt: *mut stmt,
+    pub stmt: *mut stmt,
 }
 
 impl ManagedStmt {
@@ -472,7 +649,16 @@ impl ManagedStmt {
 
     #[inline]
     pub fn column_text(&self, i: i32) -> Result<&str, ResultCode> {
-        Ok(column_text(self.stmt, i))
+        let len = column_bytes(self.stmt, i);
+        let ptr = column_text_ptr(self.stmt, i);
+        if ptr.is_null() {
+            Err(ResultCode::NULL)
+        } else {
+            Ok(unsafe {
+                let slice = core::slice::from_raw_parts(ptr as *const u8, len as usize);
+                core::str::from_utf8_unchecked(slice)
+            })
+        }
     }
 
     #[inline]
@@ -546,6 +732,16 @@ impl ManagedStmt {
     pub fn bind_int(&self, i: i32, val: i32) -> Result<ResultCode, ResultCode> {
         convert_rc(bind_int(self.stmt, i, val))
     }
+
+    #[inline]
+    pub fn bind_null(&self, i: i32) -> Result<ResultCode, ResultCode> {
+        convert_rc(bind_null(self.stmt, i))
+    }
+
+    #[inline]
+    pub fn clear_bindings(&self) -> Result<ResultCode, ResultCode> {
+        convert_rc(clear_bindings(self.stmt))
+    }
 }
 
 impl Drop for ManagedStmt {
@@ -561,14 +757,18 @@ pub trait Context {
     /// using it.
     fn result_text_owned(&self, text: String);
     fn result_text_transient(&self, text: &str);
-    fn result_text_static(&self, text: &'static str);
+    fn result_text_static(&self, text: &str);
     fn result_blob_owned(&self, blob: Vec<u8>);
-    fn result_blob_shared(&self, blob: &[u8]);
-    fn result_blob_static(&self, blob: &'static [u8]);
+    fn result_blob_transient(&self, blob: &[u8]);
+    fn result_blob_static(&self, blob: &[u8]);
     fn result_error(&self, text: &str);
     fn result_error_code(&self, code: ResultCode);
+    fn result_value(&self, value: *mut value);
+    fn result_double(&self, value: f64);
+    fn result_int64(&self, value: i64);
     fn result_null(&self);
     fn db_handle(&self) -> *mut sqlite3;
+    fn user_data(&self) -> *mut c_void;
 }
 
 impl Context for *mut context {
@@ -577,7 +777,8 @@ impl Context for *mut context {
         result_null(*self)
     }
 
-    /// TODO: do not use this right now! The drop is not dropping according to valgrind.
+    /// Passes ownership of the blob to SQLite without copying.
+    /// The blob must have been allocated with `sqlite3_malloc`!
     #[inline]
     fn result_text_owned(&self, text: String) {
         let (ptr, len, _) = text.into_raw_parts();
@@ -603,10 +804,10 @@ impl Context for *mut context {
         );
     }
 
-    /// Takes a reference to a string that is statically allocated.
+    /// Takes a reference to a string that will outlive SQLite's use of the string.
     /// SQLite will not copy this string.
     #[inline]
-    fn result_text_static(&self, text: &'static str) {
+    fn result_text_static(&self, text: &str) {
         result_text(
             *self,
             text.as_ptr() as *mut c_char,
@@ -616,15 +817,16 @@ impl Context for *mut context {
     }
 
     /// Passes ownership of the blob to SQLite without copying.
-    /// SQLite will drop the blob when it is finished with it.
+    /// The blob must have been allocated with `sqlite3_malloc`!
     #[inline]
     fn result_blob_owned(&self, blob: Vec<u8>) {
         let (ptr, len, _) = blob.into_raw_parts();
         result_blob(*self, ptr, len as i32, Destructor::CUSTOM(droprust));
     }
 
+    /// SQLite will make a copy of the blob
     #[inline]
-    fn result_blob_shared(&self, blob: &[u8]) {
+    fn result_blob_transient(&self, blob: &[u8]) {
         result_blob(
             *self,
             blob.as_ptr(),
@@ -634,7 +836,7 @@ impl Context for *mut context {
     }
 
     #[inline]
-    fn result_blob_static(&self, blob: &'static [u8]) {
+    fn result_blob_static(&self, blob: &[u8]) {
         result_blob(*self, blob.as_ptr(), blob.len() as i32, Destructor::STATIC);
     }
 
@@ -648,20 +850,202 @@ impl Context for *mut context {
         result_error_code(*self, code as c_int);
     }
 
+    #[inline]
+    fn result_value(&self, value: *mut value) {
+        result_value(*self, value);
+    }
+
+    #[inline]
+    fn result_double(&self, value: f64) {
+        result_double(*self, value);
+    }
+
+    #[inline]
+    fn result_int64(&self, value: i64) {
+        result_int64(*self, value);
+    }
+
     #[inline]
     fn db_handle(&self) -> *mut sqlite3 {
         context_db_handle(*self)
     }
+
+    #[inline]
+    fn user_data(&self) -> *mut c_void {
+        user_data(*self)
+    }
 }
 
 pub trait Stmt {
     fn sql(&self) -> &str;
+    fn bind_blob(&self, i: i32, val: &[u8], d: Destructor) -> Result<ResultCode, ResultCode>;
+    /// Gives SQLite ownership of the blob and has SQLite free it.
+    fn bind_blob_owned(&self, i: i32, val: Vec<u8>) -> Result<ResultCode, ResultCode>;
+    fn bind_value(&self, i: i32, val: *mut value) -> Result<ResultCode, ResultCode>;
+    fn bind_text(&self, i: i32, text: &str, d: Destructor) -> Result<ResultCode, ResultCode>;
+    fn bind_text_owned(&self, i: i32, text: String) -> Result<ResultCode, ResultCode>;
+    fn bind_int64(&self, i: i32, val: i64) -> Result<ResultCode, ResultCode>;
+    fn bind_int(&self, i: i32, val: i32) -> Result<ResultCode, ResultCode>;
+    fn bind_double(&self, i: i32, val: f64) -> Result<ResultCode, ResultCode>;
+    fn bind_null(&self, i: i32) -> Result<ResultCode, ResultCode>;
+
+    fn clear_bindings(&self) -> Result<ResultCode, ResultCode>;
+
+    fn column_value(&self, i: i32) -> *mut value;
+    fn column_int64(&self, i: i32) -> int64;
+    fn column_int(&self, i: i32) -> i32;
+    fn column_blob(&self, i: i32) -> &[u8];
+    fn column_double(&self, i: i32) -> f64;
+    fn column_text(&self, i: i32) -> &str;
+    fn column_bytes(&self, i: i32) -> i32;
+
+    fn finalize(&self) -> Result<ResultCode, ResultCode>;
+
+    fn reset(&self) -> Result<ResultCode, ResultCode>;
+    fn step(&self) -> Result<ResultCode, ResultCode>;
 }
 
 impl Stmt for *mut stmt {
     fn sql(&self) -> &str {
         unsafe { core::str::from_utf8_unchecked(core::ffi::CStr::from_ptr(sql(*self)).to_bytes()) }
     }
+
+    #[inline]
+    fn bind_blob(&self, i: i32, val: &[u8], d: Destructor) -> Result<ResultCode, ResultCode> {
+        convert_rc(bind_blob(
+            *self,
+            i,
+            val.as_ptr() as *const c_void,
+            val.len() as i32,
+            d,
+        ))
+    }
+
+    #[inline]
+    fn bind_blob_owned(&self, i: i32, val: Vec<u8>) -> Result<ResultCode, ResultCode> {
+        let (ptr, len, _) = val.into_raw_parts();
+        convert_rc(bind_blob(
+            *self,
+            i,
+            ptr as *const c_void,
+            len as i32,
+            Destructor::CUSTOM(droprust),
+        ))
+    }
+
+    #[inline]
+    fn bind_double(&self, i: i32, val: f64) -> Result<ResultCode, ResultCode> {
+        convert_rc(bind_double(*self, i, val))
+    }
+
+    #[inline]
+    fn bind_value(&self, i: i32, val: *mut value) -> Result<ResultCode, ResultCode> {
+        convert_rc(bind_value(*self, i, val))
+    }
+
+    #[inline]
+    fn bind_text(&self, i: i32, text: &str, d: Destructor) -> Result<ResultCode, ResultCode> {
+        convert_rc(bind_text(
+            *self,
+            i,
+            text.as_ptr() as *const c_char,
+            text.len() as i32,
+            d,
+        ))
+    }
+
+    #[inline]
+    fn bind_text_owned(&self, i: i32, text: String) -> Result<ResultCode, ResultCode> {
+        let (ptr, len, _) = text.into_raw_parts();
+        convert_rc(bind_text(
+            *self,
+            i,
+            ptr as *const c_char,
+            len as i32,
+            Destructor::CUSTOM(droprust),
+        ))
+    }
+
+    #[inline]
+    fn bind_int64(&self, i: i32, val: i64) -> Result<ResultCode, ResultCode> {
+        convert_rc(bind_int64(*self, i, val))
+    }
+
+    #[inline]
+    fn bind_int(&self, i: i32, val: i32) -> Result<ResultCode, ResultCode> {
+        convert_rc(bind_int(*self, i, val))
+    }
+
+    #[inline]
+    fn bind_null(&self, i: i32) -> Result<ResultCode, ResultCode> {
+        convert_rc(bind_null(*self, i))
+    }
+
+    #[inline]
+    fn clear_bindings(&self) -> Result<ResultCode, ResultCode> {
+        convert_rc(clear_bindings(*self))
+    }
+
+    #[inline]
+    fn column_value(&self, i: i32) -> *mut value {
+        column_value(*self, i)
+    }
+
+    #[inline]
+    fn column_int64(&self, i: i32) -> int64 {
+        column_int64(*self, i)
+    }
+
+    #[inline]
+    fn column_int(&self, i: i32) -> i32 {
+        column_int(*self, i)
+    }
+
+    #[inline]
+    fn column_blob(&self, i: i32) -> &[u8] {
+        let len = column_bytes(*self, i);
+        let ptr = column_blob(*self, i);
+        unsafe { core::slice::from_raw_parts(ptr as *const u8, len as usize) }
+    }
+
+    #[inline]
+    fn column_double(&self, i: i32) -> f64 {
+        column_double(*self, i)
+    }
+
+    #[inline]
+    fn column_text(&self, i: i32) -> &str {
+        column_text(*self, i)
+    }
+
+    #[inline]
+    fn column_bytes(&self, i: i32) -> i32 {
+        column_bytes(*self, i)
+    }
+
+    #[inline]
+    fn reset(&self) -> Result<ResultCode, ResultCode> {
+        convert_rc(reset(*self))
+    }
+
+    #[inline]
+    fn step(&self) -> Result<ResultCode, ResultCode> {
+        match ResultCode::from_i32(step(*self)) {
+            Some(ResultCode::ROW) => Ok(ResultCode::ROW),
+            Some(ResultCode::DONE) => Ok(ResultCode::DONE),
+            Some(rc) => Err(rc),
+            None => Err(ResultCode::ERROR),
+        }
+    }
+
+    #[inline]
+    fn finalize(&self) -> Result<ResultCode, ResultCode> {
+        match ResultCode::from_i32(finalize(*self)) {
+            Some(ResultCode::OK) => Ok(ResultCode::OK),
+            Some(rc) => Err(rc),
+            None => Err(ResultCode::ABORT),
+        }
+    }
 }
 
 pub trait Value {
@@ -710,3 +1094,175 @@ impl Value for *mut value {
         value_bytes(*self)
     }
 }
+
+pub trait StrRef {
+    fn set(&self, val: &str);
+}
+
+impl StrRef for *mut *mut c_char {
+    /**
+     * Sets the error message, copying the contents of `val`.
+     * If the error has already been set, future calls to `set` are ignored.
+     */
+    fn set(&self, val: &str) {
+        unsafe {
+            if **self != null_mut() {
+                return;
+            }
+            if let Ok(cstring) = CString::new(val) {
+                **self = cstring.into_raw();
+            } else {
+                if let Ok(s) = CString::new("Failed setting error message.") {
+                    **self = s.into_raw();
+                }
+            }
+        }
+    }
+}
+
+// TODO: on `T` can I enforce that T has a pointer of name `base` as first item to `vtab`?
+pub trait VTabRef {
+    fn set<T>(&self, val: Box<T>);
+}
+
+impl VTabRef for *mut *mut vtab {
+    fn set<T>(&self, val: Box<T>) {
+        unsafe {
+            let raw_val = Box::into_raw(val);
+            **self = raw_val.cast::<sqlite3_capi::vtab>();
+        }
+    }
+}
+
+pub trait CursorRef {
+    fn set<T>(&self, val: Box<T>);
+}
+
+impl CursorRef for *mut *mut vtab_cursor {
+    fn set<T>(&self, val: Box<T>) {
+        unsafe {
+            let raw_val = Box::into_raw(val);
+            **self = raw_val.cast::<sqlite3_capi::vtab_cursor>();
+        }
+    }
+}
+
+pub trait VTab {
+    fn set_err(&self, val: &str);
+}
+
+impl VTab for *mut vtab {
+    /**
+     * Sets the error message, copying the contents of `val`.
+     * If the error has already been set, future calls to `set` are ignored.
+     */
+    fn set_err(&self, val: &str) {
+        unsafe {
+            if (**self).zErrMsg != null_mut() {
+                return;
+            }
+            if let Ok(e) = CString::new(val) {
+                (**self).zErrMsg = e.into_raw();
+            }
+        }
+    }
+}
+
+// from: https://github.com/asg017/sqlite-loadable-rs/blob/main/src/table.rs#L722
+pub struct VTabArgs<'a> {
+    /// Name of the module being invoked, the argument in the USING clause.
+    /// Example: `"CREATE VIRTUAL TABLE xxx USING custom_vtab"` would have
+    /// a `module_name` of `"custom_vtab"`.
+    /// Sourced from `argv[0]`
+    pub module_name: &'a str,
+    /// Name of the database where the virtual table will be created,
+    /// typically `"main"` or `"temp"` or another name from an
+    /// [`ATTACH`'ed database](https://www.sqlite.org/lang_attach.html).
+    /// Sourced from `argv[1]`
+    pub database_name: &'a str,
+
+    /// Name of the table being created.
+    /// Example: `"CREATE VIRTUAL TABLE xxx USING custom_vtab"` would
+    /// have a `table_name` of `"xxx"`.
+    /// Sourced from `argv[2]`
+    pub table_name: &'a str,
+    /// The remaining arguments given in the constructor of the virtual
+    /// table, inside `CREATE VIRTUAL TABLE xxx USING custom_vtab(...)`.
+    /// Sourced from `argv[3:]`
+    pub arguments: Vec<&'a str>,
+}
+
+/// Generally do not use this. Does a bunch of copying.
+fn c_string_to_str<'a>(c: *const c_char) -> Result<&'a str, Utf8Error> {
+    let s = unsafe { CStr::from_ptr(c).to_str()? };
+    Ok(s)
+}
+
+pub fn parse_vtab_args<'a>(
+    argc: c_int,
+    argv: *const *const c_char,
+) -> Result<VTabArgs<'a>, Utf8Error> {
+    let raw_args = unsafe { slice::from_raw_parts(argv, argc as usize) };
+    let mut args = Vec::with_capacity(argc as usize);
+    for arg in raw_args {
+        args.push(c_string_to_str(*arg)?);
+    }
+
+    // SQLite guarantees that argv[0-2] will be filled, hence the .expects() -
+    // If SQLite is wrong, then may god save our souls
+    let module_name = args
+        .get(0)
+        .expect("argv[0] should be the name of the module");
+    let database_name = args
+        .get(1)
+        .expect("argv[1] should be the name of the database the module is in");
+    let table_name = args
+        .get(2)
+        .expect("argv[2] should be the name of the virtual table");
+    let arguments = &args[3..];
+
+    Ok(VTabArgs {
+        module_name,
+        database_name,
+        table_name,
+        arguments: arguments.to_vec(),
+    })
+}
+
+pub fn declare_vtab(db: *mut sqlite3, def: &str) -> Result<ResultCode, ResultCode> {
+    let cstring = CString::new(def)?;
+    let ret = sqlite3_capi::declare_vtab(db, cstring.as_ptr());
+    convert_rc(ret)
+}
+
+pub fn vtab_config(db: *mut sqlite3, options: u32) -> Result<ResultCode, ResultCode> {
+    let rc = sqlite3_capi::vtab_config(db, options);
+    convert_rc(rc)
+}
+
+// type xCreateC = extern "C" fn(
+//     *mut sqlite3,
+//     *mut c_void,
+//     c_int,
+//     *const *const c_char,
+//     *mut *mut vtab,
+//     *mut *mut c_char,
+// ) -> c_int;
+
+// // return a lambda that invokes f appropriately?
+// pub const fn xCreate(
+//     f: fn(
+//         db: *mut sqlite3,
+//         aux: *mut c_void,
+//         args: Vec<&str>,
+//         tab: *mut *mut vtab,   // declare tab for them?
+//         err: *mut *mut c_char, // box?
+//     ) -> Result<ResultCode, ResultCode>,
+// ) -> xCreateC {
+//     move |db, aux, argc, argv, ppvtab, errmsg| match f(db, aux, str_args, ppvtab, errmsg) {
+//         Ok(rc) => rc as c_int,
+//         Err(rc) => rc as c_int,
+//     }
+// }
+
+// *mut sqlite3, *mut c_void, Vec<&str>, *mut *mut vtab, *mut *mut c_char

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite_web/src/lib.rs

@@ -1,5 +1,4 @@
 #![no_std]
-#![feature(alloc_error_handler)]
 #![feature(core_intrinsics)]
 #![allow(non_camel_case_types)]
 

libsql-sqlite3/ext/crr/rs/sqlite-rs-embedded/sqlite_web/src/web.rs

@@ -13,41 +13,6 @@ fn panic(_info: &PanicInfo) -> ! {
 }
 
 use core::alloc::Layout;
-#[alloc_error_handler]
-fn oom(_: Layout) -> ! {
-    core::intrinsics::abort()
-}
-
-#[no_mangle]
-pub extern "C" fn __rust_alloc(size: usize, align: usize) -> *mut u8 {
-    unsafe { ALLOCATOR.alloc(Layout::from_size_align_unchecked(size, align)) }
-}
-
-#[no_mangle]
-pub extern "C" fn __rust_dealloc(ptr: *mut u8, size: usize, align: usize) {
-    unsafe { ALLOCATOR.dealloc(ptr, Layout::from_size_align_unchecked(size, align)) }
-}
-
-#[no_mangle]
-pub extern "C" fn __rust_realloc(
-    ptr: *mut u8,
-    old_size: usize,
-    align: usize,
-    size: usize,
-) -> *mut u8 {
-    unsafe {
-        ALLOCATOR.realloc(
-            ptr,
-            Layout::from_size_align_unchecked(old_size, align),
-            size,
-        )
-    }
-}
-
-#[no_mangle]
-pub extern "C" fn __rust_alloc_zeroed(size: usize, align: usize) -> *mut u8 {
-    unsafe { ALLOCATOR.alloc_zeroed(Layout::from_size_align_unchecked(size, align)) }
-}
 
 #[no_mangle]
 pub fn __rust_alloc_error_handler(_: Layout) -> ! {

libsql-sqlite3/ext/crr/src/changes-vtab-common.c

@@ -1,81 +0,0 @@
-#include "changes-vtab-common.h"
-
-#include <string.h>
-
-#include "consts.h"
-#include "util.h"
-
-/**
- * Extracts a where expression from the provided column names and list of `quote
- * concatenated` column values.
- *
- * quote concated column values can be untrusted input as we validate those
- * values.
- *
- * TODO: a future improvement would be to encode changesets into something like
- * flat buffers so we can extract out individual values and bind them to the SQL
- * statement. The values are currently represented on the wire in a text
- * encoding that is not suitable for direct binding but rather for direct
- * inclusion into the SQL string. We thus have to ensure we validate the
- * provided string.
- */
-char *crsql_extractWhereList(crsql_ColumnInfo *zColumnInfos, int columnInfosLen,
-                             const char *quoteConcatedVals) {
-  char **zzParts = 0;
-  if (columnInfosLen == 1) {
-    zzParts = sqlite3_malloc(1 * sizeof(char *));
-    zzParts[0] = crsql_strdup(quoteConcatedVals);
-  } else {
-    // zzParts will not be greater or less than columnInfosLen.
-    zzParts = crsql_splitQuoteConcat(quoteConcatedVals, columnInfosLen);
-  }
-
-  if (zzParts == 0) {
-    return 0;
-  }
-
-  for (int i = 0; i < columnInfosLen; ++i) {
-    // this is safe since pks are extracted as `quote` in the prior queries
-    // %z will de-allocate pksArr[i] so we can re-allocate it in the
-    // assignment
-    zzParts[i] =
-        sqlite3_mprintf("\"%s\" = %z", zColumnInfos[i].name, zzParts[i]);
-  }
-
-  // join2 will free the contents of zzParts given identity is a pass-thru
-  char *ret = crsql_join2((char *(*)(const char *)) & crsql_identity, zzParts,
-                          columnInfosLen, " AND ");
-  sqlite3_free(zzParts);
-  return ret;
-}
-
-/**
- * Should only be called by `quoteConcatedValuesAsList`
- */
-static char *crsql_quotedValuesAsList(char **parts, int numParts) {
-  int len = 0;
-  for (int i = 0; i < numParts; ++i) {
-    len += strlen(parts[i]);
-  }
-  len += numParts - 1;
-  char *ret = sqlite3_malloc((len + 1) * sizeof *ret);
-  crsql_joinWith(ret, parts, numParts, ',');
-  ret[len] = '\0';
-
-  return ret;
-}
-
-char *crsql_quoteConcatedValuesAsList(const char *quoteConcatedVals, int len) {
-  char **parts = crsql_splitQuoteConcat(quoteConcatedVals, len);
-  if (parts == 0) {
-    return 0;
-  }
-
-  char *ret = crsql_quotedValuesAsList(parts, len);
-  for (int i = 0; i < len; ++i) {
-    sqlite3_free(parts[i]);
-  }
-  sqlite3_free(parts);
-
-  return ret;
-}

libsql-sqlite3/ext/crr/src/changes-vtab-common.h

@@ -1,21 +0,0 @@
-#ifndef CHANGES_VTAB_COMMON_H
-#define CHANGES_VTAB_COMMON_H
-
-#include "sqlite3ext.h"
-SQLITE_EXTENSION_INIT3
-#include "tableinfo.h"
-
-#define CHANGES_SINCE_VTAB_TBL 0
-#define CHANGES_SINCE_VTAB_PK 1
-#define CHANGES_SINCE_VTAB_CID 2
-#define CHANGES_SINCE_VTAB_CVAL 3
-#define CHANGES_SINCE_VTAB_COL_VRSN 4
-#define CHANGES_SINCE_VTAB_DB_VRSN 5
-#define CHANGES_SINCE_VTAB_SITE_ID 6
-
-char *crsql_extractWhereList(crsql_ColumnInfo *zColumnInfos, int columnInfosLen,
-                             const char *quoteConcatedVals);
-
-char *crsql_quoteConcatedValuesAsList(const char *quoteConcatedVals, int len);
-
-#endif

libsql-sqlite3/ext/crr/src/changes-vtab-common.test.c

@@ -1,72 +0,0 @@
-#include "changes-vtab-common.h"
-
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "consts.h"
-#include "crsqlite.h"
-
-#ifndef CHECK_OK
-#define CHECK_OK         \
-  if (rc != SQLITE_OK) { \
-    goto fail;           \
-  }
-#endif
-
-static void testExtractWhereList() {
-  printf("ExtractWhereList\n");
-  crsql_ColumnInfo columnInfos[3];
-
-  columnInfos[0].name = "foo";
-  columnInfos[1].name = "bar";
-  columnInfos[2].name = "baz";
-
-  // Test not enough parts
-  char *whereList = crsql_extractWhereList(columnInfos, 3, "");
-  assert(whereList == 0);
-  sqlite3_free(whereList);
-
-  // Test too many parts
-  whereList = crsql_extractWhereList(columnInfos, 3, "'a'|'b'|'c'|'d'");
-  assert(whereList == 0);
-
-  // Just right
-  whereList = crsql_extractWhereList(columnInfos, 3, "'a'|'b'|'c'");
-  assert(strcmp("\"foo\" = 'a' AND \"bar\" = 'b' AND \"baz\" = 'c'",
-                whereList) == 0);
-  sqlite3_free(whereList);
-
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
-
-static void testQuoteConcatedValuesAsList() {
-  printf("QuoteConcatedValuesAsList\n");
-
-  char *l = crsql_quoteConcatedValuesAsList("5", 1);
-  assert(strcmp(l, "5") == 0);
-  sqlite3_free(l);
-
-  l = crsql_quoteConcatedValuesAsList("'h'", 1);
-  assert(strcmp(l, "'h'") == 0);
-  sqlite3_free(l);
-
-  l = crsql_quoteConcatedValuesAsList("'h'|1|X'aa'", 3);
-  assert(strcmp(l, "'h',1,X'aa'") == 0);
-  sqlite3_free(l);
-
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
-
-static void testQuotedValuesAsList() {
-  printf("QuotedValuesAsList\n");
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
-
-void crsqlChangesVtabCommonTestSuite() {
-  printf("\e[47m\e[1;30mSuite: crsql_changesVtabCommon\e[0m\n");
-  testExtractWhereList();
-  testQuoteConcatedValuesAsList();
-  testQuotedValuesAsList();
-}

libsql-sqlite3/ext/crr/src/changes-vtab-read.c

@@ -1,117 +0,0 @@
-#include "changes-vtab-read.h"
-
-#include <string.h>
-
-#include "consts.h"
-#include "util.h"
-
-/**
- * Construct the query to grab the changes made against
- * rows in a given table
- */
-char *crsql_changesQueryForTable(crsql_TableInfo *tableInfo, int idxNum) {
-  if (tableInfo->pksLen == 0) {
-    return 0;
-  }
-
-  char *zSql = sqlite3_mprintf(
-      "SELECT\
-      '%s' as tbl,\
-      %z as pks,\
-      __crsql_col_name as cid,\
-      __crsql_col_version as col_vrsn,\
-      __crsql_db_version as db_vrsn,\
-      __crsql_site_id as site_id\
-    FROM \"%s__crsql_clock\"\
-    WHERE\
-      site_id IS %s ?\
-    AND\
-      db_vrsn > ?",
-      tableInfo->tblName, crsql_quoteConcat(tableInfo->pks, tableInfo->pksLen),
-      tableInfo->tblName, (idxNum & 8) == 8 ? "" : "NOT");
-
-  return zSql;
-}
-
-// TODO: here we could do all the filtering to remove:
-// - records with no longer existing columns
-// - all rows prior to a delete entry for a row
-//
-// or we can do that in `xNext`
-// or we can compact the table on `commit_alter`
-// compacting in commit alter is likely the simplest option
-// with minimal impact on perf of normal operations
-/**
- * Union all the crr tables together to get a comprehensive
- * set of changes
- */
-char *crsql_changesUnionQuery(crsql_TableInfo **tableInfos, int tableInfosLen,
-                              int idxNum) {
-  char **unionsArr = sqlite3_malloc(tableInfosLen * sizeof(char *));
-  char *unionsStr = 0;
-  int i = 0;
-
-  // TODO: what if there are no table infos?
-  for (i = 0; i < tableInfosLen; ++i) {
-    unionsArr[i] = crsql_changesQueryForTable(tableInfos[i], idxNum);
-    if (unionsArr[i] == 0) {
-      for (int j = 0; j < i; j++) {
-        sqlite3_free(unionsArr[j]);
-      }
-      return 0;
-    }
-
-    if (i < tableInfosLen - 1) {
-      unionsArr[i] = sqlite3_mprintf("%z %s ", unionsArr[i], UNION);
-    }
-  }
-
-  // move the array of strings into a single string
-  unionsStr = crsql_join(unionsArr, tableInfosLen);
-  // free the strings in the array
-  for (i = 0; i < tableInfosLen; ++i) {
-    sqlite3_free(unionsArr[i]);
-  }
-  sqlite3_free(unionsArr);
-
-  // compose the final query
-  return sqlite3_mprintf(
-      "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id FROM (%z) ORDER BY "
-      "db_vrsn, tbl ASC",
-      unionsStr);
-  // %z frees unionsStr https://www.sqlite.org/printf.html#percentz
-}
-
-/**
- * Create the query to pull the backing data from the actual row based
- * on the version mape of changed columns.
- *
- * This pulls all columns that have changed from the row.
- * The values of the columns are quote-concated for compliance
- * with union query constraints. I.e., that all tables must have same
- * output number of columns.
- *
- * TODO: potential improvement would be to store a binary
- * representation of the data via flat buffers.
- *
- * This will fill pRowStmt in the cursor.
- *
- * TODO: We could theoretically prepare all of these queries up
- * front on vtab initialization so we don't have to
- * re-compile them for each row fetched.
- */
-char *crsql_rowPatchDataQuery(sqlite3 *db, crsql_TableInfo *tblInfo,
-                              const char *colName, const char *pks) {
-  char *pkWhereList =
-      crsql_extractWhereList(tblInfo->pks, tblInfo->pksLen, pks);
-  if (pkWhereList == 0) {
-    return 0;
-  }
-  // TODO: should we `quote([])` so it fatals on missing columns?
-  // we'd need something other than `%w` to escape [
-  // %w assumes and escapes \"
-  char *zSql = sqlite3_mprintf("SELECT quote(\"%w\") FROM \"%w\" WHERE %z",
-                               colName, tblInfo->tblName, pkWhereList);
-
-  return zSql;
-}

libsql-sqlite3/ext/crr/src/changes-vtab-read.h

@@ -1,23 +0,0 @@
-#ifndef CHANGES_VTAB_READ_H
-#define CHANGES_VTAB_READ_H
-
-#include "sqlite3ext.h"
-SQLITE_EXTENSION_INIT3
-
-#include "changes-vtab-common.h"
-#include "tableinfo.h"
-
-char *crsql_changesQueryForTable(crsql_TableInfo *tableInfo, int idxNum);
-
-#define TBL 0
-#define PKS 1
-#define CID 2
-#define COL_VRSN 3
-#define DB_VRSN 4
-#define SITE_ID 5
-char *crsql_changesUnionQuery(crsql_TableInfo **tableInfos, int tableInfosLen,
-                              int idxNum);
-char *crsql_rowPatchDataQuery(sqlite3 *db, crsql_TableInfo *tblInfo,
-                              const char *colName, const char *pks);
-
-#endif

libsql-sqlite3/ext/crr/src/changes-vtab-read.test.c

@@ -1,5 +1,3 @@
-#include "changes-vtab-read.h"
-
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -7,49 +5,10 @@
 
 #include "consts.h"
 #include "crsqlite.h"
+#include "rust.h"
 
 int crsql_close(sqlite3 *db);
 
-static void testChangesQueryForTable() {
-  printf("ChangeQueryForTable\n");
-  int rc = SQLITE_OK;
-  sqlite3 *db;
-  char *err = 0;
-  crsql_TableInfo *tblInfo = 0;
-  rc = sqlite3_open(":memory:", &db);
-
-  rc += sqlite3_exec(db, "create table foo (a primary key, b);", 0, 0, &err);
-  rc += sqlite3_exec(db, "select crsql_as_crr('foo');", 0, 0, &err);
-  rc += crsql_getTableInfo(db, "foo", &tblInfo, &err);
-  assert(rc == SQLITE_OK);
-
-  char *query = crsql_changesQueryForTable(tblInfo, 6);
-
-  assert(strcmp(query,
-                "SELECT      \'foo\' as tbl,      quote(\"a\") as pks,      "
-                "__crsql_col_name as cid,      __crsql_col_version as "
-                "col_vrsn,      __crsql_db_version as db_vrsn,      "
-                "__crsql_site_id as site_id    FROM \"foo__crsql_clock\"    "
-                "WHERE      site_id IS NOT ?    AND      db_vrsn > ?") == 0);
-  sqlite3_free(query);
-
-  query = crsql_changesQueryForTable(tblInfo, 8);
-  assert(strcmp(query,
-                "SELECT      \'foo\' as tbl,      quote(\"a\") as pks,      "
-                "__crsql_col_name as cid,      __crsql_col_version as "
-                "col_vrsn,      __crsql_db_version as db_vrsn,      "
-                "__crsql_site_id as site_id    FROM \"foo__crsql_clock\"    "
-                "WHERE      site_id IS  ?    AND      db_vrsn > ?") == 0);
-  sqlite3_free(query);
-
-  printf("\t\e[0;32mSuccess\e[0m\n");
-
-  sqlite3_free(err);
-  crsql_freeTableInfo(tblInfo);
-  crsql_close(db);
-  assert(rc == SQLITE_OK);
-}
-
 static void testChangesUnionQuery() {
   printf("ChangesUnionQuery\n");
 
@@ -68,36 +27,75 @@ static void testChangesUnionQuery() {
   rc += crsql_getTableInfo(db, "bar", &tblInfos[1], &err);
   assert(rc == SQLITE_OK);
 
-  char *query = crsql_changesUnionQuery(tblInfos, 2, 6);
+  char *query = crsql_changes_union_query(tblInfos, 2, "");
+  printf("X:%sX\n", query);
   assert(
-      strcmp(
-          query,
-          "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id FROM (SELECT      "
-          "\'foo\' as tbl,      quote(\"a\") as pks,      __crsql_col_name as "
-          "cid,      __crsql_col_version as col_vrsn,      __crsql_db_version "
-          "as db_vrsn,      __crsql_site_id as site_id    FROM "
-          "\"foo__crsql_clock\"    WHERE      site_id IS NOT ?    AND      "
-          "db_vrsn > ? UNION SELECT      \'bar\' as tbl,      quote(\"x\") as "
-          "pks,      __crsql_col_name as cid,      __crsql_col_version as "
-          "col_vrsn,      __crsql_db_version as db_vrsn,      __crsql_site_id "
-          "as site_id    FROM \"bar__crsql_clock\"    WHERE      site_id IS "
-          "NOT ?    AND      db_vrsn > ?) ORDER BY db_vrsn, tbl ASC") == 0);
+      strcmp(query,
+             "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id, _rowid_, seq, "
+             "cl FROM (SELECT\n"
+             "          'foo' as tbl,\n"
+             "          crsql_pack_columns(t1.\"a\") as pks,\n"
+             "          t1.__crsql_col_name as cid,\n"
+             "          t1.__crsql_col_version as col_vrsn,\n"
+             "          t1.__crsql_db_version as db_vrsn,\n"
+             "          t3.site_id as site_id,\n"
+             "          t1._rowid_,\n"
+             "          t1.__crsql_seq as seq,\n"
+             "          COALESCE(t2.__crsql_col_version, 1) as cl\n"
+             "      FROM \"foo__crsql_clock\" AS t1 LEFT JOIN "
+             "\"foo__crsql_clock\" AS t2 ON\n"
+             "      t1.\"a\" = t2.\"a\" AND t2.__crsql_col_name = '-1' LEFT "
+             "JOIN crsql_site_id as t3 ON t1.__crsql_site_id = t3.ordinal "
+             "UNION ALL SELECT\n"
+             "          'bar' as tbl,\n"
+             "          crsql_pack_columns(t1.\"x\") as pks,\n"
+             "          t1.__crsql_col_name as cid,\n"
+             "          t1.__crsql_col_version as col_vrsn,\n"
+             "          t1.__crsql_db_version as db_vrsn,\n"
+             "          t3.site_id as site_id,\n"
+             "          t1._rowid_,\n"
+             "          t1.__crsql_seq as seq,\n"
+             "          COALESCE(t2.__crsql_col_version, 1) as cl\n"
+             "      FROM \"bar__crsql_clock\" AS t1 LEFT JOIN "
+             "\"bar__crsql_clock\" AS t2 ON\n"
+             "      t1.\"x\" = t2.\"x\" AND t2.__crsql_col_name = '-1' LEFT "
+             "JOIN crsql_site_id as t3 ON t1.__crsql_site_id = t3.ordinal) ") ==
+      0);
   sqlite3_free(query);
 
-  query = crsql_changesUnionQuery(tblInfos, 2, 8);
-  assert(
-      strcmp(
-          query,
-          "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id FROM (SELECT      "
-          "\'foo\' as tbl,      quote(\"a\") as pks,      __crsql_col_name as "
-          "cid,      __crsql_col_version as col_vrsn,      __crsql_db_version "
-          "as db_vrsn,      __crsql_site_id as site_id    FROM "
-          "\"foo__crsql_clock\"    WHERE      site_id IS  ?    AND      "
-          "db_vrsn > ? UNION SELECT      \'bar\' as tbl,      quote(\"x\") as "
-          "pks,      __crsql_col_name as cid,      __crsql_col_version as "
-          "col_vrsn,      __crsql_db_version as db_vrsn,      __crsql_site_id "
-          "as site_id    FROM \"bar__crsql_clock\"    WHERE      site_id IS  ? "
-          "   AND      db_vrsn > ?) ORDER BY db_vrsn, tbl ASC") == 0);
+  query = crsql_changes_union_query(tblInfos, 2,
+                                    "WHERE site_id IS ? AND db_vrsn > ?");
+  assert(strcmp(query,
+                "SELECT tbl, pks, cid, col_vrsn, db_vrsn, site_id, _rowid_, "
+                "seq, cl FROM (SELECT\n"
+                "          'foo' as tbl,\n"
+                "          crsql_pack_columns(t1.\"a\") as pks,\n"
+                "          t1.__crsql_col_name as cid,\n"
+                "          t1.__crsql_col_version as col_vrsn,\n"
+                "          t1.__crsql_db_version as db_vrsn,\n"
+                "          t3.site_id as site_id,\n"
+                "          t1._rowid_,\n"
+                "          t1.__crsql_seq as seq,\n"
+                "          COALESCE(t2.__crsql_col_version, 1) as cl\n"
+                "      FROM \"foo__crsql_clock\" AS t1 LEFT JOIN "
+                "\"foo__crsql_clock\" AS t2 ON\n"
+                "      t1.\"a\" = t2.\"a\" AND t2.__crsql_col_name = '-1' LEFT "
+                "JOIN crsql_site_id as t3 ON t1.__crsql_site_id = t3.ordinal "
+                "UNION ALL SELECT\n"
+                "          'bar' as tbl,\n"
+                "          crsql_pack_columns(t1.\"x\") as pks,\n"
+                "          t1.__crsql_col_name as cid,\n"
+                "          t1.__crsql_col_version as col_vrsn,\n"
+                "          t1.__crsql_db_version as db_vrsn,\n"
+                "          t3.site_id as site_id,\n"
+                "          t1._rowid_,\n"
+                "          t1.__crsql_seq as seq,\n"
+                "          COALESCE(t2.__crsql_col_version, 1) as cl\n"
+                "      FROM \"bar__crsql_clock\" AS t1 LEFT JOIN "
+                "\"bar__crsql_clock\" AS t2 ON\n"
+                "      t1.\"x\" = t2.\"x\" AND t2.__crsql_col_name = '-1' LEFT "
+                "JOIN crsql_site_id as t3 ON t1.__crsql_site_id = t3.ordinal) "
+                "WHERE site_id IS ? AND db_vrsn > ?") == 0);
   sqlite3_free(query);
 
   printf("\t\e[0;32mSuccess\e[0m\n");
@@ -127,8 +125,8 @@ static void testRowPatchDataQuery() {
   // TC1: single pk table, 1 col change
   const char *cid = "b";
   char *pks = "1";
-  char *q = crsql_rowPatchDataQuery(db, tblInfo, cid, pks);
-  assert(strcmp(q, "SELECT quote(\"b\") FROM \"foo\" WHERE \"a\" = 1") == 0);
+  char *q = crsql_row_patch_data_query(tblInfo, cid);
+  assert(strcmp(q, "SELECT \"b\" FROM \"foo\" WHERE \"a\" IS ?") == 0);
   sqlite3_free(q);
 
   printf("\t\e[0;32mSuccess\e[0m\n");
@@ -140,7 +138,6 @@ static void testRowPatchDataQuery() {
 
 void crsqlChangesVtabReadTestSuite() {
   printf("\e[47m\e[1;30mSuite: crsql_changesVtabRead\e[0m\n");
-  testChangesQueryForTable();
   testChangesUnionQuery();
   testRowPatchDataQuery();
 }

libsql-sqlite3/ext/crr/src/changes-vtab-rowid.test.c

@@ -0,0 +1,114 @@
+/**
+ * Test that:
+ * 1. The rowid we return for a row on insert matches the rowid we get for it on
+ * read
+ * 2. That we can query the vtab by rowid??
+ * 3. The returned rowid matches the rowid used in a point query by rowid
+ * 4.
+ */
+
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "crsqlite.h"
+
+int crsql_close(sqlite3 *db);
+
+// static void testRowidForInsert() {
+//   printf("RowidForInsert\n");
+
+//   sqlite3 *db;
+//   int rc;
+//   rc = sqlite3_open(":memory:", &db);
+
+//   rc = sqlite3_exec(db, "CREATE TABLE foo (a primary key, b);", 0, 0, 0);
+//   rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0);
+//   assert(rc == SQLITE_OK);
+
+//   char *zSql =
+//       "INSERT INTO crsql_changes ([table], pk, cid, val, col_version, "
+//       "db_version) "
+//       "VALUES "
+//       "('foo', '1', 'b', '1', 1, 1) RETURNING _rowid_;";
+//   sqlite3_stmt *pStmt;
+//   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+//   assert(rc == SQLITE_OK);
+//   assert(sqlite3_step(pStmt) == SQLITE_ROW);
+//   printf("rowid: %d\n", sqlite3_column_int64(pStmt, 0));
+//   assert(sqlite3_column_int64(pStmt, 0) == 1);
+//   sqlite3_finalize(pStmt);
+
+//   // TODO: make extra crr tables and check their slab allotments and returned
+//   // rowids
+
+//   crsql_close(db);
+//   printf("\t\e[0;32mSuccess\e[0m\n");
+// }
+
+static void testRowidsForReads() {
+  printf("RowidForReads\n");
+
+  sqlite3 *db;
+  int rc;
+  rc = sqlite3_open(":memory:", &db);
+
+  rc = sqlite3_exec(db, "CREATE TABLE foo (a primary key, b);", 0, 0, 0);
+  rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+  sqlite3_exec(db, "INSERT INTO foo VALUES (1,2);", 0, 0, 0);
+  sqlite3_exec(db, "INSERT INTO foo VALUES (2,3);", 0, 0, 0);
+
+  char *zSql = "SELECT _rowid_ FROM crsql_changes";
+  sqlite3_stmt *pStmt;
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  assert(rc == SQLITE_OK);
+  assert(sqlite3_step(pStmt) == SQLITE_ROW);
+  assert(sqlite3_column_int64(pStmt, 0) == 1);
+  assert(sqlite3_step(pStmt) == SQLITE_ROW);
+  assert(sqlite3_column_int64(pStmt, 0) == 2);
+  sqlite3_finalize(pStmt);
+
+  rc = sqlite3_exec(db, "CREATE TABLE bar (a primary key, b)", 0, 0, 0);
+  rc += sqlite3_exec(db, "SELECT crsql_as_crr('bar');", 0, 0, 0);
+  rc += sqlite3_exec(db, "INSERT INTO bar VALUES (1,2);", 0, 0, 0);
+  rc += sqlite3_exec(db, "INSERT INTO bar VALUES (2,3);", 0, 0, 0);
+
+  rc += sqlite3_exec(db, "CREATE TABLE baz (a primary key, b)", 0, 0, 0);
+  rc += sqlite3_exec(db, "SELECT crsql_as_crr('baz');", 0, 0, 0);
+  rc += sqlite3_exec(db, "INSERT INTO baz VALUES (1,2);", 0, 0, 0);
+  rc += sqlite3_exec(db, "INSERT INTO baz VALUES (2,3);", 0, 0, 0);
+
+  assert(rc == SQLITE_OK);
+
+  sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int64(pStmt, 0) == 1);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int64(pStmt, 0) == 2);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int64(pStmt, 0) == 1 * ROWID_SLAB_SIZE + 1);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int64(pStmt, 0) == 1 * ROWID_SLAB_SIZE + 2);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int64(pStmt, 0) == 2 * ROWID_SLAB_SIZE + 1);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int64(pStmt, 0) == 2 * ROWID_SLAB_SIZE + 2);
+  sqlite3_finalize(pStmt);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+// static void testInsertRowidMatchesReadRowid() {
+//   printf("RowidForInsertMatchesRowidForRead\n");
+//   printf("\t\e[0;32mSuccess\e[0m\n");
+// }
+
+void crsqlChangesVtabRowidTestSuite() {
+  printf("\e[47m\e[1;30mSuite: crsql_changesVtabRowid\e[0m\n");
+  // testRowidForInsert();
+  testRowidsForReads();
+  // testInsertRowidMatchesReadRowid();
+}

libsql-sqlite3/ext/crr/src/changes-vtab-write.c

@@ -1,421 +0,0 @@
-#include "changes-vtab-write.h"
-
-#include <string.h>
-
-#include "changes-vtab-common.h"
-#include "changes-vtab.h"
-#include "consts.h"
-#include "crsqlite.h"
-#include "ext-data.h"
-#include "tableinfo.h"
-#include "util.h"
-
-/**
- *
- */
-int crsql_didCidWin(sqlite3 *db, const unsigned char *localSiteId,
-                    const char *insertTbl, const char *pkWhereList,
-                    const char *colName, const char *sanitizedInsertVal,
-                    sqlite3_int64 colVersion, char **errmsg) {
-  char *zSql = 0;
-
-  zSql = sqlite3_mprintf(
-      "SELECT __crsql_col_version FROM \"%s__crsql_clock\" WHERE %s AND %Q = "
-      "__crsql_col_name",
-      insertTbl, pkWhereList, colName);
-
-  // run zSql
-  sqlite3_stmt *pStmt = 0;
-  int rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    sqlite3_finalize(pStmt);
-    *errmsg =
-        sqlite3_mprintf("Failed preparing stmt to select local column version");
-    return -1;
-  }
-
-  rc = sqlite3_step(pStmt);
-  if (rc == SQLITE_DONE) {
-    sqlite3_finalize(pStmt);
-    // no rows returned
-    // we of course win if there's nothing there.
-    return 1;
-  }
-
-  if (rc != SQLITE_ROW) {
-    sqlite3_finalize(pStmt);
-    *errmsg = sqlite3_mprintf(
-        "Bad return code (%d) when selecting local column version", rc);
-    return -1;
-  }
-
-  sqlite3_int64 localVersion = sqlite3_column_int64(pStmt, 0);
-  sqlite3_finalize(pStmt);
-
-  if (colVersion > localVersion) {
-    return 1;
-  } else if (colVersion < localVersion) {
-    return 0;
-  }
-
-  // else -- versions are equal
-  // - pull curr value
-  // - compare for tie break
-  // TODO: pull bytes and memcmp instead of strcmp?
-  zSql = sqlite3_mprintf("SELECT quote(\"%w\") FROM \"%w\" WHERE %s", colName,
-                         insertTbl, pkWhereList);
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    sqlite3_finalize(pStmt);
-    *errmsg = sqlite3_mprintf(
-        "could not prepare statement to find row to merge with. %s", insertTbl);
-    return -1;
-  }
-
-  rc = sqlite3_step(pStmt);
-  if (rc != SQLITE_ROW) {
-    *errmsg = sqlite3_mprintf("could not find row to merge with for tbl %s",
-                              insertTbl);
-    sqlite3_finalize(pStmt);
-    return -1;
-  }
-
-  const char *localValue = (const char *)sqlite3_column_text(pStmt, 0);
-  int ret = strcmp(sanitizedInsertVal, localValue);
-  sqlite3_finalize(pStmt);
-
-  return ret > 0;
-}
-
-#define DELETED_LOCALLY -1
-int crsql_checkForLocalDelete(sqlite3 *db, const char *tblName,
-                              char *pkWhereList) {
-  char *zSql = sqlite3_mprintf(
-      "SELECT count(*) FROM \"%s__crsql_clock\" WHERE %s AND "
-      "__crsql_col_name "
-      "= %Q",
-      tblName, pkWhereList, DELETE_CID_SENTINEL);
-  sqlite3_stmt *pStmt;
-  int rc = sqlite3_prepare(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    sqlite3_finalize(pStmt);
-    return rc;
-  }
-
-  rc = sqlite3_step(pStmt);
-  if (rc != SQLITE_ROW) {
-    sqlite3_finalize(pStmt);
-    return SQLITE_ERROR;
-  }
-
-  int count = sqlite3_column_int(pStmt, 0);
-  sqlite3_finalize(pStmt);
-  if (count == 1) {
-    return DELETED_LOCALLY;
-  }
-
-  return SQLITE_OK;
-}
-
-int crsql_setWinnerClock(sqlite3 *db, crsql_TableInfo *tblInfo,
-                         const char *pkIdentifierList, const char *pkValsStr,
-                         const char *insertColName, sqlite3_int64 insertColVrsn,
-                         sqlite3_int64 insertDbVrsn, const void *insertSiteId,
-                         int insertSiteIdLen) {
-  int rc = SQLITE_OK;
-  char *zSql = sqlite3_mprintf(
-      "INSERT OR REPLACE INTO \"%s__crsql_clock\" \
-      (%s, \"__crsql_col_name\", \"__crsql_col_version\", \"__crsql_db_version\", \"__crsql_site_id\")\
-      VALUES (\
-        %s,\
-        %Q,\
-        %lld,\
-        MAX(crsql_nextdbversion(), %lld),\
-        ?\
-      )",
-      tblInfo->tblName, pkIdentifierList, pkValsStr, insertColName,
-      insertColVrsn, insertDbVrsn);
-
-  sqlite3_stmt *pStmt = 0;
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    sqlite3_finalize(pStmt);
-    return rc;
-  }
-
-  if (insertSiteId == 0) {
-    sqlite3_bind_null(pStmt, 1);
-  } else {
-    sqlite3_bind_blob(pStmt, 1, insertSiteId, insertSiteIdLen,
-                      SQLITE_TRANSIENT);
-  }
-
-  rc = sqlite3_step(pStmt);
-  sqlite3_finalize(pStmt);
-
-  if (rc == SQLITE_DONE) {
-    return SQLITE_OK;
-  } else {
-    return SQLITE_ERROR;
-  }
-}
-
-int crsql_mergePkOnlyInsert(sqlite3 *db, crsql_TableInfo *tblInfo,
-                            const char *pkValsStr, const char *pkIdentifiers,
-                            sqlite3_int64 remoteColVersion,
-                            sqlite3_int64 remoteDbVersion,
-                            const void *remoteSiteId, int remoteSiteIdLen) {
-  char *zSql = sqlite3_mprintf("INSERT OR IGNORE INTO \"%s\" (%s) VALUES (%s)",
-                               tblInfo->tblName, pkIdentifiers, pkValsStr);
-  int rc = sqlite3_exec(db, SET_SYNC_BIT, 0, 0, 0);
-  if (rc != SQLITE_OK) {
-    sqlite3_free(zSql);
-    return rc;
-  }
-
-  rc = sqlite3_exec(db, zSql, 0, 0, 0);
-  sqlite3_free(zSql);
-  sqlite3_exec(db, CLEAR_SYNC_BIT, 0, 0, 0);
-  if (rc != SQLITE_OK) {
-    return rc;
-  }
-
-  // TODO: if insert was ignored, no reason to change clock
-  return crsql_setWinnerClock(db, tblInfo, pkIdentifiers, pkValsStr,
-                              PKS_ONLY_CID_SENTINEL, remoteColVersion,
-                              remoteDbVersion, remoteSiteId, remoteSiteIdLen);
-}
-
-int crsql_mergeDelete(sqlite3 *db, crsql_TableInfo *tblInfo,
-                      const char *pkWhereList, const char *pkValsStr,
-                      const char *pkIdentifiers, sqlite3_int64 remoteColVersion,
-                      sqlite3_int64 remoteDbVersion, const void *remoteSiteId,
-                      int remoteSiteIdLen) {
-  char *zSql = sqlite3_mprintf("DELETE FROM \"%s\" WHERE %s", tblInfo->tblName,
-                               pkWhereList);
-  int rc = sqlite3_exec(db, SET_SYNC_BIT, 0, 0, 0);
-  if (rc != SQLITE_OK) {
-    sqlite3_free(zSql);
-    return rc;
-  }
-
-  rc = sqlite3_exec(db, zSql, 0, 0, 0);
-  sqlite3_free(zSql);
-  sqlite3_exec(db, CLEAR_SYNC_BIT, 0, 0, 0);
-  if (rc != SQLITE_OK) {
-    return rc;
-  }
-
-  return crsql_setWinnerClock(db, tblInfo, pkIdentifiers, pkValsStr,
-                              DELETE_CID_SENTINEL, remoteColVersion,
-                              remoteDbVersion, remoteSiteId, remoteSiteIdLen);
-}
-
-int crsql_mergeInsert(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,
-                      sqlite3_int64 *pRowid, char **errmsg) {
-  // he argv[1] parameter is the rowid of a new row to be inserted into the
-  // virtual table. If argv[1] is an SQL NULL, then the implementation must
-  // choose a rowid for the newly inserted row
-  int rc = 0;
-  crsql_Changes_vtab *pTab = (crsql_Changes_vtab *)pVTab;
-  sqlite3 *db = pTab->db;
-  char *zSql = 0;
-
-  rc = crsql_ensureTableInfosAreUpToDate(db, pTab->pExtData, errmsg);
-
-  if (rc != SQLITE_OK) {
-    *errmsg = sqlite3_mprintf("Failed to update crr table information");
-    return rc;
-  }
-
-  // column values exist in argv[2] and following.
-  const int insertTblLen =
-      sqlite3_value_bytes(argv[2 + CHANGES_SINCE_VTAB_TBL]);
-  if (insertTblLen > MAX_TBL_NAME_LEN) {
-    *errmsg = sqlite3_mprintf("crsql - table name exceeded max length");
-    return SQLITE_ERROR;
-  }
-  // safe given we only use this if it exactly matches a table name
-  // from tblInfo
-  const unsigned char *insertTbl =
-      sqlite3_value_text(argv[2 + CHANGES_SINCE_VTAB_TBL]);
-  // `splitQuoteConcat` will validate these
-  const unsigned char *insertPks =
-      sqlite3_value_text(argv[2 + CHANGES_SINCE_VTAB_PK]);
-
-  int inesrtColNameLen = sqlite3_value_bytes(argv[2 + CHANGES_SINCE_VTAB_CID]);
-  if (inesrtColNameLen > MAX_TBL_NAME_LEN) {
-    *errmsg = sqlite3_mprintf("column name exceeded max length");
-    return SQLITE_ERROR;
-  }
-  const char *insertColName =
-      (const char *)sqlite3_value_text(argv[2 + CHANGES_SINCE_VTAB_CID]);
-
-  // `splitQuoteConcat` will validate these -- even tho 1 val should do
-  // splitquoteconcat for the validation
-  const unsigned char *insertVal =
-      sqlite3_value_text(argv[2 + CHANGES_SINCE_VTAB_CVAL]);
-  sqlite3_int64 insertColVrsn =
-      sqlite3_value_int64(argv[2 + CHANGES_SINCE_VTAB_COL_VRSN]);
-  sqlite3_int64 insertDbVrsn =
-      sqlite3_value_int64(argv[2 + CHANGES_SINCE_VTAB_DB_VRSN]);
-
-  int insertSiteIdLen =
-      sqlite3_value_bytes(argv[2 + CHANGES_SINCE_VTAB_SITE_ID]);
-  if (insertSiteIdLen > SITE_ID_LEN) {
-    *errmsg = sqlite3_mprintf("crsql - site id exceeded max length");
-    return SQLITE_ERROR;
-  }
-  // safe given we only use siteid via `bind`
-  const void *insertSiteId =
-      sqlite3_value_blob(argv[2 + CHANGES_SINCE_VTAB_SITE_ID]);
-
-  crsql_TableInfo *tblInfo = crsql_findTableInfo(pTab->pExtData->zpTableInfos,
-                                                 pTab->pExtData->tableInfosLen,
-                                                 (const char *)insertTbl);
-
-  if (tblInfo == 0) {
-    *errmsg = sqlite3_mprintf(
-        "crsql - could not find the schema information for table %s",
-        insertTbl);
-    return SQLITE_ERROR;
-  }
-
-  int isDelete = strcmp(DELETE_CID_SENTINEL, insertColName) == 0;
-  int isPkOnly = strcmp(PKS_ONLY_CID_SENTINEL, insertColName) == 0;
-
-  char *pkWhereList = crsql_extractWhereList(tblInfo->pks, tblInfo->pksLen,
-                                             (const char *)insertPks);
-  if (pkWhereList == 0) {
-    *errmsg =
-        sqlite3_mprintf("crsql - failed decoding primary keys for insert");
-    return SQLITE_ERROR;
-  }
-
-  rc = crsql_checkForLocalDelete(db, tblInfo->tblName, pkWhereList);
-  if (rc == DELETED_LOCALLY) {
-    rc = SQLITE_OK;
-    // delete wins. we're all done.
-    sqlite3_free(pkWhereList);
-    return rc;
-  }
-
-  // This happens if the state is a delete
-  // We must `checkForLocalDelete` prior to merging a delete (happens above).
-  // mergeDelete assumes we've already checked for a local delete.
-  char *pkValsStr =
-      crsql_quoteConcatedValuesAsList((const char *)insertPks, tblInfo->pksLen);
-  if (pkValsStr == 0) {
-    sqlite3_free(pkWhereList);
-    *errmsg = sqlite3_mprintf("Failed sanitizing pk values");
-    return SQLITE_ERROR;
-  }
-
-  char *pkIdentifierList =
-      crsql_asIdentifierList(tblInfo->pks, tblInfo->pksLen, 0);
-  if (isDelete) {
-    rc = crsql_mergeDelete(db, tblInfo, pkWhereList, pkValsStr,
-                           pkIdentifierList, insertColVrsn, insertDbVrsn,
-                           insertSiteId, insertSiteIdLen);
-
-    sqlite3_free(pkWhereList);
-    sqlite3_free(pkValsStr);
-    sqlite3_free(pkIdentifierList);
-    return rc;
-  }
-
-  if (isPkOnly ||
-      !crsql_columnExists(insertColName, tblInfo->nonPks, tblInfo->nonPksLen)) {
-    rc = crsql_mergePkOnlyInsert(db, tblInfo, pkValsStr, pkIdentifierList,
-                                 insertColVrsn, insertDbVrsn, insertSiteId,
-                                 insertSiteIdLen);
-    sqlite3_free(pkWhereList);
-    sqlite3_free(pkValsStr);
-    sqlite3_free(pkIdentifierList);
-    return rc;
-  }
-
-  char **sanitizedInsertVal =
-      crsql_splitQuoteConcat((const char *)insertVal, 1);
-
-  if (sanitizedInsertVal == 0) {
-    sqlite3_free(pkValsStr);
-    sqlite3_free(pkIdentifierList);
-    *errmsg = sqlite3_mprintf("Failed sanitizing value for changeset (%s)",
-                              insertVal);
-    return SQLITE_ERROR;
-  }
-
-  int doesCidWin = crsql_didCidWin(
-      db, pTab->pExtData->siteId, tblInfo->tblName, pkWhereList, insertColName,
-      sanitizedInsertVal[0], insertColVrsn, errmsg);
-  sqlite3_free(pkWhereList);
-  if (doesCidWin == -1 || doesCidWin == 0) {
-    sqlite3_free(pkValsStr);
-    sqlite3_free(pkIdentifierList);
-    sqlite3_free(sanitizedInsertVal[0]);
-    sqlite3_free(sanitizedInsertVal);
-    // doesCidWin == 0? compared against our clocks, nothing wins. OK and
-    // Done.
-    if (doesCidWin == -1 && *errmsg == 0) {
-      *errmsg = sqlite3_mprintf("Failed computing cid win");
-    }
-    return doesCidWin == 0 ? SQLITE_OK : SQLITE_ERROR;
-  }
-
-  zSql = sqlite3_mprintf(
-      "INSERT INTO \"%w\" (%s, \"%w\")\
-      VALUES (%s, %s)\
-      ON CONFLICT DO UPDATE\
-      SET \"%w\" = %s",
-      tblInfo->tblName, pkIdentifierList, insertColName, pkValsStr,
-      sanitizedInsertVal[0], insertColName, sanitizedInsertVal[0]);
-
-  sqlite3_free(sanitizedInsertVal[0]);
-  sqlite3_free(sanitizedInsertVal);
-
-  rc = sqlite3_exec(db, SET_SYNC_BIT, 0, 0, errmsg);
-  if (rc != SQLITE_OK) {
-    sqlite3_free(pkValsStr);
-    sqlite3_free(pkIdentifierList);
-    sqlite3_exec(db, CLEAR_SYNC_BIT, 0, 0, 0);
-    *errmsg = sqlite3_mprintf("Failed setting sync bit");
-    return rc;
-  }
-
-  rc = sqlite3_exec(db, zSql, 0, 0, errmsg);
-  sqlite3_free(zSql);
-  sqlite3_exec(db, CLEAR_SYNC_BIT, 0, 0, 0);
-
-  if (rc != SQLITE_OK) {
-    sqlite3_free(pkValsStr);
-    sqlite3_free(pkIdentifierList);
-    *errmsg = sqlite3_mprintf("Failed inserting changeset");
-    return rc;
-  }
-
-  rc = crsql_setWinnerClock(db, tblInfo, pkIdentifierList, pkValsStr,
-                            insertColName, insertColVrsn, insertDbVrsn,
-                            insertSiteId, insertSiteIdLen);
-  sqlite3_free(pkIdentifierList);
-  sqlite3_free(pkValsStr);
-
-  if (rc != SQLITE_OK) {
-    *errmsg = sqlite3_mprintf("Failed updating winner clock");
-  }
-
-  // TODO: ... this isn't really guaranteed to be unique across
-  // the table.
-  // Is it fine if we prevent anyone from using `rowid` on a vtab?
-  // or must we convert to `without rowid`?
-  *pRowid = insertDbVrsn;
-  return rc;
-}

libsql-sqlite3/ext/crr/src/changes-vtab-write.h

@@ -1,17 +0,0 @@
-#ifndef CHANGES_VTAB_WRITE_H
-#define CHANGES_VTAB_WRITE_H
-
-#include "sqlite3ext.h"
-SQLITE_EXTENSION_INIT3
-
-#include "tableinfo.h"
-
-int crsql_mergeInsert(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,
-                      sqlite3_int64 *pRowid, char **errmsg);
-
-int crsql_didCidWin(sqlite3 *db, const unsigned char *localSiteId,
-                    const char *insertTbl, const char *pkWhereList,
-                    const char *colName, const char *sanitizedInsertVal,
-                    sqlite3_int64 dbVersion, char **errmsg);
-
-#endif

libsql-sqlite3/ext/crr/src/changes-vtab-write.test.c

@@ -1,57 +0,0 @@
-#include "changes-vtab-write.h"
-
-#include <assert.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "consts.h"
-#include "crsqlite.h"
-
-int crsql_close(sqlite3 *db);
-
-// static void memTestMergeInsert()
-// {
-//   // test delete case
-//   // test nothing to merge case
-//   // test normal merge
-//   // test error / early returns
-// }
-
-// static void testMergeInsert()
-// {
-// }
-
-// static void testChangesTabConflictSets()
-// {
-// }
-
-// static void testDidCidWin()
-// {
-//   printf("AllChangedCids\n");
-
-//   int rc = SQLITE_OK;
-//   sqlite3 *db;
-//   rc = sqlite3_open(":memory:", &db);
-//   char *err = 0;
-
-//   // test
-//   // crsql_allChangedCids(
-//   //   db,
-//   //   "",
-//   //   "",
-//   //   "",
-
-//   // );
-
-//   printf("\t\e[0;32mSuccess\e[0m\n");
-// }
-
-void crsqlChangesVtabWriteTestSuite() {
-  printf("\e[47m\e[1;30mSuite: crsql_changesVtabWrite\e[0m\n");
-
-  // TODO: most vtab write cases are covered in `crsqlite.test.c`
-  // we should, however, create tests that are narrower in scope here.
-
-  // testDidCidWin();
-}

libsql-sqlite3/ext/crr/src/changes-vtab.c

@@ -4,14 +4,14 @@
 #include <stdint.h>
 #include <string.h>
 
-#include "changes-vtab-common.h"
-#include "changes-vtab-read.h"
-#include "changes-vtab-write.h"
 #include "consts.h"
 #include "crsqlite.h"
 #include "ext-data.h"
+#include "rust.h"
 #include "util.h"
 
+int crsql_changes_next(sqlite3_vtab_cursor *cur);
+
 /**
  * Created when the virtual table is initialized.
  * This happens when the vtab is first used in a given connection.
@@ -26,12 +26,10 @@ static int changesConnect(sqlite3 *db, void *pAux, int argc,
 
   rc = sqlite3_declare_vtab(
       db,
-      // If we go without rowid we need to concat `table || !'! pk` to be the
-      // primary key as xUpdate requires a single column to be the primary key
-      // if we use without rowid.
-      "CREATE TABLE x([table] TEXT NOT NULL, [pk] TEXT NOT NULL, [cid] TEXT "
-      "NOT NULL, [val], [col_version] INTEGER NOT NULL, [db_version] INTEGER "
-      "NOT NULL, [site_id] BLOB)");
+      "CREATE TABLE x([table] TEXT NOT NULL, [pk] BLOB NOT NULL, [cid] TEXT "
+      "NOT NULL, [val] ANY, [col_version] INTEGER NOT NULL, [db_version] "
+      "INTEGER NOT NULL, [site_id] BLOB, [cl] INTEGER NOT NULL, [seq] "
+      "INTEGER NOT NULL)");
   if (rc != SQLITE_OK) {
     *pzErr = sqlite3_mprintf("Could not define the table");
     return rc;
@@ -92,7 +90,10 @@ static int changesCrsrFinalize(crsql_Changes_cursor *crsr) {
   int rc = SQLITE_OK;
   rc += sqlite3_finalize(crsr->pChangesStmt);
   crsr->pChangesStmt = 0;
-  rc += sqlite3_finalize(crsr->pRowStmt);
+  if (crsr->pRowStmt != 0) {
+    rc += sqlite3_clear_bindings(crsr->pRowStmt);
+    rc += sqlite3_reset(crsr->pRowStmt);
+  }
   crsr->pRowStmt = 0;
 
   crsr->dbVersion = MIN_POSSIBLE_DB_VERSION;
@@ -120,212 +121,6 @@ static int changesClose(sqlite3_vtab_cursor *cur) {
   return SQLITE_OK;
 }
 
-/**
- * version is guaranteed to be unique (it increases on every write)
- * thus we use it for the rowid.
- *
- * Depending on how sqlite treats calls to `xUpdate` we may
- * shift to a `without rowid` table and use `table + pk` concated
- * as the primary key. xUpdate requires a single column to act
- * as the primary key, hence the concatenation that'd be required.
- */
-static int changesRowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid) {
-  crsql_Changes_cursor *pCur = (crsql_Changes_cursor *)cur;
-  *pRowid = pCur->dbVersion;
-  return SQLITE_OK;
-}
-
-/**
- * Returns true if the cursor has been moved off the last row.
- * `pChangesStmt` is finalized and set to null when this is the case as we
- * finalize `pChangeStmt` in `changesNext` when it returns `SQLITE_DONE`
- */
-static int changesEof(sqlite3_vtab_cursor *cur) {
-  crsql_Changes_cursor *pCur = (crsql_Changes_cursor *)cur;
-  return pCur->pChangesStmt == 0;
-}
-
-// char **crsql_extractValList() {
-
-// }
-
-/**
- * Advances our Changes_cursor to its next row of output.
- */
-static int changesNext(sqlite3_vtab_cursor *cur) {
-  crsql_Changes_cursor *pCur = (crsql_Changes_cursor *)cur;
-  sqlite3_vtab *pTabBase = (sqlite3_vtab *)(pCur->pTab);
-  int rc = SQLITE_OK;
-
-  if (pCur->pChangesStmt == 0) {
-    pTabBase->zErrMsg = sqlite3_mprintf(
-        "crsql internal error: in an unexpected state. pChangesStmt is "
-        "null.");
-    return SQLITE_ERROR;
-  }
-
-  if (pCur->pRowStmt != 0) {
-    // Finalize the prior row result
-    // before getting the next row.
-    // Do not re-use the statement since we could be entering
-    // a new table.
-    // An optimization would be to keep (rewind) it if we're processing the
-    // same table for many rows.
-    sqlite3_finalize(pCur->pRowStmt);
-    pCur->pRowStmt = 0;
-  }
-
-  // step to next
-  // if no row, tear down (finalize) statements
-  // set statements to null
-  rc = sqlite3_step(pCur->pChangesStmt);
-  if (rc != SQLITE_ROW) {
-    // tear down since we're done
-    return changesCrsrFinalize(pCur);
-  }
-
-  const char *tbl = (const char *)sqlite3_column_text(pCur->pChangesStmt, TBL);
-  const char *pks = (const char *)sqlite3_column_text(pCur->pChangesStmt, PKS);
-  const char *cid = (const char *)sqlite3_column_text(pCur->pChangesStmt, CID);
-  sqlite3_int64 dbVersion = sqlite3_column_int64(pCur->pChangesStmt, DB_VRSN);
-  pCur->dbVersion = dbVersion;
-
-  crsql_TableInfo *tblInfo =
-      crsql_findTableInfo(pCur->pTab->pExtData->zpTableInfos,
-                          pCur->pTab->pExtData->tableInfosLen, tbl);
-  if (tblInfo == 0) {
-    pTabBase->zErrMsg = sqlite3_mprintf(
-        "crsql internal error. Could not find schema for table %s", tbl);
-    changesCrsrFinalize(pCur);
-    return SQLITE_ERROR;
-  }
-
-  if (tblInfo->pksLen == 0) {
-    crsql_freeTableInfo(tblInfo);
-    pTabBase->zErrMsg = sqlite3_mprintf(
-        "crr table %s is missing primary key columns", tblInfo->tblName);
-    return SQLITE_ERROR;
-  }
-
-  if (strcmp(DELETE_CID_SENTINEL, cid) == 0) {
-    pCur->rowType = ROW_TYPE_DELETE;
-    return SQLITE_OK;
-  } else if (strcmp(PKS_ONLY_CID_SENTINEL, cid) == 0) {
-    pCur->rowType = ROW_TYPE_PKONLY;
-    return SQLITE_OK;
-  } else {
-    pCur->rowType = ROW_TYPE_UPDATE;
-  }
-
-  char *zSql = crsql_rowPatchDataQuery(pCur->pTab->db, tblInfo, cid, pks);
-  if (zSql == 0) {
-    pTabBase->zErrMsg = sqlite3_mprintf(
-        "crsql internal error generationg raw data fetch query for table "
-        "%s",
-        tbl);
-    return SQLITE_ERROR;
-  }
-
-  sqlite3_stmt *pRowStmt;
-  rc = sqlite3_prepare_v2(pCur->pTab->db, zSql, -1, &pRowStmt, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    pTabBase->zErrMsg = sqlite3_mprintf(
-        "crsql internal error preparing row data fetch statement");
-    sqlite3_finalize(pRowStmt);
-    return rc;
-  }
-
-  rc = sqlite3_step(pRowStmt);
-  if (rc != SQLITE_ROW) {
-    sqlite3_finalize(pRowStmt);
-    // getting 0 rows for something we have clock entries for is not an
-    // error it could just be the case that the thing was deleted so we have
-    // nothing to retrieve to fill in values for do we re-write cids in this
-    // case?
-    if (rc == SQLITE_DONE) {
-      return SQLITE_OK;
-    }
-    pTabBase->zErrMsg =
-        sqlite3_mprintf("crsql internal error fetching row data");
-    return SQLITE_ERROR;
-  } else {
-    rc = SQLITE_OK;
-  }
-
-  pCur->pRowStmt = pRowStmt;
-
-  return rc;
-}
-
-/**
- * Returns volums for the row at which
- * the cursor currently resides.
- */
-static int changesColumn(
-    sqlite3_vtab_cursor *cur, /* The cursor */
-    sqlite3_context *ctx,     /* First argument to sqlite3_result_...() */
-    int i                     /* Which column to return */
-) {
-  crsql_Changes_cursor *pCur = (crsql_Changes_cursor *)cur;
-  switch (i) {
-      // we clean up the cursor on moving to the next result
-      // so no need to tell sqlite to free these values.
-    case CHANGES_SINCE_VTAB_TBL:
-      sqlite3_result_value(ctx, sqlite3_column_value(pCur->pChangesStmt, TBL));
-      break;
-    case CHANGES_SINCE_VTAB_PK:
-      sqlite3_result_value(ctx, sqlite3_column_value(pCur->pChangesStmt, PKS));
-      break;
-    case CHANGES_SINCE_VTAB_CVAL:
-      // pRowStmt is null if the event was a delete. i.e., there is no row
-      // data.
-      // TODO: there's an edge case here where we can end up replicating a
-      // bunch of nulls for a row that is deleted but has prior events
-      // proceeding the delete. So on row delete we should, in our delete
-      // trigger, go drop all state records for the row except the delete
-      // event. "all" is actually quite small given we only keep max 1
-      // record per col in a row. so this drop is feasible on delete.
-      if (pCur->pRowStmt == 0) {
-        sqlite3_result_null(ctx);
-      } else {
-        sqlite3_result_value(ctx, sqlite3_column_value(pCur->pRowStmt, 0));
-      }
-      break;
-    case CHANGES_SINCE_VTAB_CID:
-      if (pCur->rowType == ROW_TYPE_PKONLY) {
-        sqlite3_result_text(ctx, PKS_ONLY_CID_SENTINEL, -1, SQLITE_STATIC);
-      } else if (pCur->rowType == ROW_TYPE_DELETE || pCur->pRowStmt == 0) {
-        sqlite3_result_text(ctx, DELETE_CID_SENTINEL, -1, SQLITE_STATIC);
-      } else {
-        sqlite3_result_value(ctx,
-                             sqlite3_column_value(pCur->pChangesStmt, CID));
-      }
-      break;
-    case CHANGES_SINCE_VTAB_COL_VRSN:
-      sqlite3_result_value(ctx,
-                           sqlite3_column_value(pCur->pChangesStmt, COL_VRSN));
-      break;
-    case CHANGES_SINCE_VTAB_DB_VRSN:
-      sqlite3_result_int64(ctx, pCur->dbVersion);
-      break;
-    case CHANGES_SINCE_VTAB_SITE_ID:
-      if (sqlite3_column_type(pCur->pChangesStmt, SITE_ID) == SQLITE_NULL) {
-        sqlite3_result_blob(ctx, pCur->pTab->pExtData->siteId, SITE_ID_LEN,
-                            SQLITE_STATIC);
-      } else {
-        sqlite3_result_value(ctx,
-                             sqlite3_column_value(pCur->pChangesStmt, SITE_ID));
-      }
-      break;
-    default:
-      return SQLITE_ERROR;
-  }
-  // sqlite3_result_value(ctx, sqlite3_column_value(pCur->pRowStmt, i));
-  return SQLITE_OK;
-}
-
 /**
  * Invoked to kick off the pulling of rows from the virtual table.
  * Provides the constraints with which the vtab can work with
@@ -333,91 +128,8 @@ static int changesColumn(
  *
  * Provided constraints are filled in by the changesBestIndex method.
  */
-static int changesFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
-                         const char *idxStr, int argc, sqlite3_value **argv) {
-  int rc = SQLITE_OK;
-  crsql_Changes_cursor *pCrsr = (crsql_Changes_cursor *)pVtabCursor;
-  crsql_Changes_vtab *pTab = pCrsr->pTab;
-  sqlite3_vtab *pTabBase = (sqlite3_vtab *)pTab;
-  sqlite3 *db = pTab->db;
-
-  // This should never happen. pChangesStmt should be finalized
-  // before filter is ever invoked.
-  if (pCrsr->pChangesStmt) {
-    sqlite3_finalize(pCrsr->pChangesStmt);
-    pCrsr->pChangesStmt = 0;
-  }
-
-  // construct and prepare our union for fetching changes
-  rc = crsql_ensureTableInfosAreUpToDate(db, pTab->pExtData,
-                                         &(pTabBase->zErrMsg));
-  if (rc != SQLITE_OK) {
-    return rc;
-  }
-
-  // nothing to fetch, no crrs exist.
-  if (pTab->pExtData->tableInfosLen == 0) {
-    return SQLITE_OK;
-  }
-
-  char *zSql = crsql_changesUnionQuery(pTab->pExtData->zpTableInfos,
-                                       pTab->pExtData->tableInfosLen, idxNum);
-
-  if (zSql == 0) {
-    pTabBase->zErrMsg = sqlite3_mprintf(
-        "crsql internal error generating the query to extract changes.");
-    return SQLITE_ERROR;
-  }
-
-  sqlite3_stmt *pStmt = 0;
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-  if (rc != SQLITE_OK) {
-    pTabBase->zErrMsg = sqlite3_mprintf(
-        "crsql internal error preparing the statement to extract changes.");
-    sqlite3_finalize(pStmt);
-    return rc;
-  }
-
-  // pull user provided params to `getChanges`
-  int i = 0;
-  sqlite3_int64 versionBound = MIN_POSSIBLE_DB_VERSION;
-  const char *requestorSiteId = "aa";
-  int siteIdType = SQLITE_BLOB;
-  int requestorSiteIdLen = 1;
-  if (idxNum & 2) {
-    versionBound = sqlite3_value_int64(argv[i]);
-    ++i;
-  }
-  if (idxNum & 4) {
-    siteIdType = sqlite3_value_type(argv[i]);
-    requestorSiteIdLen = sqlite3_value_bytes(argv[i]);
-    if (requestorSiteIdLen != 0) {
-      requestorSiteId = (const char *)sqlite3_value_blob(argv[i]);
-    } else {
-      requestorSiteIdLen = 1;
-    }
-    ++i;
-  }
-
-  // now bind the params.
-  // for each table info we need to bind 2 params:
-  // 1. the site id
-  // 2. the version
-  int j = 1;
-  for (i = 0; i < pTab->pExtData->tableInfosLen; ++i) {
-    if (siteIdType == SQLITE_NULL) {
-      sqlite3_bind_null(pStmt, j++);
-    } else {
-      sqlite3_bind_blob(pStmt, j++, requestorSiteId, requestorSiteIdLen,
-                        SQLITE_STATIC);
-    }
-    sqlite3_bind_int64(pStmt, j++, versionBound);
-  }
-
-  pCrsr->pChangesStmt = pStmt;
-  return changesNext((sqlite3_vtab_cursor *)pCrsr);
-}
+int crsql_changes_filter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
+                         const char *idxStr, int argc, sqlite3_value **argv);
 
 /*
 ** SQLite will invoke this method one or more times while planning a query
@@ -426,138 +138,44 @@ static int changesFilter(sqlite3_vtab_cursor *pVtabCursor, int idxNum,
 ** plan.
 ** TODO: should we support `where table` filters?
 */
-static int changesBestIndex(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo) {
-  int idxNum = 0;
-  int versionIdx = -1;
-  int requestorIdx = -1;
+int crsql_changes_best_index(sqlite3_vtab *tab, sqlite3_index_info *pIdxInfo);
 
-  for (int i = 0; i < pIdxInfo->nConstraint; i++) {
-    const struct sqlite3_index_constraint *pConstraint =
-        &pIdxInfo->aConstraint[i];
-    switch (pConstraint->iColumn) {
-      case CHANGES_SINCE_VTAB_DB_VRSN:
-        if (pConstraint->op != SQLITE_INDEX_CONSTRAINT_GT) {
-          tab->zErrMsg = sqlite3_mprintf(
-              "crsql_changes.version only supports the greater than "
-              "operator. "
-              "E.g., version > x");
-          return SQLITE_CONSTRAINT;
-        }
-        versionIdx = i;
-        idxNum |= 2;
-        break;
-      case CHANGES_SINCE_VTAB_SITE_ID:
-        if (pConstraint->op != SQLITE_INDEX_CONSTRAINT_NE &&
-            pConstraint->op != SQLITE_INDEX_CONSTRAINT_ISNOT &&
-            pConstraint->op != SQLITE_INDEX_CONSTRAINT_EQ &&
-            pConstraint->op != SQLITE_INDEX_CONSTRAINT_IS &&
-            pConstraint->op != SQLITE_INDEX_CONSTRAINT_ISNOTNULL &&
-            pConstraint->op != SQLITE_INDEX_CONSTRAINT_ISNULL) {
-          tab->zErrMsg = sqlite3_mprintf(
-              "crsql_changes.site_id only supports the IS, IS NOT, =, != "
-              "operators");
-          return SQLITE_CONSTRAINT;
-        }
-        requestorIdx = i;
-        pIdxInfo->aConstraintUsage[i].argvIndex = 2;
-        pIdxInfo->aConstraintUsage[i].omit = 1;
-        idxNum |= 4;
-
-        if (pConstraint->op == SQLITE_INDEX_CONSTRAINT_EQ ||
-            pConstraint->op == SQLITE_INDEX_CONSTRAINT_IS ||
-            pConstraint->op == SQLITE_INDEX_CONSTRAINT_ISNULL) {
-          idxNum |= 8;
-        }
-        break;
-    }
-  }
-
-  // both constraints are present
-  if ((idxNum & 6) == 6) {
-    pIdxInfo->estimatedCost = (double)1;
-    pIdxInfo->estimatedRows = 1;
-
-    pIdxInfo->aConstraintUsage[versionIdx].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[versionIdx].omit = 1;
-    pIdxInfo->aConstraintUsage[requestorIdx].argvIndex = 2;
-    pIdxInfo->aConstraintUsage[requestorIdx].omit = 1;
-  }
-  // only the version constraint is present
-  else if ((idxNum & 2) == 2) {
-    pIdxInfo->estimatedCost = (double)10;
-    pIdxInfo->estimatedRows = 10;
-
-    pIdxInfo->aConstraintUsage[versionIdx].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[versionIdx].omit = 1;
-  }
-  // only the requestor constraint is present
-  else if ((idxNum & 4) == 4) {
-    pIdxInfo->estimatedCost = (double)2147483647;
-    pIdxInfo->estimatedRows = 2147483647;
-
-    pIdxInfo->aConstraintUsage[requestorIdx].argvIndex = 1;
-    pIdxInfo->aConstraintUsage[requestorIdx].omit = 1;
-  }
-  // no constraints are present
-  else {
-    pIdxInfo->estimatedCost = (double)2147483647;
-    pIdxInfo->estimatedRows = 2147483647;
-  }
-
-  pIdxInfo->idxNum = idxNum;
-  return SQLITE_OK;
-}
-
-static int changesApply(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,
-                        sqlite3_int64 *pRowid) {
-  int argv0Type = sqlite3_value_type(argv[0]);
-  char *errmsg = 0;
-  int rc = SQLITE_OK;
-  // if (argc == 1 && argv[0] != 0)
-  // {
-  //   // delete statement
-  //   return crsql_mergeDelete();
-  // }
-  if (argc > 1 && argv0Type == SQLITE_NULL) {
-    // insert statement
-    // argv[1] is the rowid.. but why would it ever be filled for us?
-    rc = crsql_mergeInsert(pVTab, argc, argv, pRowid, &errmsg);
-    if (rc != SQLITE_OK) {
-      pVTab->zErrMsg = errmsg;
-    }
-    return rc;
-  } else {
-    pVTab->zErrMsg = sqlite3_mprintf(
-        "Only INSERT and SELECT statements are allowed against the crsql "
-        "changes table.");
-    return SQLITE_MISUSE;
-  }
-
-  return SQLITE_OK;
-}
+int crsql_changes_update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,
+                         sqlite3_int64 *pRowid);
+// If xBegin is not defined xCommit is not called.
+int crsql_changes_begin(sqlite3_vtab *pVTab);
+int crsql_changes_commit(sqlite3_vtab *pVTab);
+int crsql_changes_rowid(sqlite3_vtab_cursor *cur, sqlite_int64 *pRowid);
+int crsql_changes_column(
+    sqlite3_vtab_cursor *cur, /* The cursor */
+    sqlite3_context *ctx,     /* First argument to sqlite3_result_...() */
+    int i                     /* Which column to return */
+);
+int crsql_changes_eof(sqlite3_vtab_cursor *cur);
 
 sqlite3_module crsql_changesModule = {
     /* iVersion    */ 0,
     /* xCreate     */ 0,
     /* xConnect    */ changesConnect,
-    /* xBestIndex  */ changesBestIndex,
+    /* xBestIndex  */ crsql_changes_best_index,
     /* xDisconnect */ changesDisconnect,
     /* xDestroy    */ 0,
     /* xOpen       */ changesOpen,
     /* xClose      */ changesClose,
-    /* xFilter     */ changesFilter,
-    /* xNext       */ changesNext,
-    /* xEof        */ changesEof,
-    /* xColumn     */ changesColumn,
-    /* xRowid      */ changesRowid,
-    /* xUpdate     */ changesApply,
-    /* xBegin      */ 0,
+    /* xFilter     */ crsql_changes_filter,
+    /* xNext       */ crsql_changes_next,
+    /* xEof        */ crsql_changes_eof,
+    /* xColumn     */ crsql_changes_column,
+    /* xRowid      */ crsql_changes_rowid,
+    /* xUpdate     */ crsql_changes_update,
+    /* xBegin      */ crsql_changes_begin,
     /* xSync       */ 0,
-    /* xCommit     */ 0,
+    /* xCommit     */ crsql_changes_commit,
     /* xRollback   */ 0,
     /* xFindMethod */ 0,
     /* xRename     */ 0,
     /* xSavepoint  */ 0,
     /* xRelease    */ 0,
     /* xRollbackTo */ 0,
-    /* xShadowName */ 0};
+    /* xShadowName */ 0,
+    /* xPreparedSql */ 0};

libsql-sqlite3/ext/crr/src/changes-vtab.h

@@ -4,7 +4,7 @@
  *
  * To fetch a changeset:
  * ```sql
- * SELECT * FROM crsql_chages WHERE site_id != SITE_ID AND version > V
+ * SELECT * FROM crsql_chages WHERE site_id IS NOT SITE_ID AND version > V
  * ```
  *
  * The site id parameter is used to prevent a site from fetching its own
@@ -103,6 +103,9 @@ struct crsql_Changes_cursor {
 
   sqlite3_int64 dbVersion;
   int rowType;
+
+  sqlite3_int64 changesRowid;
+  int tblInfoIdx;
 };
 
 #endif

libsql-sqlite3/ext/crr/src/changes-vtab.test.c

@@ -25,12 +25,20 @@ static void testManyPkTable() {
   rc += sqlite3_exec(db, "INSERT INTO foo VALUES (4,5,6);", 0, 0, 0);
   assert(rc == SQLITE_OK);
 
-  rc += sqlite3_prepare_v2(db, "SELECT * FROM crsql_changes()", -1, &pStmt, 0);
+  rc += sqlite3_prepare_v2(db, "SELECT [table], quote(pk) FROM crsql_changes",
+                           -1, &pStmt, 0);
   assert(rc == SQLITE_OK);
 
   while (sqlite3_step(pStmt) == SQLITE_ROW) {
     const unsigned char *pk = sqlite3_column_text(pStmt, 1);
-    assert(strcmp("4|5", (char *)pk) == 0);
+    // pk: 4, 5
+    // X'0209040905'
+    // 02 -> columns
+    // 09 -> 1 byte integer
+    // 04 -> 4
+    // 09 -> 1 byte integer
+    // 05 -> 5
+    assert(strcmp("X'0209040905'", (char *)pk) == 0);
   }
 
   sqlite3_finalize(pStmt);
@@ -38,6 +46,92 @@ static void testManyPkTable() {
   printf("\t\e[0;32mSuccess\e[0m\n");
 }
 
+static void assertCount(sqlite3 *db, const char *sql, int expected) {
+  sqlite3_stmt *pStmt;
+  int rc = sqlite3_prepare_v2(db, sql, -1, &pStmt, 0);
+  assert(rc == SQLITE_OK);
+  assert(sqlite3_step(pStmt) == SQLITE_ROW);
+  printf("expected: %d, actual: %d\n", expected, sqlite3_column_int(pStmt, 0));
+  assert(sqlite3_column_int(pStmt, 0) == expected);
+  sqlite3_finalize(pStmt);
+}
+
+static void testFilters() {
+  printf("Filters\n");
+
+  sqlite3 *db;
+  int rc;
+  rc = sqlite3_open(":memory:", &db);
+
+  rc = sqlite3_exec(db, "CREATE TABLE foo (a primary key, b);", 0, 0, 0);
+  rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+  rc += sqlite3_exec(db, "INSERT INTO foo VALUES (1,2);", 0, 0, 0);
+  rc += sqlite3_exec(db, "INSERT INTO foo VALUES (2,3);", 0, 0, 0);
+  rc += sqlite3_exec(db, "INSERT INTO foo VALUES (3,4);", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  printf("no filters\n");
+  // 6 - 1 for each row creation, 1 for each b
+  assertCount(db, "SELECT count(*) FROM crsql_changes", 3);
+
+  // now test:
+  // 1. site_id comparison
+  // 2. db_version comparison
+
+  printf("is null\n");
+  assertCount(db, "SELECT count(*) FROM crsql_changes WHERE site_id IS NULL",
+              3);
+
+  printf("is not null\n");
+  assertCount(
+      db, "SELECT count(*) FROM crsql_changes WHERE site_id IS NOT NULL", 0);
+
+  printf("equals\n");
+  assertCount(
+      db, "SELECT count(*) FROM crsql_changes WHERE site_id = crsql_site_id()",
+      0);
+
+  // 0 rows is actually correct ANSI sql behavior. NULLs are never equal, or not
+  // equal, to anything in ANSI SQL. So users must use `IS NOT` to check rather
+  // than !=.
+  //
+  // https://stackoverflow.com/questions/60017275/why-null-is-not-equal-to-anything-is-a-false-statement
+  printf("not equals\n");
+  assertCount(
+      db, "SELECT count(*) FROM crsql_changes WHERE site_id != crsql_site_id()",
+      0);
+
+  printf("is not\n");
+  // All rows are currently null for site_id
+  assertCount(
+      db,
+      "SELECT count(*) FROM crsql_changes WHERE site_id IS NOT crsql_site_id()",
+      3);
+
+  // compare on db_version _and_ site_id
+
+  // compare upper and lower bound on db_version
+  printf("double bounded version\n");
+  assertCount(db,
+              "SELECT count(*) FROM crsql_changes WHERE db_version >= 1 AND "
+              "db_version < 2",
+              1);
+
+  printf("OR condition\n");
+  assertCount(db,
+              "SELECT count(*) FROM crsql_changes WHERE db_version > 2 OR "
+              "site_id IS NULL",
+              3);
+
+  // compare on pks, table name, other not perfectly supported columns
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+// test value extraction under all filter conditions
+
 // static void testSinglePksTable()
 // {
 // }
@@ -57,4 +151,5 @@ static void testManyPkTable() {
 void crsqlChangesVtabTestSuite() {
   printf("\e[47m\e[1;30mSuite: crsql_changesVtab\e[0m\n");
   testManyPkTable();
+  testFilters();
 }

libsql-sqlite3/ext/crr/src/consts.h

@@ -7,11 +7,6 @@
 #define MIN_POSSIBLE_DB_VERSION 0L
 
 #define __CRSQL_CLOCK_LEN 13
-// NB: crsql_quoteConcat
-#define QC_DELIM '|'
-
-#define DELETE_CID_SENTINEL "__crsql_del"
-#define PKS_ONLY_CID_SENTINEL "__crsql_pko"
 
 #define CRR_SPACE 0
 #define USER_SPACE 1
@@ -20,15 +15,26 @@
   "SELECT tbl_name FROM sqlite_master WHERE type='table' AND tbl_name LIKE " \
   "'%__crsql_clock'"
 
-#define SET_SYNC_BIT "select crsql_internal_sync_bit(1)"
-#define CLEAR_SYNC_BIT "select crsql_internal_sync_bit(0)"
+#define SET_SYNC_BIT "SELECT crsql_internal_sync_bit(1)"
+#define CLEAR_SYNC_BIT "SELECT crsql_internal_sync_bit(0)"
 
-#define TBL_SITE_ID "__crsql_siteid"
-#define TBL_DB_VERSION "__crsql_dbversion"
+#define TBL_SITE_ID "__crsql_site_id"
+#define TBL_DB_VERSION "__crsql_db_version"
 #define TBL_SCHEMA "crsql_master"
-#define UNION "UNION"
+#define UNION_ALL "UNION ALL"
 
 #define MAX_TBL_NAME_LEN 2048
 #define SITE_ID_LEN 16
 
+// Version int:
+// M - major
+// m - minor
+// p - patch
+// b - build
+// MM.mm.pp.bb
+// 00 00 00 00
+// Given we can't prefix an int with 0s, read from right to left.
+// Rightmost is always `bb`
+#define CRSQLITE_VERSION 130000
+
 #endif

libsql-sqlite3/ext/crr/src/crsqlite.c

@@ -1,5 +1,6 @@
 #include "crsqlite.h"
 SQLITE_EXTENSION_INIT1
+LIBSQL_EXTENSION_INIT1
 
 #include <assert.h>
 #include <ctype.h>
@@ -10,138 +11,22 @@ SQLITE_EXTENSION_INIT1
 #include "changes-vtab.h"
 #include "consts.h"
 #include "ext-data.h"
+#include "rust.h"
 #include "tableinfo.h"
-#include "triggers.h"
 #include "util.h"
 
-static void uuid(unsigned char *blob) {
-  sqlite3_randomness(16, blob);
-  blob[6] = (blob[6] & 0x0f) + 0x40;
-  blob[8] = (blob[8] & 0x3f) + 0x80;
-}
-
-/**
- * The site id table is used to persist the site id and
- * populate `siteIdBlob` on initialization of a connection.
- */
-static int createSiteIdAndSiteIdTable(sqlite3 *db, unsigned char *ret) {
-  int rc = SQLITE_OK;
-  sqlite3_stmt *pStmt = 0;
-  char *zSql = 0;
-
-  zSql = sqlite3_mprintf("CREATE TABLE \"%s\" (site_id)", TBL_SITE_ID);
-  rc = sqlite3_exec(db, zSql, 0, 0, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    return rc;
-  }
-
-  zSql = sqlite3_mprintf("INSERT INTO \"%s\" (site_id) VALUES(?)", TBL_SITE_ID);
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    sqlite3_finalize(pStmt);
-    return rc;
-  }
-
-  uuid(ret);
-  rc = sqlite3_bind_blob(pStmt, 1, ret, SITE_ID_LEN, SQLITE_STATIC);
-  if (rc != SQLITE_OK) {
-    return rc;
-  }
-  rc = sqlite3_step(pStmt);
-  if (rc != SQLITE_DONE) {
-    sqlite3_finalize(pStmt);
-    return rc;
-  }
-
-  sqlite3_finalize(pStmt);
-  return SQLITE_OK;
-}
-
-static int initPeerTrackingTable(sqlite3 *db, char **pzErrMsg) {
-  return sqlite3_exec(
-      db,
-      "CREATE TABLE IF NOT EXISTS crsql_tracked_peers (\"site_id\" "
-      "BLOB NOT NULL, \"version\" INTEGER NOT NULL, \"seq\" INTEGER DEFAULT 0, "
-      "\"tag\" INTEGER, \"event\" "
-      "INTEGER, PRIMARY "
-      "KEY (\"site_id\", \"tag\", \"event\")) STRICT;",
-      0, 0, pzErrMsg);
-}
-
-/**
- * Loads the siteId into memory. If a site id
- * cannot be found for the given database one is created
- * and saved to the site id table.
- */
-static int initSiteId(sqlite3 *db, unsigned char *ret) {
-  char *zSql = 0;
-  sqlite3_stmt *pStmt = 0;
-  int rc = SQLITE_OK;
-  int tableExists = 0;
-  const void *siteIdFromTable = 0;
-
-  // look for site id table
-  tableExists = crsql_doesTableExist(db, TBL_SITE_ID);
-
-  if (tableExists == 0) {
-    return createSiteIdAndSiteIdTable(db, ret);
-  }
-
-  // read site id from the table and return it
-  zSql = sqlite3_mprintf("SELECT site_id FROM %Q", TBL_SITE_ID);
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    return rc;
-  }
-
-  rc = sqlite3_step(pStmt);
-  if (rc != SQLITE_ROW) {
-    sqlite3_finalize(pStmt);
-    return rc;
-  }
-
-  siteIdFromTable = sqlite3_column_blob(pStmt, 0);
-  memcpy(ret, siteIdFromTable, SITE_ID_LEN);
-  sqlite3_finalize(pStmt);
-
-  return SQLITE_OK;
-}
-
-static int createSchemaTableIfNotExists(sqlite3 *db) {
-  int rc = SQLITE_OK;
-
-  rc = sqlite3_exec(db, "SAVEPOINT crsql_create_schema_table;", 0, 0, 0);
-  if (rc != SQLITE_OK) {
-    return rc;
-  }
-
-  char *zSql = sqlite3_mprintf(
-      "CREATE TABLE IF NOT EXISTS \"%s\" (\"key\" TEXT PRIMARY KEY, \"value\" "
-      "TEXT);",
-      TBL_SCHEMA);
-  rc = sqlite3_exec(db, zSql, 0, 0, 0);
-  sqlite3_free(zSql);
-
-  if (rc != SQLITE_OK) {
-    sqlite3_exec(db, "ROLLBACK;", 0, 0, 0);
-    return rc;
-  }
-
-  sqlite3_exec(db, "RELEASE crsql_create_schema_table;", 0, 0, 0);
-
-  return rc;
-}
+// see
+// https://github.com/chromium/chromium/commit/579b3dd0ea41a40da8a61ab87a8b0bc39e158998
+// & https://github.com/rust-lang/rust/issues/73632 &
+// https://sourcegraph.com/github.com/chromium/chromium/-/commit/579b3dd0ea41a40da8a61ab87a8b0bc39e158998?visible=1
+#ifdef CRSQLITE_WASM
+unsigned char __rust_no_alloc_shim_is_unstable;
+#endif
 
 /**
  * return the uuid which uniquely identifies this database.
  *
- * `select crsql_siteid()`
+ * `select crsql_site_id()`
  *
  * @param context
  * @param argc
@@ -156,7 +41,7 @@ static void siteIdFunc(sqlite3_context *context, int argc,
 /**
  * Return the current version of the database.
  *
- * `select crsql_dbversion()`
+ * `select crsql_db_version()`
  */
 static void dbVersionFunc(sqlite3_context *context, int argc,
                           sqlite3_value **argv) {
@@ -176,7 +61,7 @@ static void dbVersionFunc(sqlite3_context *context, int argc,
 /**
  * Return the next version of the database for use in inserts/updates/deletes
  *
- * `select crsql_nextdbversion()`
+ * `select crsql_next_db_version()`
  *
  * Nit: this should be same as `crsql_db_version`
  * If you change this behavior you need to change trigger behaviors
@@ -188,6 +73,8 @@ static void nextDbVersionFunc(sqlite3_context *context, int argc,
   char *errmsg = 0;
   crsql_ExtData *pExtData = (crsql_ExtData *)sqlite3_user_data(context);
   sqlite3 *db = sqlite3_context_db_handle(context);
+  // "getDbVersion" is really just filling the cached db version value if
+  // invalid
   int rc = crsql_getDbVersion(db, pExtData, &errmsg);
   if (rc != SQLITE_OK) {
     sqlite3_result_error(context, errmsg, -1);
@@ -195,66 +82,45 @@ static void nextDbVersionFunc(sqlite3_context *context, int argc,
     return;
   }
 
-  sqlite3_result_int64(context, pExtData->dbVersion + 1);
-}
-
-/**
- * The clock table holds the versions for each column of a given row.
- *
- * These version are set to the dbversion at the time of the write to the
- * column.
- *
- * The dbversion is updated on transaction commit.
- * This allows us to find all columns written in the same transaction
- * albeit with caveats.
- *
- * The caveats being that two partiall overlapping transactions will
- * clobber the full transaction picture given we only keep latest
- * state and not a full causal history.
- *
- * @param tableInfo
- */
-int crsql_createClockTable(sqlite3 *db, crsql_TableInfo *tableInfo,
-                           char **err) {
-  char *zSql = 0;
-  char *pkList = 0;
-  int rc = SQLITE_OK;
-
-  pkList = crsql_asIdentifierList(tableInfo->pks, tableInfo->pksLen, 0);
-  zSql = sqlite3_mprintf(
-      "CREATE TABLE IF NOT EXISTS \"%s__crsql_clock\" (\
-      %s,\
-      \"__crsql_col_name\" NOT NULL,\
-      \"__crsql_col_version\" NOT NULL,\
-      \"__crsql_db_version\" NOT NULL,\
-      \"__crsql_site_id\",\
-      PRIMARY KEY (%s, \"__crsql_col_name\")\
-    )",
-      tableInfo->tblName, pkList, pkList);
-  sqlite3_free(pkList);
-
-  rc = sqlite3_exec(db, zSql, 0, 0, err);
-  sqlite3_free(zSql);
-  if (rc != SQLITE_OK) {
-    return rc;
+  sqlite3_int64 providedVersion = 0;
+  if (argc == 1) {
+    providedVersion = sqlite3_value_int64(argv[0]);
   }
 
-  zSql = sqlite3_mprintf(
-      "CREATE INDEX IF NOT EXISTS \"%s__crsql_clock_dbv_idx\" ON "
-      "\"%s__crsql_clock\" (\"__crsql_db_version\")",
-      tableInfo->tblName, tableInfo->tblName);
-  sqlite3_exec(db, zSql, 0, 0, err);
-  sqlite3_free(zSql);
+  // now return max of:
+  // dbVersion + 1, pendingDbVersion, arg (if there is one)
+  // and set pendingDbVersion to that max
+  sqlite3_int64 ret = pExtData->dbVersion + 1;
+  if (ret < pExtData->pendingDbVersion) {
+    ret = pExtData->pendingDbVersion;
+  }
+  if (ret < providedVersion) {
+    ret = providedVersion;
+  }
+  pExtData->pendingDbVersion = ret;
 
-  return rc;
+  sqlite3_result_int64(context, ret);
+}
+
+static void incrementAndGetSeqFunc(sqlite3_context *context, int argc,
+                                   sqlite3_value **argv) {
+  crsql_ExtData *pExtData = (crsql_ExtData *)sqlite3_user_data(context);
+  sqlite3_result_int(context, pExtData->seq);
+  pExtData->seq += 1;
+}
+
+static void getSeqFunc(sqlite3_context *context, int argc,
+                       sqlite3_value **argv) {
+  crsql_ExtData *pExtData = (crsql_ExtData *)sqlite3_user_data(context);
+  sqlite3_result_int(context, pExtData->seq);
 }
 
 /**
  * Create a new crr --
  * all triggers, views, tables
  */
-static int createCrr(sqlite3_context *context, sqlite3 *db,
-                     const char *schemaName, const char *tblName, char **err) {
+int crsql_createCrr(sqlite3 *db, const char *schemaName, const char *tblName,
+                    int isCommitAlter, int noTx, char **err) {
   int rc = SQLITE_OK;
   crsql_TableInfo *tableInfo = 0;
 
@@ -277,11 +143,11 @@ static int createCrr(sqlite3_context *context, sqlite3 *db,
     return rc;
   }
 
-  rc = crsql_createClockTable(db, tableInfo, err);
+  rc = crsql_create_clock_table(db, tableInfo, err);
   if (rc == SQLITE_OK) {
     rc = crsql_remove_crr_triggers_if_exist(db, tableInfo->tblName);
     if (rc == SQLITE_OK) {
-      rc = crsql_createCrrTriggers(db, tableInfo, err);
+      rc = crsql_create_crr_triggers(db, tableInfo, err);
     }
   }
 
@@ -294,8 +160,8 @@ static int createCrr(sqlite3_context *context, sqlite3 *db,
   for (size_t i = 0; i < tableInfo->nonPksLen; i++) {
     nonPkNames[i] = tableInfo->nonPks[i].name;
   }
-  rc = crsql_backfill_table(context, tblName, pkNames, tableInfo->pksLen,
-                            nonPkNames, tableInfo->nonPksLen);
+  rc = crsql_backfill_table(db, tblName, pkNames, tableInfo->pksLen, nonPkNames,
+                            tableInfo->nonPksLen, isCommitAlter, noTx);
   sqlite3_free(pkNames);
   sqlite3_free(nonPkNames);
 
@@ -316,6 +182,7 @@ static void crsqlSyncBit(sqlite3_context *context, int argc,
   // Args? We're setting the value of the bit
   int newValue = sqlite3_value_int(argv[0]);
   *syncBit = newValue;
+  sqlite3_result_int(context, newValue);
 }
 
 /**
@@ -355,7 +222,7 @@ static void crsqlMakeCrrFunc(sqlite3_context *context, int argc,
     return;
   }
 
-  rc = createCrr(context, db, schemaName, tblName, &errmsg);
+  rc = crsql_createCrr(db, schemaName, tblName, 0, 0, &errmsg);
   if (rc != SQLITE_OK) {
     sqlite3_result_error(context, errmsg, -1);
     sqlite3_result_error_code(context, rc);
@@ -408,20 +275,8 @@ static void crsqlBeginAlterFunc(sqlite3_context *context, int argc,
   }
 }
 
-int crsql_compactPostAlter(sqlite3 *db, const char *tblName, char **errmsg) {
-  // 1. remove all entries in the clock table that have a column
-  // name that does not exist
-  // NOTE!: this is bugged, right? Doesn't this compact out pk_only and delete
-  // sentinels?
-  char *zSql = sqlite3_mprintf(
-      "DELETE FROM \"%w__crsql_clock\" WHERE \"__crsql_col_name\" NOT IN "
-      "(SELECT name FROM pragma_table_info(%Q))",
-      tblName, tblName);
-  int rc = sqlite3_exec(db, zSql, 0, 0, errmsg);
-  sqlite3_free(zSql);
-
-  return rc;
-}
+int crsql_compact_post_alter(sqlite3 *db, const char *tblName,
+                             crsql_ExtData *pExtData, char **errmsg);
 
 static void crsqlCommitAlterFunc(sqlite3_context *context, int argc,
                                  sqlite3_value **argv) {
@@ -448,9 +303,10 @@ static void crsqlCommitAlterFunc(sqlite3_context *context, int argc,
     tblName = (const char *)sqlite3_value_text(argv[0]);
   }
 
-  rc = crsql_compactPostAlter(db, tblName, &errmsg);
+  crsql_ExtData *pExtData = (crsql_ExtData *)sqlite3_user_data(context);
+  rc = crsql_compact_post_alter(db, tblName, pExtData, &errmsg);
   if (rc == SQLITE_OK) {
-    rc = createCrr(context, db, schemaName, tblName, &errmsg);
+    rc = crsql_createCrr(db, schemaName, tblName, 1, 0, &errmsg);
   }
   if (rc == SQLITE_OK) {
     rc = sqlite3_exec(db, "RELEASE alter_crr", 0, 0, &errmsg);
@@ -475,17 +331,31 @@ static void crsqlFinalize(sqlite3_context *context, int argc,
   crsql_finalize(pExtData);
 }
 
+static void crsqlRowsImpacted(sqlite3_context *context, int argc,
+                              sqlite3_value **argv) {
+  crsql_ExtData *pExtData = (crsql_ExtData *)sqlite3_user_data(context);
+  sqlite3_result_int(context, pExtData->rowsImpacted);
+}
+
 static int commitHook(void *pUserData) {
   crsql_ExtData *pExtData = (crsql_ExtData *)pUserData;
 
-  pExtData->dbVersion = -1;
+  pExtData->dbVersion = pExtData->pendingDbVersion;
+  pExtData->pendingDbVersion = -1;
+  pExtData->seq = 0;
   return SQLITE_OK;
 }
 
 static void rollbackHook(void *pUserData) {
   crsql_ExtData *pExtData = (crsql_ExtData *)pUserData;
 
-  pExtData->dbVersion = -1;
+  pExtData->pendingDbVersion = -1;
+  pExtData->seq = 0;
+}
+
+static void closeHook(void *pUserData, sqlite3 *db) {
+  crsql_ExtData *pExtData = (crsql_ExtData *)pUserData;
+  crsql_finalize(pExtData);
 }
 
 int sqlite3_crsqlrustbundle_init(sqlite3 *db, char **pzErrMsg,
@@ -495,43 +365,86 @@ int sqlite3_crsqlrustbundle_init(sqlite3 *db, char **pzErrMsg,
 __declspec(dllexport)
 #endif
     int sqlite3_crsqlite_init(sqlite3 *db, char **pzErrMsg,
-                              const sqlite3_api_routines *pApi) {
+                              const sqlite3_api_routines *pApi,
+                              const libsql_api_routines *pLibsqlApi) {
   int rc = SQLITE_OK;
 
   SQLITE_EXTENSION_INIT2(pApi);
+  LIBSQL_EXTENSION_INIT2(pLibsqlApi)
 
-  rc = initPeerTrackingTable(db, pzErrMsg);
+  // TODO: should be moved lower once we finish migrating to rust.
+  // RN it is safe here since the rust bundle init is largely just reigstering
+  // function pointers. we need to init the rust bundle otherwise sqlite api
+  // methods are not isntalled when we start calling rust
+  rc = sqlite3_crsqlrustbundle_init(db, pzErrMsg, pApi);
+
+  rc = crsql_init_peer_tracking_table(db);
   if (rc != SQLITE_OK) {
     return rc;
   }
 
-  crsql_ExtData *pExtData = crsql_newExtData(db);
+  // Register a thread & connection local bit to toggle on or off
+  // our triggers depending on the source of updates to a table.
+  int *syncBit = sqlite3_malloc(sizeof *syncBit);
+  *syncBit = 0;
+  rc = sqlite3_create_function_v2(
+      db, "crsql_internal_sync_bit",
+      -1,                              // num args: -1 -> 0 or more
+      SQLITE_UTF8 | SQLITE_INNOCUOUS,  // configuration
+      syncBit,                         // user data
+      crsqlSyncBit,
+      0,            // step
+      0,            // final
+      sqlite3_free  // destroy / free syncBit
+  );
+  if (rc != SQLITE_OK) {
+    return rc;
+  }
+
+  if (rc == SQLITE_OK) {
+    rc = crsql_maybe_update_db(db, pzErrMsg);
+  }
+
+  unsigned char *siteIdBuffer = sqlite3_malloc(SITE_ID_LEN * sizeof(char *));
+  if (rc == SQLITE_OK) {
+    rc = crsql_init_site_id(db, siteIdBuffer);
+  }
+
+  crsql_ExtData *pExtData = crsql_newExtData(db, siteIdBuffer);
   if (pExtData == 0) {
     return SQLITE_ERROR;
   }
 
-  rc = initSiteId(db, pExtData->siteId);
-  rc += createSchemaTableIfNotExists(db);
-
   if (rc == SQLITE_OK) {
     rc = sqlite3_create_function(
-        db, "crsql_siteid", 0,
+        db, "crsql_site_id", 0,
         // siteid never changes -- deterministic and innnocuous
         SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, pExtData,
         siteIdFunc, 0, 0);
   }
   if (rc == SQLITE_OK) {
-    rc = sqlite3_create_function_v2(db, "crsql_dbversion", 0,
+    rc = sqlite3_create_function_v2(db, "crsql_db_version", 0,
                                     // dbversion can change on each invocation.
                                     SQLITE_UTF8 | SQLITE_INNOCUOUS, pExtData,
                                     dbVersionFunc, 0, 0, freeConnectionExtData);
   }
   if (rc == SQLITE_OK) {
-    rc = sqlite3_create_function(db, "crsql_nextdbversion", 0,
+    rc = sqlite3_create_function(db, "crsql_next_db_version", -1,
                                  // dbversion can change on each invocation.
                                  SQLITE_UTF8 | SQLITE_INNOCUOUS, pExtData,
                                  nextDbVersionFunc, 0, 0);
   }
+  if (rc == SQLITE_OK) {
+    rc = sqlite3_create_function(db, "crsql_increment_and_get_seq", 0,
+                                 SQLITE_UTF8 | SQLITE_INNOCUOUS, pExtData,
+                                 incrementAndGetSeqFunc, 0, 0);
+  }
+  if (rc == SQLITE_OK) {
+    rc = sqlite3_create_function(
+        db, "crsql_get_seq", 0,
+        SQLITE_UTF8 | SQLITE_INNOCUOUS | SQLITE_DETERMINISTIC, pExtData,
+        getSeqFunc, 0, 0);
+  }
 
   if (rc == SQLITE_OK) {
     // Only register a commit hook, not update or pre-update, since all rows
@@ -553,7 +466,7 @@ __declspec(dllexport)
 
   if (rc == SQLITE_OK) {
     rc = sqlite3_create_function(db, "crsql_commit_alter", -1,
-                                 SQLITE_UTF8 | SQLITE_DIRECTONLY, 0,
+                                 SQLITE_UTF8 | SQLITE_DIRECTONLY, pExtData,
                                  crsqlCommitAlterFunc, 0, 0);
   }
 
@@ -565,20 +478,9 @@ __declspec(dllexport)
   }
 
   if (rc == SQLITE_OK) {
-    // Register a thread & connection local bit to toggle on or off
-    // our triggers depending on the source of updates to a table.
-    int *syncBit = sqlite3_malloc(sizeof *syncBit);
-    *syncBit = 0;
-    rc = sqlite3_create_function_v2(
-        db, "crsql_internal_sync_bit",
-        -1,                              // num args: -1 -> 0 or more
-        SQLITE_UTF8 | SQLITE_INNOCUOUS,  // configuration
-        syncBit,                         // user data
-        crsqlSyncBit,
-        0,            // step
-        0,            // final
-        sqlite3_free  // destroy / free syncBit
-    );
+    rc = sqlite3_create_function(db, "crsql_rows_impacted", 0,
+                                 SQLITE_UTF8 | SQLITE_INNOCUOUS, pExtData,
+                                 crsqlRowsImpacted, 0, 0);
   }
 
   if (rc == SQLITE_OK) {
@@ -589,9 +491,9 @@ __declspec(dllexport)
   if (rc == SQLITE_OK) {
     // TODO: get the prior callback so we can call it rather than replace
     // it?
+    libsql_close_hook(db, closeHook, pExtData);
     sqlite3_commit_hook(db, commitHook, pExtData);
     sqlite3_rollback_hook(db, rollbackHook, pExtData);
-    rc = sqlite3_crsqlrustbundle_init(db, pzErrMsg, pApi);
   }
 
   return rc;

libsql-sqlite3/ext/crr/src/crsqlite.h

@@ -3,6 +3,7 @@
 
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT3
+LIBSQL_EXTENSION_INIT3
 
 #include <stdint.h>
 
@@ -14,10 +15,4 @@ SQLITE_EXTENSION_INIT3
 #define STATIC
 #endif
 
-int crsql_createClockTable(sqlite3 *db, crsql_TableInfo *tableInfo, char **err);
-int crsql_backfill_table(sqlite3_context *context, const char *tblName,
-                         const char **zpkNames, int pkCount,
-                         const char **zNonPkNames, int nonPkCount);
-int crsql_is_crr(sqlite3 *db, const char *tblName);
-
 #endif

libsql-sqlite3/ext/crr/src/crsqlite.test.c

@@ -5,8 +5,8 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "changes-vtab-common.h"
 #include "consts.h"
+#include "rust.h"
 #include "tableinfo.h"
 #include "util.h"
 
@@ -17,6 +17,15 @@
   }
 #endif
 
+#define CHANGES_SINCE_VTAB_TBL 0
+#define CHANGES_SINCE_VTAB_PK 1
+#define CHANGES_SINCE_VTAB_CID 2
+#define CHANGES_SINCE_VTAB_CVAL 3
+#define CHANGES_SINCE_VTAB_COL_VRSN 4
+#define CHANGES_SINCE_VTAB_DB_VRSN 5
+#define CHANGES_SINCE_VTAB_SITE_ID 6
+#define CHANGES_SINCE_VTAB_SEQ 7
+
 int crsql_close(sqlite3 *db);
 
 /**
@@ -28,13 +37,13 @@ int crsql_close(sqlite3 *db);
  * @param since
  * @return int
  */
-static int syncLeftToRight(sqlite3 *db1, sqlite3 *db2, sqlite3_int64 since) {
+int syncLeftToRight(sqlite3 *db1, sqlite3 *db2, sqlite3_int64 since) {
   sqlite3_stmt *pStmtRead = 0;
   sqlite3_stmt *pStmtWrite = 0;
   sqlite3_stmt *pStmt = 0;
   int rc = SQLITE_OK;
 
-  rc += sqlite3_prepare_v2(db2, "SELECT crsql_siteid()", -1, &pStmt, 0);
+  rc += sqlite3_prepare_v2(db2, "SELECT crsql_site_id()", -1, &pStmt, 0);
   if (sqlite3_step(pStmt) != SQLITE_ROW) {
     sqlite3_finalize(pStmt);
     return SQLITE_ERROR;
@@ -50,16 +59,18 @@ static int syncLeftToRight(sqlite3 *db1, sqlite3 *db2, sqlite3_int64 since) {
   rc += sqlite3_bind_value(pStmtRead, 1, sqlite3_column_value(pStmt, 0));
   assert(rc == SQLITE_OK);
   rc += sqlite3_prepare_v2(
-      db2, "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?)", -1,
+      db2, "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", -1,
       &pStmtWrite, 0);
   assert(rc == SQLITE_OK);
   // printf("err: %s\n", err);
 
   while (sqlite3_step(pStmtRead) == SQLITE_ROW) {
-    for (int i = 0; i < 7; ++i) {
-      sqlite3_bind_value(pStmtWrite, i + 1, sqlite3_column_value(pStmtRead, i));
+    for (int i = 0; i < 9; ++i) {
+      assert(sqlite3_bind_value(pStmtWrite, i + 1,
+                                sqlite3_column_value(pStmtRead, i)) ==
+             SQLITE_OK);
     }
-    sqlite3_step(pStmtWrite);
+    assert(sqlite3_step(pStmtWrite) == SQLITE_DONE);
     sqlite3_reset(pStmtWrite);
   }
 
@@ -96,13 +107,13 @@ static void testCreateClockTable() {
   rc = crsql_getTableInfo(db, "boo", &tc4, &err);
   CHECK_OK
 
-  rc = crsql_createClockTable(db, tc1, &err);
+  rc = crsql_create_clock_table(db, tc1, &err);
   CHECK_OK
-  rc = crsql_createClockTable(db, tc2, &err);
+  rc = crsql_create_clock_table(db, tc2, &err);
   CHECK_OK
-  rc = crsql_createClockTable(db, tc3, &err);
+  rc = crsql_create_clock_table(db, tc3, &err);
   CHECK_OK
-  rc = crsql_createClockTable(db, tc4, &err);
+  rc = crsql_create_clock_table(db, tc4, &err);
   CHECK_OK
 
   crsql_freeTableInfo(tc1);
@@ -127,7 +138,7 @@ static char *getQuotedSiteId(sqlite3 *db) {
   sqlite3_stmt *pStmt = 0;
   int rc = SQLITE_OK;
 
-  rc += sqlite3_prepare_v2(db, "SELECT quote(crsql_siteid())", -1, &pStmt, 0);
+  rc += sqlite3_prepare_v2(db, "SELECT quote(crsql_site_id())", -1, &pStmt, 0);
   assert(rc == SQLITE_OK);
   if (sqlite3_step(pStmt) != SQLITE_ROW) {
     sqlite3_finalize(pStmt);
@@ -240,7 +251,7 @@ static void teste2e() {
   db3siteid = getQuotedSiteId(db3);
 
   rc += sqlite3_exec(db1, "insert into foo values (1, 2.0e2);", 0, 0, &err);
-  rc += sqlite3_exec(db2, "insert into foo values (2, X'1232');", 0, 0, &err);
+  rc += sqlite3_exec(db1, "insert into foo values (2, X'1232');", 0, 0, &err);
   assert(rc == SQLITE_OK);
 
   syncLeftToRight(db1, db2, 0);
@@ -269,11 +280,11 @@ static void teste2e() {
   // printf("db2sid: %s\n", db2siteid);
   // printf("db3sid: %s\n", db3siteid);
   // printf("tempsid: %s\n", tmpSiteid);
-  assert(strcmp(tmpSiteid, db1siteid) == 0);
+  assert(strcmp(tmpSiteid, "NULL") == 0);
 
   rc = sqlite3_step(pStmt3);
   assert(rc == SQLITE_ROW);
-  assert(strcmp((const char *)sqlite3_column_text(pStmt3, 0), db2siteid) == 0);
+  assert(strcmp((const char *)sqlite3_column_text(pStmt3, 0), "NULL") == 0);
   sqlite3_finalize(pStmt3);
 
   rc = sqlite3_prepare_v2(db2, "SELECT * FROM foo ORDER BY a ASC", -1, &pStmt2,
@@ -342,9 +353,9 @@ static void testSelectChangesAfterChangingColumnName() {
   // clock records should now be for column `c` with a `null` value.
   // nit: test if a default value is set for the column
   while ((rc = sqlite3_step(pStmt)) == SQLITE_ROW) {
-    ++numRows;
     assert(strcmp((const char *)sqlite3_column_text(pStmt, 0), "c") == 0);
-    assert(strcmp((const char *)sqlite3_column_text(pStmt, 1), "NULL") == 0);
+    assert(sqlite3_column_type(pStmt, 1) == SQLITE_NULL);
+    ++numRows;
   }
   sqlite3_finalize(pStmt);
   // we should still have a change given we never dropped the row
@@ -354,7 +365,7 @@ static void testSelectChangesAfterChangingColumnName() {
   // insert some rows post schema change
   rc = sqlite3_exec(db, "INSERT INTO foo VALUES (2, 3);", 0, 0, 0);
   rc += sqlite3_prepare_v2(
-      db, "SELECT * FROM crsql_changes WHERE db_version > 1", -1, &pStmt, 0);
+      db, "SELECT * FROM crsql_changes WHERE db_version >= 1", -1, &pStmt, 0);
   assert(rc == SQLITE_OK);
   numRows = 0;
   // Columns that no long exist post-alter should not
@@ -362,87 +373,108 @@ static void testSelectChangesAfterChangingColumnName() {
   while ((rc = sqlite3_step(pStmt)) == SQLITE_ROW) {
     assert(strcmp("foo", (const char *)sqlite3_column_text(
                              pStmt, CHANGES_SINCE_VTAB_TBL)) == 0);
-    assert(strcmp("2", (const char *)sqlite3_column_text(
-                           pStmt, CHANGES_SINCE_VTAB_PK)) == 0);
-    assert(strcmp("c", (const char *)sqlite3_column_text(
-                           pStmt, CHANGES_SINCE_VTAB_CID)) == 0);
-    assert(strcmp("3", (const char *)sqlite3_column_text(
-                           pStmt, CHANGES_SINCE_VTAB_CVAL)) == 0);
+    const unsigned char *pkBlob = (const unsigned char *)sqlite3_column_blob(
+        pStmt, CHANGES_SINCE_VTAB_PK);
+
+    if (numRows == 0) {
+      assert(pkBlob[0] == 0x01);
+      assert(pkBlob[1] == 0x09);
+      assert(pkBlob[2] == 0x01);
+    } else {
+      assert(pkBlob[0] == 0x01);
+      assert(pkBlob[1] == 0x09);
+      assert(pkBlob[2] == 0x02);
+    }
+
+    if (numRows == 0) {
+      assert(strcmp("c", (const char *)sqlite3_column_text(
+                             pStmt, CHANGES_SINCE_VTAB_CID)) == 0);
+    }
+    if (numRows == 1) {
+      assert(strcmp("c", (const char *)sqlite3_column_text(
+                             pStmt, CHANGES_SINCE_VTAB_CID)) == 0);
+      assert(3 == sqlite3_column_int(pStmt, CHANGES_SINCE_VTAB_CVAL));
+    }
+
     ++numRows;
   }
   sqlite3_finalize(pStmt);
-  assert(numRows == 1);
+  assert(numRows == 2);
   assert(rc == SQLITE_DONE);
 
   crsql_close(db);
   printf("\t\e[0;32mSuccess\e[0m\n");
 }
 
-static void testInsertChangesWithUnkownColumnNames() {
-  printf("InsertChangesWithUnknownColumnName\n");
+// We no longer support this given we fixup metadata on migration.
+// Maybe we should support it though?
+// static void testInsertChangesWithUnkownColumnNames() {
+//   printf("InsertChangesWithUnknownColumnName\n");
 
-  int rc = SQLITE_OK;
-  sqlite3 *db1;
-  sqlite3 *db2;
-  rc = sqlite3_open(":memory:", &db1);
-  rc += sqlite3_open(":memory:", &db2);
+//   int rc = SQLITE_OK;
+//   sqlite3 *db1;
+//   sqlite3 *db2;
+//   rc = sqlite3_open(":memory:", &db1);
+//   rc += sqlite3_open(":memory:", &db2);
 
-  rc += sqlite3_exec(db1, "CREATE TABLE foo(a primary key, b);", 0, 0, 0);
-  rc += sqlite3_exec(db1, "SELECT crsql_as_crr('foo')", 0, 0, 0);
-  rc += sqlite3_exec(db2, "CREATE TABLE foo(a primary key, c);", 0, 0, 0);
-  rc += sqlite3_exec(db2, "SELECT crsql_as_crr('foo')", 0, 0, 0);
-  assert(rc == SQLITE_OK);
+//   rc += sqlite3_exec(db1, "CREATE TABLE foo(a primary key, b);", 0, 0, 0);
+//   rc += sqlite3_exec(db1, "SELECT crsql_as_crr('foo')", 0, 0, 0);
+//   rc += sqlite3_exec(db2, "CREATE TABLE foo(a primary key, c);", 0, 0, 0);
+//   rc += sqlite3_exec(db2, "SELECT crsql_as_crr('foo')", 0, 0, 0);
+//   assert(rc == SQLITE_OK);
 
-  rc += sqlite3_exec(db1, "INSERT INTO foo VALUES (1, 2);", 0, 0, 0);
-  rc += sqlite3_exec(db2, "INSERT INTO foo VALUES (2, 3);", 0, 0, 0);
-  assert(rc == SQLITE_OK);
+//   rc += sqlite3_exec(db1, "INSERT INTO foo VALUES (1, 2);", 0, 0, 0);
+//   rc += sqlite3_exec(db2, "INSERT INTO foo VALUES (2, 3);", 0, 0, 0);
+//   assert(rc == SQLITE_OK);
 
-  sqlite3_stmt *pStmtRead = 0;
-  sqlite3_stmt *pStmtWrite = 0;
-  rc +=
-      sqlite3_prepare_v2(db1, "SELECT * FROM crsql_changes", -1, &pStmtRead, 0);
-  rc += sqlite3_prepare_v2(
-      db2, "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?)", -1,
-      &pStmtWrite, 0);
-  assert(rc == SQLITE_OK);
+//   sqlite3_stmt *pStmtRead = 0;
+//   sqlite3_stmt *pStmtWrite = 0;
+//   rc +=
+//       sqlite3_prepare_v2(db1, "SELECT * FROM crsql_changes", -1, &pStmtRead,
+//       0);
+//   rc += sqlite3_prepare_v2(
+//       db2, "INSERT INTO crsql_changes VALUES (?, ?, ?, ?, ?, ?, ?, ?)", -1,
+//       &pStmtWrite, 0);
+//   assert(rc == SQLITE_OK);
 
-  while (sqlite3_step(pStmtRead) == SQLITE_ROW) {
-    for (int i = 0; i < 7; ++i) {
-      sqlite3_bind_value(pStmtWrite, i + 1, sqlite3_column_value(pStmtRead, i));
-    }
+//   while (sqlite3_step(pStmtRead) == SQLITE_ROW) {
+//     for (int i = 0; i < 7; ++i) {
+//       sqlite3_bind_value(pStmtWrite, i + 1, sqlite3_column_value(pStmtRead,
+//       i));
+//     }
 
-    sqlite3_step(pStmtWrite);
-    sqlite3_reset(pStmtWrite);
-  }
-  sqlite3_finalize(pStmtWrite);
-  sqlite3_finalize(pStmtRead);
+//     sqlite3_step(pStmtWrite);
+//     sqlite3_reset(pStmtWrite);
+//   }
+//   sqlite3_finalize(pStmtWrite);
+//   sqlite3_finalize(pStmtRead);
 
-  // select all from db2.
-  // it should have a row for pk 1.
-  sqlite3_prepare_v2(db2, "SELECT * FROM foo ORDER BY a ASC", -1, &pStmtRead,
-                     0);
-  int comparisons = 0;
-  while (sqlite3_step(pStmtRead) == SQLITE_ROW) {
-    if (comparisons == 0) {
-      assert(sqlite3_column_int(pStmtRead, 0) == 1);
-      assert(sqlite3_column_type(pStmtRead, 1) == SQLITE_NULL);
-    } else {
-      assert(sqlite3_column_int(pStmtRead, 0) == 2);
-      assert(sqlite3_column_int(pStmtRead, 1) == 3);
-    }
-    comparisons += 1;
-  }
-  sqlite3_finalize(pStmtRead);
+//   // select all from db2.
+//   // it should have a row for pk 1.
+//   sqlite3_prepare_v2(db2, "SELECT * FROM foo ORDER BY a ASC", -1, &pStmtRead,
+//                      0);
+//   int comparisons = 0;
+//   while (sqlite3_step(pStmtRead) == SQLITE_ROW) {
+//     if (comparisons == 0) {
+//       assert(sqlite3_column_int(pStmtRead, 0) == 1);
+//       assert(sqlite3_column_type(pStmtRead, 1) == SQLITE_NULL);
+//     } else {
+//       assert(sqlite3_column_int(pStmtRead, 0) == 2);
+//       assert(sqlite3_column_int(pStmtRead, 1) == 3);
+//     }
+//     comparisons += 1;
+//   }
+//   sqlite3_finalize(pStmtRead);
 
-  assert(comparisons == 2);
-  crsql_close(db1);
-  crsql_close(db2);
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
+//   assert(comparisons == 2);
+//   crsql_close(db1);
+//   crsql_close(db2);
+//   printf("\t\e[0;32mSuccess\e[0m\n");
+// }
 
 static sqlite3_int64 getDbVersion(sqlite3 *db) {
   sqlite3_stmt *pStmt = 0;
-  int rc = sqlite3_prepare_v2(db, "SELECT crsql_dbversion()", -1, &pStmt, 0);
+  int rc = sqlite3_prepare_v2(db, "SELECT crsql_db_version()", -1, &pStmt, 0);
   if (rc != SQLITE_OK) {
     return -1;
   }
@@ -586,34 +618,16 @@ static void testPullingOnlyLocalChanges() {
   // `IS NOT NULL` also fails to call the virtual table bestIndex function with
   // any constraints p pIdxInfo->nConstraint
   sqlite3_prepare_v2(db,
-                     "SELECT count(*) FROM crsql_changes WHERE site_id = NULL",
+                     "SELECT count(*) FROM crsql_changes WHERE site_id IS NULL",
                      -1, &pStmt, 0);
 
   rc = sqlite3_step(pStmt);
   assert(rc == SQLITE_ROW);
 
   int count = sqlite3_column_int(pStmt, 0);
-  // we created 2 local changes, we should get 2 changes back
-  assert(count == 2);
-  sqlite3_finalize(pStmt);
-
-  sqlite3_prepare_v2(db,
-                     "SELECT count(*) FROM crsql_changes WHERE site_id != NULL",
-                     -1, &pStmt, 0);
-  rc = sqlite3_step(pStmt);
-  assert(rc == SQLITE_ROW);
-  count = sqlite3_column_int(pStmt, 0);
-  // we asked for changes that were not local
-  assert(count == 0);
-  sqlite3_finalize(pStmt);
-
-  sqlite3_prepare_v2(db,
-                     "SELECT count(*) FROM crsql_changes WHERE site_id IS NULL",
-                     -1, &pStmt, 0);
-  rc = sqlite3_step(pStmt);
-  assert(rc == SQLITE_ROW);
-  count = sqlite3_column_int(pStmt, 0);
-  // we asked for changes that were not local
+  // we created 2 local changes, we should get 2 changes back. Well 4 really
+  // since row creation is an event.
+  printf("count: %d\n", count);
   assert(count == 2);
   sqlite3_finalize(pStmt);
 
@@ -646,7 +660,7 @@ void crsqlTestSuite() {
   testCreateClockTable();
   teste2e();
   testSelectChangesAfterChangingColumnName();
-  testInsertChangesWithUnkownColumnNames();
+  // testInsertChangesWithUnkownColumnNames();
   testLamportCondition();
   noopsDoNotMoveClocks();
   testPullingOnlyLocalChanges();

libsql-sqlite3/ext/crr/src/ext-data.c

@@ -4,44 +4,54 @@
 #include "get-table.h"
 #include "util.h"
 
-crsql_ExtData *crsql_newExtData(sqlite3 *db) {
+void crsql_init_stmt_cache(crsql_ExtData *pExtData);
+void crsql_clear_stmt_cache(crsql_ExtData *pExtData);
+
+crsql_ExtData *crsql_newExtData(sqlite3 *db, unsigned char *siteIdBuffer) {
   crsql_ExtData *pExtData = sqlite3_malloc(sizeof *pExtData);
 
   pExtData->pPragmaSchemaVersionStmt = 0;
   int rc = sqlite3_prepare_v3(db, "PRAGMA schema_version", -1,
                               SQLITE_PREPARE_PERSISTENT,
                               &(pExtData->pPragmaSchemaVersionStmt), 0);
-  if (rc != SQLITE_OK) {
-    sqlite3_finalize(pExtData->pPragmaSchemaVersionStmt);
-    return 0;
-  }
   pExtData->pPragmaDataVersionStmt = 0;
-  rc = sqlite3_prepare_v3(db, "PRAGMA data_version", -1,
-                          SQLITE_PREPARE_PERSISTENT,
-                          &(pExtData->pPragmaDataVersionStmt), 0);
-  if (rc != SQLITE_OK) {
-    sqlite3_finalize(pExtData->pPragmaDataVersionStmt);
-    sqlite3_finalize(pExtData->pPragmaSchemaVersionStmt);
-    return 0;
-  }
+  rc += sqlite3_prepare_v3(db, "PRAGMA data_version", -1,
+                           SQLITE_PREPARE_PERSISTENT,
+                           &(pExtData->pPragmaDataVersionStmt), 0);
+  pExtData->pSetSyncBitStmt = 0;
+  rc += sqlite3_prepare_v3(db, SET_SYNC_BIT, -1, SQLITE_PREPARE_PERSISTENT,
+                           &(pExtData->pSetSyncBitStmt), 0);
+  pExtData->pClearSyncBitStmt = 0;
+  rc += sqlite3_prepare_v3(db, CLEAR_SYNC_BIT, -1, SQLITE_PREPARE_PERSISTENT,
+                           &(pExtData->pClearSyncBitStmt), 0);
 
-  if (rc != SQLITE_OK) {
-    sqlite3_finalize(pExtData->pPragmaDataVersionStmt);
-    sqlite3_finalize(pExtData->pPragmaSchemaVersionStmt);
-    return 0;
-  }
+  pExtData->pSetSiteIdOrdinalStmt = 0;
+  rc += sqlite3_prepare_v3(
+      db, "INSERT INTO crsql_site_id (site_id) VALUES (?) RETURNING ordinal",
+      -1, SQLITE_PREPARE_PERSISTENT, &(pExtData->pSetSiteIdOrdinalStmt), 0);
+
+  pExtData->pSelectSiteIdOrdinalStmt = 0;
+  rc += sqlite3_prepare_v3(
+      db, "SELECT ordinal FROM crsql_site_id WHERE site_id = ?", -1,
+      SQLITE_PREPARE_PERSISTENT, &(pExtData->pSelectSiteIdOrdinalStmt), 0);
 
   pExtData->dbVersion = -1;
+  pExtData->pendingDbVersion = -1;
+  pExtData->seq = 0;
   pExtData->pragmaSchemaVersion = -1;
   pExtData->pragmaDataVersion = -1;
   pExtData->pragmaSchemaVersionForTableInfos = -1;
-  pExtData->siteId = sqlite3_malloc(SITE_ID_LEN * sizeof *(pExtData->siteId));
+  pExtData->siteId = siteIdBuffer;
   pExtData->pDbVersionStmt = 0;
   pExtData->zpTableInfos = 0;
   pExtData->tableInfosLen = 0;
+  pExtData->rowsImpacted = 0;
+  pExtData->pStmtCache = 0;
+  crsql_init_stmt_cache(pExtData);
 
-  rc = crsql_fetchPragmaDataVersion(db, pExtData);
-  if (rc == -1) {
+  int pv = crsql_fetchPragmaDataVersion(db, pExtData);
+  if (pv == -1 || rc != SQLITE_OK) {
+    crsql_freeExtData(pExtData);
     return 0;
   }
 
@@ -53,7 +63,12 @@ void crsql_freeExtData(crsql_ExtData *pExtData) {
   sqlite3_finalize(pExtData->pDbVersionStmt);
   sqlite3_finalize(pExtData->pPragmaSchemaVersionStmt);
   sqlite3_finalize(pExtData->pPragmaDataVersionStmt);
+  sqlite3_finalize(pExtData->pSetSyncBitStmt);
+  sqlite3_finalize(pExtData->pClearSyncBitStmt);
+  sqlite3_finalize(pExtData->pSetSiteIdOrdinalStmt);
+  sqlite3_finalize(pExtData->pSelectSiteIdOrdinalStmt);
   crsql_freeAllTableInfos(pExtData->zpTableInfos, pExtData->tableInfosLen);
+  crsql_clear_stmt_cache(pExtData);
   sqlite3_free(pExtData);
 }
 
@@ -66,9 +81,18 @@ void crsql_finalize(crsql_ExtData *pExtData) {
   sqlite3_finalize(pExtData->pDbVersionStmt);
   sqlite3_finalize(pExtData->pPragmaSchemaVersionStmt);
   sqlite3_finalize(pExtData->pPragmaDataVersionStmt);
+  sqlite3_finalize(pExtData->pSetSyncBitStmt);
+  sqlite3_finalize(pExtData->pClearSyncBitStmt);
+  sqlite3_finalize(pExtData->pSetSiteIdOrdinalStmt);
+  sqlite3_finalize(pExtData->pSelectSiteIdOrdinalStmt);
+  crsql_clear_stmt_cache(pExtData);
   pExtData->pDbVersionStmt = 0;
   pExtData->pPragmaSchemaVersionStmt = 0;
   pExtData->pPragmaDataVersionStmt = 0;
+  pExtData->pSetSyncBitStmt = 0;
+  pExtData->pClearSyncBitStmt = 0;
+  pExtData->pSetSiteIdOrdinalStmt = 0;
+  pExtData->pSelectSiteIdOrdinalStmt = 0;
 }
 
 #define DB_VERSION_SCHEMA_VERSION 0
@@ -221,7 +245,7 @@ int crsql_fetchDbVersionFromStorage(sqlite3 *db, crsql_ExtData *pExtData,
 }
 
 /**
- * This will return the db version if it exists in `pExtData`
+ * This fills the dbVersion into `pExtData` if it is not already cached there
  *
  * If it does not exist there, it will fetch the current db version
  * from the database.
@@ -231,11 +255,12 @@ int crsql_fetchDbVersionFromStorage(sqlite3 *db, crsql_ExtData *pExtData,
 int crsql_getDbVersion(sqlite3 *db, crsql_ExtData *pExtData, char **errmsg) {
   int rc = SQLITE_OK;
 
-  // version is cached. We clear the cached version
-  // at the end of each transaction so it is safe to return this
-  // without checking the schema version.
-  // It is an error to use crsqlite in such a way that you modify
-  // a schema and fetch changes in the same transaction.
+  // Version is cached. We update the cached version at the end of every
+  // transaction to match what is written to disk. The cache is only invalid if
+  // another connection also made writes. We detect this via `pragmaDataVersion`
+  // and force a re-fetch of the db version when `pragmaDataVersion` changes. It
+  // is an error to use crsqlite in such a way that you modify a schema and
+  // fetch changes in the same transaction.
   rc = crsql_fetchPragmaDataVersion(db, pExtData);
   if (rc == -1) {
     *errmsg = sqlite3_mprintf("failed to fetch PRAGMA data_version");

libsql-sqlite3/ext/crr/src/ext-data.h

@@ -3,9 +3,10 @@
 
 #include "sqlite3ext.h"
 SQLITE_EXTENSION_INIT3
-
 #include "tableinfo.h"
 
+// NOTE: any changes here must be updated in `c.rs` until we've finished porting
+// to rust.
 typedef struct crsql_ExtData crsql_ExtData;
 struct crsql_ExtData {
   // perma statement -- used to check db schema version
@@ -14,9 +15,12 @@ struct crsql_ExtData {
   int pragmaDataVersion;
 
   // this gets set at the start of each transaction on the first invocation
-  // to crsql_nextdbversion()
+  // to crsql_next_db_version()
   // and re-set on transaction commit or rollback.
   sqlite3_int64 dbVersion;
+  // the version that the db will be set to at the end of the transaction
+  // if that transaction were to commit at the time this value is checked.
+  sqlite3_int64 pendingDbVersion;
   int pragmaSchemaVersion;
 
   // we need another schema version number that tracks when we checked it
@@ -27,9 +31,21 @@ struct crsql_ExtData {
   sqlite3_stmt *pDbVersionStmt;
   crsql_TableInfo **zpTableInfos;
   int tableInfosLen;
+
+  // tracks the number of rows impacted by all inserts into crsql_changes in the
+  // current transaction. This number is reset on transaction commit.
+  int rowsImpacted;
+
+  int seq;
+
+  sqlite3_stmt *pSetSyncBitStmt;
+  sqlite3_stmt *pClearSyncBitStmt;
+  sqlite3_stmt *pSetSiteIdOrdinalStmt;
+  sqlite3_stmt *pSelectSiteIdOrdinalStmt;
+  void *pStmtCache;
 };
 
-crsql_ExtData *crsql_newExtData(sqlite3 *db);
+crsql_ExtData *crsql_newExtData(sqlite3 *db, unsigned char *siteIdBuffer);
 void crsql_freeExtData(crsql_ExtData *pExtData);
 int crsql_fetchPragmaSchemaVersion(sqlite3 *db, crsql_ExtData *pExtData,
                                    int which);

libsql-sqlite3/ext/crr/src/ext-data.test.c

@@ -16,7 +16,9 @@ static void textNewExtData() {
   int rc = SQLITE_OK;
   rc = sqlite3_open(":memory:", &db);
   assert(rc == SQLITE_OK);
-  crsql_ExtData *pExtData = crsql_newExtData(db);
+  unsigned char *siteIdBuffer =
+      sqlite3_malloc(SITE_ID_LEN * sizeof *(siteIdBuffer));
+  crsql_ExtData *pExtData = crsql_newExtData(db, siteIdBuffer);
 
   assert(pExtData->dbVersion == -1);
   // statement used to determine schema version
@@ -52,7 +54,8 @@ static void testFreeExtData() {
   int rc;
   rc = sqlite3_open(":memory:", &db);
   assert(rc == SQLITE_OK);
-  crsql_ExtData *pExtData = crsql_newExtData(db);
+  unsigned char *siteIdBuffer = sqlite3_malloc(SITE_ID_LEN * sizeof(char *));
+  crsql_ExtData *pExtData = crsql_newExtData(db, siteIdBuffer);
 
   crsql_finalize(pExtData);
   crsql_freeExtData(pExtData);
@@ -66,7 +69,8 @@ static void testFinalize() {
   int rc;
   rc = sqlite3_open(":memory:", &db);
   assert(rc == SQLITE_OK);
-  crsql_ExtData *pExtData = crsql_newExtData(db);
+  unsigned char *siteIdBuffer = sqlite3_malloc(SITE_ID_LEN * sizeof(char *));
+  crsql_ExtData *pExtData = crsql_newExtData(db, siteIdBuffer);
 
   crsql_finalize(pExtData);
   assert(pExtData->pDbVersionStmt == 0);
@@ -86,7 +90,8 @@ static void testFetchPragmaSchemaVersion() {
   int didChange = 0;
   rc = sqlite3_open(":memory:", &db);
   assert(rc == SQLITE_OK);
-  crsql_ExtData *pExtData = crsql_newExtData(db);
+  unsigned char *siteIdBuffer = sqlite3_malloc(SITE_ID_LEN * sizeof(char *));
+  crsql_ExtData *pExtData = crsql_newExtData(db, siteIdBuffer);
 
   // fetch the schema info for db version update
   didChange = crsql_fetchPragmaSchemaVersion(db, pExtData, 0);
@@ -153,8 +158,10 @@ static void testFetchPragmaDataVersion() {
   rc = sqlite3_exec(db1, "CREATE TABLE fpdv (a)", 0, 0, &errmsg);
   assert(rc == SQLITE_OK);
 
-  crsql_ExtData *pExtData1 = crsql_newExtData(db1);
-  crsql_ExtData *pExtData2 = crsql_newExtData(db2);
+  unsigned char *siteIdBuffer = sqlite3_malloc(SITE_ID_LEN * sizeof(char *));
+  crsql_ExtData *pExtData1 = crsql_newExtData(db1, siteIdBuffer);
+  siteIdBuffer = sqlite3_malloc(SITE_ID_LEN * sizeof(char *));
+  crsql_ExtData *pExtData2 = crsql_newExtData(db2, siteIdBuffer);
 
   // should not change after init
   rc = crsql_fetchPragmaDataVersion(db1, pExtData1);
@@ -204,7 +211,8 @@ static void testRecreateDbVersionStmt() {
   sqlite3 *db;
   int rc;
   rc = sqlite3_open(":memory:", &db);
-  crsql_ExtData *pExtData = crsql_newExtData(db);
+  unsigned char *siteIdBuffer = sqlite3_malloc(SITE_ID_LEN * sizeof(char *));
+  crsql_ExtData *pExtData = crsql_newExtData(db, siteIdBuffer);
 
   rc = crsql_recreateDbVersionStmt(db, pExtData);
 
@@ -237,7 +245,8 @@ static void fetchDbVersionFromStorage() {
   int rc;
   char *errmsg;
   rc = sqlite3_open(":memory:", &db);
-  crsql_ExtData *pExtData = crsql_newExtData(db);
+  unsigned char *siteIdBuffer = sqlite3_malloc(SITE_ID_LEN * sizeof(char *));
+  crsql_ExtData *pExtData = crsql_newExtData(db, siteIdBuffer);
 
   rc = crsql_fetchDbVersionFromStorage(db, pExtData, &errmsg);
   // no clock tables, no version.

libsql-sqlite3/ext/crr/src/fuzzer.cc

@@ -8,5 +8,6 @@ SQLITE_EXTENSION_INIT3
 
 extern "C" int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size)
 {
+  // Note: all fuzzing is done from Python in `test_sync_prop`
     return 0;
 }

libsql-sqlite3/ext/crr/src/is-crr.test.c

@@ -2,6 +2,7 @@
 #include <stdio.h>
 
 #include "crsqlite.h"
+#include "rust.h"
 
 int crsql_close(sqlite3 *db);
 

libsql-sqlite3/ext/crr/src/rows-impacted.test.c

@@ -0,0 +1,379 @@
+#include <assert.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "crsqlite.h"
+
+int crsql_close(sqlite3 *db);
+
+static sqlite3 *createDb() {
+  int rc = SQLITE_OK;
+  sqlite3 *db;
+  rc = sqlite3_open(":memory:", &db);
+  rc += sqlite3_exec(db, "CREATE TABLE foo (a primary key, b)", 0, 0, 0);
+  rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo')", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+  return db;
+}
+
+static void testSingleInsertSingleTx() {
+  printf("SingleInsertSingleTx\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', "
+                     "2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  // creation + setting of column
+  assert(sqlite3_column_int(pStmt, 0) == 1);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  // rows impacted gets reset
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  int impacted = sqlite3_column_int(pStmt, 0);
+  assert(impacted == 0);
+  sqlite3_finalize(pStmt);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+static void testManyInsertsInATx() {
+  printf("ManyInsertsInATx\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', "
+                     "2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010902', 'b', "
+                     "2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010903', 'b', "
+                     "2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 3);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  // rows impacted gets reset
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 0);
+  sqlite3_finalize(pStmt);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+static void testMultipartInsertInTx() {
+  printf("MultipartInsertInTx\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', "
+                     "2, 1, 1, NULL, 1, 1), "
+                     "('foo', X'010902', 'b', 2, 1, 1, NULL, 1, 1), ('foo', "
+                     "X'010903', 'b', 2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 3);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  // rows impacted gets reset
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 0);
+  sqlite3_finalize(pStmt);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+// count should reset between transactions
+static void testManyTxns() {
+  printf("ManyTxns\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', "
+                     "2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 1);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010902', 'b', "
+                     "2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010903', 'b', "
+                     "2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  int impacted = sqlite3_column_int(pStmt, 0);
+  assert(impacted == 2);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+// You can't do this. `crsql_rows_impacted` is evaulated before the insert is
+// run thus the right value can never be returned by `RETURNING` in this setup.
+// static void testReturningInTx() {
+//   printf("RetruningInTx\n");
+//   int rc = SQLITE_OK;
+//   char *err = 0;
+//   sqlite3 *db = createDb();
+//   sqlite3_stmt *pStmt = 0;
+
+//   rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+//   rc += sqlite3_prepare_v2(
+//       db,
+//       "INSERT INTO crsql_changes VALUES ('foo', 1, 'b', 2, 1, 1, NULL), "
+//       "('foo', 2, 'b', 2, 1, 1, NULL), ('foo', 3, 'b', 2, 1, 1, NULL) "
+//       "RETURNING crsql_rows_impacted()",
+//       -1, &pStmt, 0);
+//   rc += sqlite3_step(pStmt);
+//   int impacted = sqlite3_column_int(pStmt, 0);
+//   printf("impacted: %d\n", impacted);
+//   assert(impacted == 3);
+//   sqlite3_finalize(pStmt);
+//   rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+//   assert(rc == SQLITE_OK);
+
+//   crsql_close(db);
+//   printf("\t\e[0;32mSuccess\e[0m\n");
+// }
+
+static void testUpdateThatDoesNotChangeAnything() {
+  printf("UpdateThatDoesNotChangeAnything\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0);
+
+  rc += sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', "
+                     "crsql_pack_columns(1), 'b', 2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 0);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  // now test value <
+  rc += sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', "
+                     "crsql_pack_columns(1), 'b', 0, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 0);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  // now test clock <
+  rc += sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', "
+                     "crsql_pack_columns(1), 'b', 2, 0, 0, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 0);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+static void testDeleteThatDoesNotChangeAnything() {
+  printf("DeleteThatDoesNotChangeAnything\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0);
+  rc = sqlite3_exec(db, "DELETE FROM foo", 0, 0, 0);
+
+  rc += sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(
+      db,
+      "INSERT INTO crsql_changes VALUES ('foo', crsql_pack_columns(1), "
+      "'-1', NULL, 2, 2, NULL, 1, 1)",  //__crsql_del
+      0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 0);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+static void testDelete() {
+  printf("Delete\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0);
+
+  rc += sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010901', "
+                     "'-1', NULL, 2, 2, NULL, 2, 1)",  //__crsql_del
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 1);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+static void testCreateThatDoesNotChangeAnything() {
+  printf("UpdateThatDoesNotChangeAnything\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0);
+
+  rc += sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', "
+                     "2, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 0);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+static void testValueWin() {
+  printf("ValueWin\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0);
+
+  rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', "
+                     "3, 1, 1, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 1);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+static void testClockWin() {
+  printf("ClockWin\n");
+  int rc = SQLITE_OK;
+  char *err = 0;
+  sqlite3 *db = createDb();
+  sqlite3_stmt *pStmt = 0;
+
+  rc = sqlite3_exec(db, "INSERT INTO foo VALUES (1, 2)", 0, 0, 0);
+
+  rc = sqlite3_exec(db, "BEGIN", 0, 0, 0);
+  rc += sqlite3_exec(db,
+                     "INSERT INTO crsql_changes VALUES ('foo', X'010901', 'b', "
+                     "2, 2, 2, NULL, 1, 1)",
+                     0, 0, &err);
+  sqlite3_prepare_v2(db, "SELECT crsql_rows_impacted()", -1, &pStmt, 0);
+  sqlite3_step(pStmt);
+  assert(sqlite3_column_int(pStmt, 0) == 1);
+  sqlite3_finalize(pStmt);
+  rc += sqlite3_exec(db, "COMMIT", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+void rowsImpactedTestSuite() {
+  printf("\e[47m\e[1;30mSuite: rows_impacted\e[0m\n");
+
+  testSingleInsertSingleTx();
+  testManyInsertsInATx();
+  testMultipartInsertInTx();
+  testManyTxns();
+  testUpdateThatDoesNotChangeAnything();
+  testDeleteThatDoesNotChangeAnything();
+  testCreateThatDoesNotChangeAnything();
+  testValueWin();
+  testClockWin();
+  testDelete();
+}

libsql-sqlite3/ext/crr/src/rust.h

@@ -0,0 +1,31 @@
+#ifndef CRSQLITE_RUST_H
+#define CRSQLITE_RUST_H
+
+#include "crsqlite.h"
+
+// Parts of CR-SQLite are written in Rust and parts are in C.
+// As we gradually convert more code to Rust, we'll have to expose
+// structures to the old C-code that hasn't been converted yet.
+// These are those definitions.
+
+int crsql_backfill_table(sqlite3 *db, const char *tblName,
+                         const char **zpkNames, int pkCount,
+                         const char **zNonPkNames, int nonPkCount,
+                         int isCommitAlter, int noTx);
+int crsql_is_crr(sqlite3 *db, const char *tblName);
+int crsql_compare_sqlite_values(const sqlite3_value *l, const sqlite3_value *r);
+int crsql_create_crr_triggers(sqlite3 *db, crsql_TableInfo *tableInfo,
+                              char **err);
+int crsql_remove_crr_triggers_if_exist(sqlite3 *db, const char *tblName);
+
+char *crsql_changes_union_query(crsql_TableInfo **tableInfos, int tableInfosLen,
+                                const char *idxStr);
+char *crsql_row_patch_data_query(crsql_TableInfo *tblInfo, const char *colName);
+int crsql_create_clock_table(sqlite3 *db, crsql_TableInfo *tableInfo,
+                             char **err);
+int crsql_init_site_id(sqlite3 *db, unsigned char *ret);
+int crsql_init_peer_tracking_table(sqlite3 *db);
+int crsql_create_schema_table_if_not_exists(sqlite3 *db);
+int crsql_maybe_update_db(sqlite3 *db, char **pzErrMsg);
+
+#endif

libsql-sqlite3/ext/crr/src/sandbox.test.c

@@ -0,0 +1,35 @@
+#include <assert.h>
+#include <stdio.h>
+
+#include "crsqlite.h"
+#include "rust.h"
+
+int crsql_close(sqlite3 *db);
+int syncLeftToRight(sqlite3 *db1, sqlite3 *db2, sqlite3_int64 since);
+
+static void testSandbox() {
+  printf("Sandbox\n");
+  sqlite3 *db1;
+  sqlite3 *db2;
+  int rc;
+  rc = sqlite3_open(":memory:", &db1);
+  rc += sqlite3_open(":memory:", &db2);
+
+  rc += sqlite3_exec(db1, "CREATE TABLE foo (a primary key);", 0, 0, 0);
+  rc += sqlite3_exec(db2, "CREATE TABLE foo (a primary key);", 0, 0, 0);
+  rc += sqlite3_exec(db1, "SELECT crsql_as_crr('foo')", 0, 0, 0);
+  rc += sqlite3_exec(db2, "SELECT crsql_as_crr('foo')", 0, 0, 0);
+  rc += sqlite3_exec(db1, "INSERT INTO foo VALUES (1)", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  assert(syncLeftToRight(db1, db2, 0) == SQLITE_OK);
+
+  crsql_close(db1);
+  crsql_close(db2);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
+void crsqlSandboxSuite() {
+  testSandbox();
+  printf("\e[47m\e[1;30mSuite: sandbox\e[0m\n");
+}

libsql-sqlite3/ext/crr/src/tableinfo.c

@@ -10,59 +10,11 @@
 #include "get-table.h"
 #include "util.h"
 
-// Bug here? see crsql_asIdentifierListStr
-char *crsql_asIdentifierList(crsql_ColumnInfo *in, size_t inlen, char *prefix) {
-  if (inlen <= 0) {
-    return 0;
-  }
-
-  char **mapped = sqlite3_malloc(inlen * sizeof(char *));
-  int finalLen = 0;
-  char *ret = 0;
-
-  for (size_t i = 0; i < inlen; ++i) {
-    mapped[i] = sqlite3_mprintf("%s\"%w\"", prefix, in[i].name);
-    finalLen += strlen(mapped[i]);
-  }
-  // -1 for spearator not appended to last thing
-  finalLen += inlen - 1;
-
-  // + 1 for null terminator
-  ret = sqlite3_malloc(finalLen * sizeof(char) + 1);
-  ret[finalLen] = '\0';
-
-  crsql_joinWith(ret, mapped, inlen, ',');
-
-  // free everything we allocated, except ret.
-  // caller will free ret.
-  for (size_t i = 0; i < inlen; ++i) {
-    sqlite3_free(mapped[i]);
-  }
-  sqlite3_free(mapped);
-
-  return ret;
-}
-
 void crsql_freeColumnInfoContents(crsql_ColumnInfo *columnInfo) {
   sqlite3_free(columnInfo->name);
   sqlite3_free(columnInfo->type);
 }
 
-static char *quote(const char *in) {
-  return sqlite3_mprintf("quote(\"%s\")", in);
-}
-
-char *crsql_quoteConcat(crsql_ColumnInfo *cols, int len) {
-  char **names = sqlite3_malloc(len * sizeof(char *));
-  for (int i = 0; i < len; ++i) {
-    names[i] = cols[i].name;
-  }
-
-  char *ret = crsql_join2(&quote, names, len, " || '|' || ");
-  sqlite3_free(names);
-  return ret;
-}
-
 static void crsql_freeColumnInfos(crsql_ColumnInfo *columnInfos, int len) {
   if (columnInfos == 0) {
     return;
@@ -287,6 +239,26 @@ crsql_TableInfo *crsql_findTableInfo(crsql_TableInfo **tblInfos, int len,
   return 0;
 }
 
+int crsql_indexofTableInfo(crsql_TableInfo **tblInfos, int len,
+                           const char *tblName) {
+  for (int i = 0; i < len; ++i) {
+    if (strcmp(tblInfos[i]->tblName, tblName) == 0) {
+      return i;
+    }
+  }
+
+  return -1;
+}
+
+sqlite3_int64 crsql_slabRowid(int idx, sqlite3_int64 rowid) {
+  if (idx < 0) {
+    return -1;
+  }
+
+  sqlite3_int64 modulo = rowid % ROWID_SLAB_SIZE;
+  return idx * ROWID_SLAB_SIZE + modulo;
+}
+
 /**
  * Pulls all table infos for all crrs present in the database.
  * Run once at vtab initialization -- see docs on crsql_Changes_vtab
@@ -403,7 +375,33 @@ int crsql_isTableCompatible(sqlite3 *db, const char *tblName, char **errmsg) {
     return 0;
   }
 
-  // No foreign key constraints
+  // No auto-increment primary keys
+  zSql =
+      "SELECT 1 FROM sqlite_master WHERE name = ? AND type = 'table' AND sql "
+      "LIKE '%autoincrement%' limit 1";
+  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
+
+  rc += sqlite3_bind_text(pStmt, 1, tblName, -1, SQLITE_STATIC);
+  if (rc != SQLITE_OK) {
+    *errmsg = sqlite3_mprintf("Failed to analyze autoincrement status for %s",
+                              tblName);
+    return 0;
+  }
+  rc = sqlite3_step(pStmt);
+  sqlite3_finalize(pStmt);
+  if (rc == SQLITE_ROW) {
+    *errmsg = sqlite3_mprintf(
+        "%s has auto-increment primary keys. This is likely a mistake as two "
+        "concurrent nodes will assign unrelated rows the same primary key. "
+        "Either use a primary key that represents the identity of your row or "
+        "use a database friendly UUID such as UUIDv7",
+        tblName);
+    return 0;
+  } else if (rc != SQLITE_DONE) {
+    return 0;
+  }
+
+  // No checked foreign key constraints
   zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_foreign_key_list('%s')",
                          tblName);
   rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
@@ -421,9 +419,10 @@ int crsql_isTableCompatible(sqlite3 *db, const char *tblName, char **errmsg) {
     sqlite3_finalize(pStmt);
     if (count != 0) {
       *errmsg = sqlite3_mprintf(
-          "Table %s has foreign key constraints. CRRs must not have "
-          "checked "
-          "foreign key constraints as they can be violated by row level "
+          "Table %s has checked foreign key constraints. CRRs may have foreign "
+          "keys but must not have "
+          "checked foreign key constraints as they can be violated by row "
+          "level "
           "security or replication.",
           tblName);
       return 0;
@@ -471,14 +470,3 @@ int crsql_isTableCompatible(sqlite3 *db, const char *tblName, char **errmsg) {
 
   return 1;
 }
-
-int crsql_columnExists(const char *colName, crsql_ColumnInfo *colInfos,
-                       int colInfosLen) {
-  for (int i = 0; i < colInfosLen; ++i) {
-    if (strcmp(colInfos[i].name, colName) == 0) {
-      return 1;
-    }
-  }
-
-  return 0;
-}

libsql-sqlite3/ext/crr/src/tableinfo.h

@@ -7,6 +7,9 @@ SQLITE_EXTENSION_INIT3
 #include <ctype.h>
 #include <stddef.h>
 
+// 10 trillion = 10,000,000,000,000
+#define ROWID_SLAB_SIZE 10000000000000
+
 typedef struct crsql_ColumnInfo crsql_ColumnInfo;
 struct crsql_ColumnInfo {
   int cid;
@@ -37,19 +40,18 @@ crsql_ColumnInfo *crsql_extractBaseCols(crsql_ColumnInfo *colInfos,
 void crsql_freeColumnInfoContents(crsql_ColumnInfo *columnInfo);
 void crsql_freeTableInfo(crsql_TableInfo *tableInfo);
 
+// TODO: this should be pullTableInfo
 int crsql_getTableInfo(sqlite3 *db, const char *tblName,
                        crsql_TableInfo **pTableInfo, char **pErrMsg);
 
-char *crsql_asIdentifierList(crsql_ColumnInfo *in, size_t inlen, char *prefix);
-
 void crsql_freeAllTableInfos(crsql_TableInfo **tableInfos, int len);
 crsql_TableInfo *crsql_findTableInfo(crsql_TableInfo **tblInfos, int len,
                                      const char *tblName);
-char *crsql_quoteConcat(crsql_ColumnInfo *cols, int len);
+int crsql_indexofTableInfo(crsql_TableInfo **tblInfos, int len,
+                           const char *tblName);
+sqlite3_int64 crsql_slabRowid(int idx, sqlite3_int64 rowid);
 int crsql_pullAllTableInfos(sqlite3 *db, crsql_TableInfo ***pzpTableInfos,
                             int *rTableInfosLen, char **errmsg);
 int crsql_isTableCompatible(sqlite3 *db, const char *tblName, char **errmsg);
-int crsql_columnExists(const char *colName, crsql_ColumnInfo *colInfos,
-                       int colInfosLen);
 
 #endif

libsql-sqlite3/ext/crr/src/tableinfo.test.c

@@ -76,34 +76,34 @@ static void testGetTableInfo() {
   crsql_close(db);
 }
 
-static void testAsIdentifierList() {
-  printf("AsIdentifierList\n");
+// static void testAsIdentifierList() {
+//   printf("AsIdentifierList\n");
 
-  crsql_ColumnInfo tc1[3];
-  tc1[0].name = "one";
-  tc1[1].name = "two";
-  tc1[2].name = "three";
+//   crsql_ColumnInfo tc1[3];
+//   tc1[0].name = "one";
+//   tc1[1].name = "two";
+//   tc1[2].name = "three";
 
-  crsql_ColumnInfo tc2[0];
+//   crsql_ColumnInfo tc2[0];
 
-  crsql_ColumnInfo tc3[1];
-  tc3[0].name = "one";
-  char *result;
+//   crsql_ColumnInfo tc3[1];
+//   tc3[0].name = "one";
+//   char *result;
 
-  result = crsql_asIdentifierList(tc1, 3, 0);
-  assert(strcmp(result, "\"one\",\"two\",\"three\"") == 0);
-  sqlite3_free(result);
+//   result = crsql_asIdentifierList(tc1, 3, 0);
+//   assert(strcmp(result, "\"one\",\"two\",\"three\"") == 0);
+//   sqlite3_free(result);
 
-  result = crsql_asIdentifierList(tc2, 0, 0);
-  assert(result == 0);
-  sqlite3_free(result);
+//   result = crsql_asIdentifierList(tc2, 0, 0);
+//   assert(result == 0);
+//   sqlite3_free(result);
 
-  result = crsql_asIdentifierList(tc3, 1, 0);
-  assert(strcmp(result, "\"one\"") == 0);
-  sqlite3_free(result);
+//   result = crsql_asIdentifierList(tc3, 1, 0);
+//   assert(strcmp(result, "\"one\"") == 0);
+//   sqlite3_free(result);
 
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
+//   printf("\t\e[0;32mSuccess\e[0m\n");
+// }
 
 static void testFindTableInfo() {
   printf("FindTableInfo\n");
@@ -128,26 +128,6 @@ static void testFindTableInfo() {
   printf("\t\e[0;32mSuccess\e[0m\n");
 }
 
-static void testQuoteConcat() {
-  printf("QuoteConcat\n");
-
-  int len = 3;
-  crsql_ColumnInfo colInfos[3];
-
-  colInfos[0].name = "a";
-  colInfos[1].name = "b";
-  colInfos[2].name = "c";
-
-  char *quoted = crsql_quoteConcat(colInfos, len);
-
-  assert(strcmp(quoted,
-                "quote(\"a\") || '|' || quote(\"b\") || '|' || quote(\"c\")") ==
-         0);
-
-  sqlite3_free(quoted);
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
-
 static void testIsTableCompatible() {
   printf("IsTableCompatible\n");
   sqlite3 *db = 0;
@@ -215,6 +195,20 @@ static void testIsTableCompatible() {
   rc = crsql_isTableCompatible(db, "atable", &errmsg);
   assert(rc == 1);
 
+  // no autoincrement
+  rc = sqlite3_exec(
+      db, "CREATE TABLE woom (a integer primary key autoincrement)", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+  rc = crsql_isTableCompatible(db, "woom", &errmsg);
+  assert(rc == 0);
+  sqlite3_free(errmsg);
+
+  // aliased rowid
+  rc = sqlite3_exec(db, "CREATE TABLE loom (a integer primary key)", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+  rc = crsql_isTableCompatible(db, "loom", &errmsg);
+  assert(rc == 1);
+
   rc = sqlite3_exec(
       db, "CREATE TABLE atable2 (\"id\" TEXT PRIMARY KEY, x TEXT) STRICT;", 0,
       0, 0);
@@ -238,13 +232,64 @@ static void testIsTableCompatible() {
   crsql_close(db);
 }
 
+static void testSlabRowid() {
+  printf("SlabRowid\n");
+  sqlite3 *db = 0;
+  char *errmsg = 0;
+  int rc = SQLITE_OK;
+
+  rc = sqlite3_open(":memory:", &db);
+  rc += sqlite3_exec(db, "CREATE TABLE foo (a PRIMARY KEY)", 0, 0, 0);
+  rc += sqlite3_exec(db, "CREATE TABLE bar (a PRIMARY KEY)", 0, 0, 0);
+  rc += sqlite3_exec(db, "CREATE TABLE baz (a PRIMARY KEY)", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  rc += sqlite3_exec(db, "SELECT crsql_as_crr('foo');", 0, 0, 0);
+  rc += sqlite3_exec(db, "SELECT crsql_as_crr('bar');", 0, 0, 0);
+  rc += sqlite3_exec(db, "SELECT crsql_as_crr('baz');", 0, 0, 0);
+  assert(rc == SQLITE_OK);
+
+  // now pull all table infos
+  crsql_TableInfo **tblInfos = 0;
+  int tblInfosLen = 0;
+  rc = crsql_pullAllTableInfos(db, &tblInfos, &tblInfosLen, &errmsg);
+  assert(rc == SQLITE_OK);
+
+  // now get the slab rowid for each table
+  sqlite3_int64 fooSlabRowid = crsql_slabRowid(0, 1);
+  sqlite3_int64 barSlabRowid = crsql_slabRowid(1, 2);
+  sqlite3_int64 bazSlabRowid = crsql_slabRowid(2, 3);
+
+  // now assert each one
+  assert(fooSlabRowid == 1);
+  assert(barSlabRowid == 2 + ROWID_SLAB_SIZE);
+  assert(bazSlabRowid == 3 + ROWID_SLAB_SIZE * 2);
+
+  // now test the modulo
+  assert(crsql_slabRowid(0, ROWID_SLAB_SIZE) == 0);
+  assert(crsql_slabRowid(0, ROWID_SLAB_SIZE + 1) == 1);
+
+  fooSlabRowid = crsql_slabRowid(0, ROWID_SLAB_SIZE + 1);
+  barSlabRowid = crsql_slabRowid(1, ROWID_SLAB_SIZE + 2);
+  bazSlabRowid = crsql_slabRowid(2, ROWID_SLAB_SIZE * 2 + 3);
+
+  assert(fooSlabRowid == 1);
+  assert(barSlabRowid == 2 + ROWID_SLAB_SIZE);
+  assert(bazSlabRowid == 3 + ROWID_SLAB_SIZE * 2);
+
+  crsql_freeAllTableInfos(tblInfos, tblInfosLen);
+
+  crsql_close(db);
+  printf("\t\e[0;32mSuccess\e[0m\n");
+}
+
 void crsqlTableInfoTestSuite() {
   printf("\e[47m\e[1;30mSuite: crsql_tableInfo\e[0m\n");
 
-  testAsIdentifierList();
+  // testAsIdentifierList();
   testGetTableInfo();
   testFindTableInfo();
-  testQuoteConcat();
   testIsTableCompatible();
+  testSlabRowid();
   // testPullAllTableInfos();
 }

libsql-sqlite3/ext/crr/src/tests.c

@@ -19,11 +19,13 @@ void crsqlTestSuite();
 void crsqlTriggersTestSuite();
 void crsqlChangesVtabReadTestSuite();
 void crsqlChangesVtabTestSuite();
-void crsqlChangesVtabWriteTestSuite();
 void crsqlChangesVtabCommonTestSuite();
 void crsqlExtDataTestSuite();
 void crsqlFractSuite();
 void crsqlIsCrrTestSuite();
+void rowsImpactedTestSuite();
+void crsqlChangesVtabRowidTestSuite();
+void crsqlSandboxSuite();
 
 int main(int argc, char *argv[]) {
   char *suite = "all";
@@ -36,14 +38,15 @@ int main(int argc, char *argv[]) {
   SUITE("triggers") crsqlTriggersTestSuite();
   SUITE("vtab") crsqlChangesVtabTestSuite();
   SUITE("vtabread") crsqlChangesVtabReadTestSuite();
-  SUITE("vtabwrite") crsqlChangesVtabWriteTestSuite();
-  SUITE("vtabcommon") crsqlChangesVtabCommonTestSuite();
   SUITE("extdata") crsqlExtDataTestSuite();
   // integration tests should come at the end given fixing unit tests will
   // likely fix integration tests
   SUITE("crsql") crsqlTestSuite();
   SUITE("fract") crsqlFractSuite();
   SUITE("is_crr") crsqlIsCrrTestSuite();
+  SUITE("rows_impacted") rowsImpactedTestSuite();
+  SUITE("rowid") crsqlChangesVtabRowidTestSuite();
+  SUITE("sandbox") crsqlSandboxSuite();
 
   sqlite3_shutdown();
 }

libsql-sqlite3/ext/crr/src/triggers.c

@@ -1,263 +0,0 @@
-#include "triggers.h"
-
-#include <stdint.h>
-#include <string.h>
-
-#include "consts.h"
-#include "tableinfo.h"
-#include "util.h"
-
-int crsql_createInsertTrigger(sqlite3 *db, crsql_TableInfo *tableInfo,
-                              char **err) {
-  char *zSql;
-  char *pkList = 0;
-  char *pkNewList = 0;
-  int rc = SQLITE_OK;
-  char *joinedSubTriggers;
-
-  pkList = crsql_asIdentifierList(tableInfo->pks, tableInfo->pksLen, 0);
-  pkNewList = crsql_asIdentifierList(tableInfo->pks, tableInfo->pksLen, "NEW.");
-
-  joinedSubTriggers = crsql_insertTriggerQuery(tableInfo, pkList, pkNewList);
-
-  zSql = sqlite3_mprintf(
-      "CREATE TRIGGER IF NOT EXISTS \"%s__crsql_itrig\"\
-      AFTER INSERT ON \"%s\"\
-    BEGIN\
-      %s\
-    END;",
-      tableInfo->tblName, tableInfo->tblName, joinedSubTriggers);
-
-  sqlite3_free(joinedSubTriggers);
-
-  rc = sqlite3_exec(db, zSql, 0, 0, err);
-  sqlite3_free(zSql);
-
-  sqlite3_free(pkList);
-  sqlite3_free(pkNewList);
-
-  return rc;
-}
-
-char *crsql_insertTriggerQuery(crsql_TableInfo *tableInfo, char *pkList,
-                               char *pkNewList) {
-  const int length = tableInfo->nonPksLen == 0 ? 1 : tableInfo->nonPksLen;
-  char **subTriggers = sqlite3_malloc(length * sizeof(char *));
-  char *joinedSubTriggers;
-
-  // We need a CREATE_SENTINEL to stand in for the create event so we can
-  // replicate PKs If we have a create sentinel how will we insert the created
-  // rows without a requirement of nullability on every column? Keep some
-  // event data for create that represents the initial state of the row?
-  // Future improvement.
-  if (tableInfo->nonPksLen == 0) {
-    subTriggers[0] = sqlite3_mprintf(
-        "INSERT INTO \"%s__crsql_clock\" (\
-        %s,\
-        __crsql_col_name,\
-        __crsql_col_version,\
-        __crsql_db_version,\
-        __crsql_site_id\
-      ) SELECT \
-        %s,\
-        %Q,\
-        1,\
-        crsql_nextdbversion(),\
-        NULL\
-      WHERE crsql_internal_sync_bit() = 0 ON CONFLICT DO UPDATE SET\
-        __crsql_col_version = __crsql_col_version + 1,\
-        __crsql_db_version = crsql_nextdbversion(),\
-        __crsql_site_id = NULL;\n",
-        tableInfo->tblName, pkList, pkNewList, PKS_ONLY_CID_SENTINEL);
-  }
-  for (int i = 0; i < tableInfo->nonPksLen; ++i) {
-    subTriggers[i] = sqlite3_mprintf(
-        "INSERT INTO \"%s__crsql_clock\" (\
-        %s,\
-        __crsql_col_name,\
-        __crsql_col_version,\
-        __crsql_db_version,\
-        __crsql_site_id\
-      ) SELECT \
-        %s,\
-        %Q,\
-        1,\
-        crsql_nextdbversion(),\
-        NULL\
-      WHERE crsql_internal_sync_bit() = 0 ON CONFLICT DO UPDATE SET\
-        __crsql_col_version = __crsql_col_version + 1,\
-        __crsql_db_version = crsql_nextdbversion(),\
-        __crsql_site_id = NULL;\n",
-        tableInfo->tblName, pkList, pkNewList, tableInfo->nonPks[i].name);
-  }
-
-  joinedSubTriggers = crsql_join(subTriggers, length);
-
-  for (int i = 0; i < length; ++i) {
-    sqlite3_free(subTriggers[i]);
-  }
-  sqlite3_free(subTriggers);
-
-  return joinedSubTriggers;
-}
-
-// TODO (#50): we need to handle the case where someone _changes_ a primary key
-// column's value we should:
-// 1. detect this
-// 2. treat _every_ column as updated
-// 3. write a delete sentinel against the _old_ pk combination
-//
-// 1 is moot.
-// 2 is done via changing trigger conditions to: `WHERE sync_bit = 0 AND (NEW.c
-// != OLD.c OR NEW.pk_c1 != OLD.pk_c1 OR NEW.pk_c2 != ...) 3 is done with a new
-// trigger based on only pks
-int crsql_createUpdateTrigger(sqlite3 *db, crsql_TableInfo *tableInfo,
-                              char **err) {
-  char *zSql;
-  char *pkList = 0;
-  char *pkNewList = 0;
-  int rc = SQLITE_OK;
-  const int length = tableInfo->nonPksLen == 0 ? 1 : tableInfo->nonPksLen;
-  char **subTriggers = sqlite3_malloc(length * sizeof(char *));
-  char *joinedSubTriggers;
-
-  pkList = crsql_asIdentifierList(tableInfo->pks, tableInfo->pksLen, 0);
-  pkNewList = crsql_asIdentifierList(tableInfo->pks, tableInfo->pksLen, "NEW.");
-
-  // If we updated a table that _only_ has primary key columns
-  // this is the same thing as
-  // a
-  // 1. delete of the old row
-  // followed by
-  // 2. create of a new row
-  // SQLite already calls the delete trigger for the old row
-  // for case 1 so that's covered.
-  //
-  // TODO: Do we not also need to record a creation event
-  // if a pk was changed for a non pk only table?
-  if (tableInfo->nonPksLen == 0) {
-    subTriggers[0] = sqlite3_mprintf(
-        "INSERT INTO \"%s__crsql_clock\" (\
-        %s,\
-        __crsql_col_name,\
-        __crsql_col_version,\
-        __crsql_db_version,\
-        __crsql_site_id\
-      ) SELECT \
-        %s,\
-        %Q,\
-        1,\
-        crsql_nextdbversion(),\
-        NULL\
-      WHERE crsql_internal_sync_bit() = 0 ON CONFLICT DO UPDATE SET\
-        __crsql_col_version = __crsql_col_version + 1,\
-        __crsql_db_version = crsql_nextdbversion(),\
-        __crsql_site_id = NULL;\n",
-        tableInfo->tblName, pkList, pkNewList, PKS_ONLY_CID_SENTINEL);
-  }
-
-  for (int i = 0; i < tableInfo->nonPksLen; ++i) {
-    // updates are conditionally inserted on the new value not being
-    // the same as the old value.
-    subTriggers[i] = sqlite3_mprintf(
-        "INSERT INTO \"%s__crsql_clock\" (\
-        %s,\
-        __crsql_col_name,\
-        __crsql_col_version,\
-        __crsql_db_version,\
-        __crsql_site_id\
-      ) SELECT %s, %Q, 1, crsql_nextdbversion(), NULL WHERE crsql_internal_sync_bit() = 0 AND NEW.\"%w\" != OLD.\"%w\"\
-      ON CONFLICT DO UPDATE SET\
-        __crsql_col_version = __crsql_col_version + 1,\
-        __crsql_db_version = crsql_nextdbversion(),\
-        __crsql_site_id = NULL;\n",
-        tableInfo->tblName, pkList, pkNewList, tableInfo->nonPks[i].name,
-        tableInfo->nonPks[i].name, tableInfo->nonPks[i].name);
-  }
-  joinedSubTriggers = crsql_join(subTriggers, length);
-
-  for (int i = 0; i < length; ++i) {
-    sqlite3_free(subTriggers[i]);
-  }
-  sqlite3_free(subTriggers);
-
-  zSql = sqlite3_mprintf(
-      "CREATE TRIGGER IF NOT EXISTS \"%s__crsql_utrig\"\
-      AFTER UPDATE ON \"%s\"\
-    BEGIN\
-      %s\
-    END;",
-      tableInfo->tblName, tableInfo->tblName, joinedSubTriggers);
-
-  sqlite3_free(joinedSubTriggers);
-
-  rc = sqlite3_exec(db, zSql, 0, 0, err);
-  sqlite3_free(zSql);
-
-  sqlite3_free(pkList);
-  sqlite3_free(pkNewList);
-
-  return rc;
-}
-
-char *crsql_deleteTriggerQuery(crsql_TableInfo *tableInfo) {
-  char *zSql;
-  char *pkList = 0;
-  char *pkOldList = 0;
-
-  pkList = crsql_asIdentifierList(tableInfo->pks, tableInfo->pksLen, 0);
-  pkOldList = crsql_asIdentifierList(tableInfo->pks, tableInfo->pksLen, "OLD.");
-
-  zSql = sqlite3_mprintf(
-      "CREATE TRIGGER IF NOT EXISTS \"%s__crsql_dtrig\"\
-      AFTER DELETE ON \"%s\"\
-    BEGIN\
-      INSERT INTO \"%s__crsql_clock\" (\
-        %s,\
-        __crsql_col_name,\
-        __crsql_col_version,\
-        __crsql_db_version,\
-        __crsql_site_id\
-      ) SELECT \
-        %s,\
-        %Q,\
-        1,\
-        crsql_nextdbversion(),\
-        NULL\
-      WHERE crsql_internal_sync_bit() = 0 ON CONFLICT DO UPDATE SET\
-      __crsql_col_version = __crsql_col_version + 1,\
-      __crsql_db_version = crsql_nextdbversion(),\
-      __crsql_site_id = NULL;\
-      END; ",
-      tableInfo->tblName, tableInfo->tblName, tableInfo->tblName, pkList,
-      pkOldList, DELETE_CID_SENTINEL);
-
-  sqlite3_free(pkList);
-  sqlite3_free(pkOldList);
-
-  return zSql;
-}
-
-int crsql_createDeleteTrigger(sqlite3 *db, crsql_TableInfo *tableInfo,
-                              char **err) {
-  int rc = SQLITE_OK;
-
-  char *zSql = crsql_deleteTriggerQuery(tableInfo);
-  rc = sqlite3_exec(db, zSql, 0, 0, err);
-  sqlite3_free(zSql);
-
-  return rc;
-}
-
-int crsql_createCrrTriggers(sqlite3 *db, crsql_TableInfo *tableInfo,
-                            char **err) {
-  int rc = crsql_createInsertTrigger(db, tableInfo, err);
-  if (rc == SQLITE_OK) {
-    rc = crsql_createUpdateTrigger(db, tableInfo, err);
-  }
-  if (rc == SQLITE_OK) {
-    rc = crsql_createDeleteTrigger(db, tableInfo, err);
-  }
-
-  return rc;
-}

libsql-sqlite3/ext/crr/src/triggers.h

@@ -1,25 +0,0 @@
-#ifndef CRSQLITE_TRIGGERS_H
-#define CRSQLITE_TRIGGERS_H
-
-#include <ctype.h>
-
-#include "crsqlite.h"
-
-int crsql_createCrrTriggers(sqlite3 *db, crsql_TableInfo *tableInfo,
-                            char **err);
-
-int crsql_createInsertTrigger(sqlite3 *db, crsql_TableInfo *tableInfo,
-                              char **err);
-
-int crsql_createUpdateTrigger(sqlite3 *db, crsql_TableInfo *tableInfo,
-                              char **err);
-
-int crsql_createDeleteTrigger(sqlite3 *db, crsql_TableInfo *tableInfo,
-                              char **err);
-char *crsql_deleteTriggerQuery(crsql_TableInfo *tableInfo);
-
-char *crsql_insertTriggerQuery(crsql_TableInfo *tableInfo, char *pkList,
-                               char *pkNewList);
-int crsql_remove_crr_triggers_if_exist(sqlite3 *db, const char *tblName);
-
-#endif

libsql-sqlite3/ext/crr/src/triggers.test.c

@@ -1,5 +1,3 @@
-#include "triggers.h"
-
 #include <assert.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -7,6 +5,7 @@
 
 #include "consts.h"
 #include "crsqlite.h"
+#include "rust.h"
 #include "tableinfo.h"
 #include "util.h"
 
@@ -30,13 +29,7 @@ static void testCreateTriggers() {
   rc = crsql_getTableInfo(db, "foo", &tableInfo, &errMsg);
 
   if (rc == SQLITE_OK) {
-    rc = crsql_createInsertTrigger(db, tableInfo, &errMsg);
-  }
-  if (rc == SQLITE_OK) {
-    rc = crsql_createUpdateTrigger(db, tableInfo, &errMsg);
-  }
-  if (rc == SQLITE_OK) {
-    rc = crsql_createDeleteTrigger(db, tableInfo, &errMsg);
+    rc = crsql_create_crr_triggers(db, tableInfo, &errMsg);
   }
 
   crsql_freeTableInfo(tableInfo);
@@ -53,77 +46,9 @@ static void testCreateTriggers() {
   printf("\t\e[0;32mSuccess\e[0m\n");
 }
 
-static void testDeleteTriggerQuery() {
-  printf("DeleteTriggerQuery\n");
-  sqlite3 *db = 0;
-  crsql_TableInfo *tableInfo;
-  char *errMsg = 0;
-  int rc = sqlite3_open(":memory:", &db);
-
-  rc +=
-      sqlite3_exec(db, "CREATE TABLE \"foo\" (\"a\" PRIMARY KEY, \"b\", \"c\")",
-                   0, 0, &errMsg);
-  rc += crsql_getTableInfo(db, "foo", &tableInfo, &errMsg);
-  rc += sqlite3_exec(db, "DROP TABLE foo", 0, 0, &errMsg);
-
-  char *query = crsql_deleteTriggerQuery(tableInfo);
-  assert(strcmp("CREATE TRIGGER IF NOT EXISTS \"foo__crsql_dtrig\"      AFTER "
-                "DELETE ON \"foo\"    BEGIN      INSERT INTO "
-                "\"foo__crsql_clock\" (        \"a\",        __crsql_col_name, "
-                "       __crsql_col_version,        __crsql_db_version,        "
-                "__crsql_site_id      ) SELECT         OLD.\"a\",        "
-                "\'__crsql_del\',        1,        crsql_nextdbversion(),      "
-                "  NULL      WHERE crsql_internal_sync_bit() = 0 ON CONFLICT "
-                "DO UPDATE SET      __crsql_col_version = __crsql_col_version "
-                "+ 1,      __crsql_db_version = crsql_nextdbversion(),      "
-                "__crsql_site_id = NULL;      END; ",
-                query) == 0);
-
-  crsql_freeTableInfo(tableInfo);
-  crsql_close(db);
-  sqlite3_free(query);
-  assert(rc == SQLITE_OK);
-
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
-
-static void testInsertTriggerQuery() {
-  printf("InsertTriggerQuery\n");
-  sqlite3 *db = 0;
-  crsql_TableInfo *tableInfo;
-  char *errMsg = 0;
-  int rc = sqlite3_open(":memory:", &db);
-
-  rc += sqlite3_exec(
-      db,
-      "CREATE TABLE \"foo\" (\"a\", \"b\", \"c\", PRIMARY KEY (\"a\", \"b\"))",
-      0, 0, &errMsg);
-  rc += crsql_getTableInfo(db, "foo", &tableInfo, &errMsg);
-  assert(rc == SQLITE_OK);
-
-  char *query = crsql_insertTriggerQuery(tableInfo, "a, b", "NEW.a, NEW.b");
-  char *expected =
-      "INSERT INTO \"foo__crsql_clock\" (        a, b,        "
-      "__crsql_col_name,        __crsql_col_version,        "
-      "__crsql_db_version,        __crsql_site_id      ) SELECT         NEW.a, "
-      "NEW.b,        \'c\',        1,        crsql_nextdbversion(),        "
-      "NULL      WHERE crsql_internal_sync_bit() = 0 ON CONFLICT DO UPDATE SET "
-      "       __crsql_col_version = __crsql_col_version + 1,        "
-      "__crsql_db_version = crsql_nextdbversion(),        __crsql_site_id = "
-      "NULL;\n";
-
-  assert(strcmp(expected, query) == 0);
-
-  crsql_freeTableInfo(tableInfo);
-  crsql_close(db);
-  sqlite3_free(query);
-}
-
 void crsqlTriggersTestSuite() {
   printf("\e[47m\e[1;30mSuite: crsqlTriggers\e[0m\n");
 
-  testDeleteTriggerQuery();
   testCreateTriggers();
-  testInsertTriggerQuery();
-  // testTriggerSyncBitInteraction();
+  // testTriggerSyncBitInteraction(); <-- implemented in rust tests
 }

libsql-sqlite3/ext/crr/src/util.c

@@ -13,6 +13,8 @@ size_t crsql_strnlen(const char *s, size_t n) {
   return p ? p - s : n;
 }
 
+// TODO: I don't think we need these crsql_ specific ones anymore now that we've
+// set the allocator symbol in the WASM builds
 char *crsql_strndup(const char *s, size_t n) {
   size_t l = crsql_strnlen(s, n);
   char *d = sqlite3_malloc(l + 1);
@@ -106,188 +108,6 @@ char *crsql_join2(char *(*map)(const char *), char **in, size_t len,
   return ret;
 }
 
-/**
- * Given a pointer to the inside of a string literal,
- * scan until we get to the end of the literal.
- *
- * Fails if we hit the end of the string without finding an unescaped
- * literal termination.
- */
-const char *crsql_scanToEndOfLiteral(const char *in) {
-  while (*in != '\0') {
-    // hit a quote
-    if (*in == '\'') {
-      // with no quote after?
-      // it is unescaped
-      if (*(in + 1) != '\'') {
-        return (in + 1);
-      } else {
-        // was a quote after? move into that quote
-        // then past the quote at end of loop.
-        in += 1;
-      }
-    }
-    in += 1;
-  }
-
-  // we made it to the end of the string? well then
-  // there was a bare quote which is an error case.
-  return 0;
-}
-
-/**
- * Advance len characters through the string.
- * Fails (returns 0) if we cannot advance at least that much.
- */
-const char *crsql_safelyAdvanceThroughString(const char *in, int len) {
-  for (int i = 0; i < len; ++i) {
-    if (*in == '\0') {
-      return 0;
-    }
-
-    in += 1;
-  }
-  return in;
-}
-
-/**
- * Looks for the provided delimiter and returns a pointer to the character
- * after that delim.
- *
- * The technically allows many `e` and `.` characters in a number.
- * We can improve this to only allow them where they should occur but
- * this is good enough for our purposes of ensuring safe input.
- *
- * If no delim is found, returns a pointer to the end of the string.
- */
-const char *crsql_consumeDigitsToDelimiter(const char *in, char delim) {
-  int decimalCount = 0;
-  int exponentCount = 0;
-  while (*in != '\0' && *in != delim) {
-    if (*in < 48 || *in > 57) {
-      if (*in == '.') {
-        if (decimalCount > 0) {
-          return 0;
-        }
-        ++decimalCount;
-      } else if (*in == 'e') {
-        if (exponentCount > 0) {
-          return 0;
-        }
-        ++exponentCount;
-        if (*(in + 1) == '-' || *(in + 1) == '+') {
-          in += 1;
-        }
-      } else {
-        return 0;
-      }
-    }
-    in += 1;
-  }
-  return in;
-}
-
-char **crsql_splitQuoteConcat(const char *in, int partsLen) {
-  const char *curr = in;
-  const char *last = in;
-  char **zzParts = sqlite3_malloc(partsLen * sizeof(char *));
-  int partIdx = 0;
-  while (curr != 0 && *curr != '\0' && partIdx < partsLen) {
-    if (*curr == '\'') {
-      // scan till consumed string literal
-      curr += 1;
-      curr = crsql_scanToEndOfLiteral(curr);
-    } else if (*curr == 'X') {
-      // scan till consumed hex
-      curr += 1;
-      if (*curr != '\'') {
-        // unexpected result
-        // set curr = 0 so we can cleanup and exit
-        curr = 0;
-      } else {
-        curr += 1;
-        curr = crsql_scanToEndOfLiteral(curr);
-      }
-    } else if (*curr == 'N') {
-      // scan till consumed NULL
-      curr = crsql_safelyAdvanceThroughString(curr, 4);
-    } else {
-      if (*curr == '-') {
-        curr += 1;
-      }
-      // scan till we hit the delimiter, consuming the digits
-      curr = crsql_consumeDigitsToDelimiter(curr, QC_DELIM);
-    }
-
-    if (curr == 0) {
-      // we had an error in scanning
-      // free what we've allocated thus far and return null.
-      for (int i = 0; i < partIdx; ++i) {
-        sqlite3_free(zzParts[i]);
-      }
-      sqlite3_free(zzParts);
-      return 0;
-    }
-
-    // pull from last to curr
-    // advance last
-    zzParts[partIdx] = sqlite3_malloc(((curr - last) + 1) * sizeof(char *));
-    strncpy(zzParts[partIdx], last, curr - last);
-    zzParts[partIdx][curr - last] = '\0';
-
-    // pointing at a delim? Move off it.
-    if (*curr == QC_DELIM) {
-      curr += 1;
-    }
-
-    last = curr;
-    partIdx += 1;
-  }
-
-  if (partIdx != partsLen || *curr != '\0') {
-    // we did not consume the whole string or get the number of parts
-    // expected. this is an error case.
-    for (int i = 0; i < partIdx; ++i) {
-      sqlite3_free(zzParts[i]);
-    }
-    sqlite3_free(zzParts);
-    return 0;
-  }
-
-  return zzParts;
-}
-
-// TODO:
-// have this take a function pointer that extracts the string so we can
-// delete crsql_asIdentifierList
-char *crsql_asIdentifierListStr(char **in, size_t inlen, char delim) {
-  int finalLen = 0;
-  char *ret = 0;
-  char **mapped = sqlite3_malloc(inlen * sizeof(char *));
-
-  for (size_t i = 0; i < inlen; ++i) {
-    mapped[i] = sqlite3_mprintf("\"%w\"", in[i]);
-    finalLen += strlen(mapped[i]);
-  }
-  // -1 for spearator not appended to last thing
-  finalLen += inlen - 1;
-
-  // + 1 for null terminator
-  ret = sqlite3_malloc(finalLen * sizeof(char) + 1);
-  ret[finalLen] = '\0';
-
-  crsql_joinWith(ret, mapped, inlen, delim);
-
-  // free everything we allocated, except ret.
-  // caller will free ret.
-  for (size_t i = 0; i < inlen; ++i) {
-    sqlite3_free(mapped[i]);
-  }
-  sqlite3_free(mapped);
-
-  return ret;
-}
-
 /**
  * @brief Given a list of clock table names, construct a union query to get the
  * max clock value for our site.
@@ -311,7 +131,7 @@ char *crsql_getDbVersionUnionQuery(int numRows, char **tableNames) {
         // so skip that
         tableNames[i + 1],
         // If we have more tables to process, union them in
-        i < numRows - 1 ? UNION : "");
+        i < numRows - 1 ? UNION_ALL : "");
   }
 
   // move the array of strings into a single string
@@ -324,32 +144,15 @@ char *crsql_getDbVersionUnionQuery(int numRows, char **tableNames) {
 
   // compose the final query
   // and update the pointer to the string to point to it.
-  ret = sqlite3_mprintf("SELECT max(version) as version FROM (%z)", unionsStr);
+  ret = sqlite3_mprintf(
+      "SELECT max(version) as version FROM (%z UNION SELECT value as "
+      "version "
+      "FROM crsql_master WHERE key = 'pre_compact_dbversion')",
+      unionsStr);
   // %z frees unionsStr https://www.sqlite.org/printf.html#percentz
   return ret;
 }
 
-/**
- * Check if tblName exists.
- * Caller is responsible for freeing tblName.
- *
- * Returns -1 on error.
- */
-int crsql_doesTableExist(sqlite3 *db, const char *tblName) {
-  char *zSql;
-  int ret = 0;
-
-  zSql = sqlite3_mprintf(
-      "SELECT count(*) as c FROM sqlite_master WHERE type='table' AND "
-      "tbl_name "
-      "= '%s'",
-      tblName);
-  ret = crsql_getCount(db, zSql);
-  sqlite3_free(zSql);
-
-  return ret;
-}
-
 int crsql_getCount(sqlite3 *db, char *zSql) {
   int rc = SQLITE_OK;
   int count = 0;
@@ -372,103 +175,3 @@ int crsql_getCount(sqlite3 *db, char *zSql) {
 
   return count;
 }
-
-/**
- * Given an index name, return all the columns in that index.
- * Fills pIndexedCols with an array of strings.
- * Caller is responsible for freeing pIndexedCols.
- */
-int crsql_getIndexedCols(sqlite3 *db, const char *indexName,
-                         char ***pIndexedCols, int *pIndexedColsLen,
-                         char **pErrMsg) {
-  int rc = SQLITE_OK;
-  int numCols = 0;
-  char **indexedCols;
-  sqlite3_stmt *pStmt = 0;
-  *pIndexedCols = 0;
-  *pIndexedColsLen = 0;
-
-  char *zSql = sqlite3_mprintf("SELECT count(*) FROM pragma_index_info('%s')",
-                               indexName);
-  numCols = crsql_getCount(db, zSql);
-  sqlite3_free(zSql);
-
-  if (numCols <= 0) {
-    return numCols;
-  }
-
-  zSql = sqlite3_mprintf(
-      "SELECT \"name\" FROM pragma_index_info('%s') ORDER BY \"seqno\" ASC",
-      indexName);
-  rc = sqlite3_prepare_v2(db, zSql, -1, &pStmt, 0);
-  sqlite3_free(zSql);
-  if (rc != SQLITE_OK) {
-    *pErrMsg = sqlite3_mprintf("Failed preparing pragma_index_info('%s') stmt",
-                               indexName);
-    sqlite3_finalize(pStmt);
-    return rc;
-  }
-
-  rc = sqlite3_step(pStmt);
-  if (rc != SQLITE_ROW) {
-    sqlite3_finalize(pStmt);
-    return SQLITE_OK;
-  }
-
-  int j = 0;
-  indexedCols = sqlite3_malloc(numCols * sizeof(char *));
-  while (rc == SQLITE_ROW) {
-    assert(j < numCols);
-
-    indexedCols[j] = crsql_strdup((const char *)sqlite3_column_text(pStmt, 0));
-
-    rc = sqlite3_step(pStmt);
-    ++j;
-  }
-  sqlite3_finalize(pStmt);
-
-  if (rc != SQLITE_DONE) {
-    for (int i = 0; i < j; ++i) {
-      sqlite3_free(indexedCols[i]);
-    }
-    sqlite3_free(indexedCols);
-    *pErrMsg = sqlite3_mprintf("Failed allocating index info");
-    return rc;
-  }
-
-  *pIndexedCols = indexedCols;
-  *pIndexedColsLen = numCols;
-
-  return SQLITE_OK;
-}
-
-int crsql_isIdentifierOpenQuote(char c) {
-  switch (c) {
-    case '[':
-      return 1;
-    case '`':
-      return 1;
-    case '"':
-      return 1;
-  }
-
-  return 0;
-}
-
-int crsql_siteIdCmp(const void *zLeft, int leftLen, const void *zRight,
-                    int rightLen) {
-  int minLen = leftLen < rightLen ? leftLen : rightLen;
-  int cmp = memcmp(zLeft, zRight, minLen);
-
-  if (cmp == 0) {
-    if (leftLen > rightLen) {
-      return 1;
-    } else if (leftLen < rightLen) {
-      return -1;
-    }
-
-    return 0;
-  }
-
-  return cmp > 0 ? 1 : -1;
-}

libsql-sqlite3/ext/crr/src/util.h

@@ -13,24 +13,12 @@ char *crsql_getDbVersionUnionQuery(int numRows, char **tableNames);
 
 char *crsql_join(char **in, size_t inlen);
 
-int crsql_doesTableExist(sqlite3 *db, const char *tblName);
-
 int crsql_getCount(sqlite3 *db, char *zSql);
 
 void crsql_joinWith(char *dest, char **src, size_t srcLen, char delim);
-char *crsql_asIdentifierListStr(char **idents, size_t identsLen, char delim);
-
-int crsql_getIndexedCols(sqlite3 *db, const char *indexName,
-                         char ***pIndexedCols, int *pIndexedColsLen,
-                         char **pErrMsg);
 
 char *crsql_join2(char *(*map)(const char *), char **in, size_t len,
                   char *delim);
 const char *crsql_identity(const char *x);
-int crsql_isIdentifierOpenQuote(char c);
-char **crsql_split(const char *in, char *delim, int partsLen);
-int crsql_siteIdCmp(const void *zLeft, int leftLen, const void *zRight,
-                    int rightLen);
-char **crsql_splitQuoteConcat(const char *in, int partsLen);
 
 #endif

libsql-sqlite3/ext/crr/src/util.test.c

@@ -30,42 +30,29 @@ static void testGetVersionUnionQuery() {
   printf("GetVersionUnionQuery\n");
 
   query = crsql_getDbVersionUnionQuery(numRows_tc1, tableNames_tc1);
-  assert(strcmp(query,
-                "SELECT max(version) as version FROM (SELECT "
-                "max(__crsql_db_version) as version FROM \"foo\"  )") == 0);
+  printf("query: %s", query);
+  assert(
+      strcmp(
+          query,
+          "SELECT max(version) as version FROM (SELECT max(__crsql_db_version) "
+          "as version FROM \"foo\"   UNION SELECT value as version FROM "
+          "crsql_master WHERE key = 'pre_compact_dbversion')") == 0);
   sqlite3_free(query);
 
   query = crsql_getDbVersionUnionQuery(numRows_tc2, tableNames_tc2);
-  assert(strcmp(query,
-                "SELECT max(version) as version FROM (SELECT "
-                "max(__crsql_db_version) as version FROM \"foo\" UNION SELECT "
-                "max(__crsql_db_version) as version FROM \"bar\" UNION SELECT "
-                "max(__crsql_db_version) as version FROM \"baz\"  )") == 0);
+  assert(
+      strcmp(
+          query,
+          "SELECT max(version) as version FROM (SELECT max(__crsql_db_version) "
+          "as version FROM \"foo\" UNION ALL SELECT max(__crsql_db_version) as "
+          "version FROM \"bar\" UNION ALL SELECT max(__crsql_db_version) as "
+          "version FROM \"baz\"   UNION SELECT value as version FROM "
+          "crsql_master WHERE key = 'pre_compact_dbversion')") == 0);
   sqlite3_free(query);
 
   printf("\t\e[0;32mSuccess\e[0m\n");
 }
 
-static void testDoesTableExist() {
-  sqlite3 *db;
-  int rc;
-  printf("DoesTableExist\n");
-
-  rc = sqlite3_open(":memory:", &db);
-  if (rc) {
-    fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(db));
-    crsql_close(db);
-    return;
-  }
-
-  assert(crsql_doesTableExist(db, "foo") == 0);
-  sqlite3_exec(db, "CREATE TABLE foo (a, b)", 0, 0, 0);
-  assert(crsql_doesTableExist(db, "foo") == 1);
-
-  crsql_close(db);
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
-
 static void testGetCount() {
   sqlite3 *db = 0;
   int rc = SQLITE_OK;
@@ -96,61 +83,6 @@ static void testJoinWith() {
   printf("\t\e[0;32mSuccess\e[0m\n");
 }
 
-static void testGetIndexedCols() {
-  printf("GetIndexedCols\n");
-
-  sqlite3 *db = 0;
-  int rc = SQLITE_OK;
-  char **indexedCols = 0;
-  int indexedColsLen;
-  char *pErrMsg = 0;
-
-  rc = sqlite3_open(":memory:", &db);
-  sqlite3_exec(db, "CREATE TABLE foo (a);", 0, 0, 0);
-  sqlite3_exec(db, "CREATE TABLE bar (a primary key);", 0, 0, 0);
-
-  rc = crsql_getIndexedCols(db, "sqlite_autoindex_foo_1", &indexedCols,
-                            &indexedColsLen, &pErrMsg);
-  CHECK_OK
-
-  assert(indexedColsLen == 0);
-  assert(indexedCols == 0);
-
-  rc = crsql_getIndexedCols(db, "sqlite_autoindex_bar_1", &indexedCols,
-                            &indexedColsLen, &pErrMsg);
-  CHECK_OK
-
-  assert(indexedColsLen == 1);
-  assert(strcmp(indexedCols[0], "a") == 0);
-
-  sqlite3_free(indexedCols[0]);
-  sqlite3_free(indexedCols);
-
-  crsql_close(db);
-  printf("\t\e[0;32mSuccess\e[0m\n");
-  return;
-
-fail:
-  crsql_close(db);
-  sqlite3_free(pErrMsg);
-  printf("bad return code: %d\n", rc);
-}
-
-static void testAsIdentifierListStr() {
-  printf("AsIdentifierListStr\n");
-
-  char *tc1[] = {"one", "two", "three"};
-  char *res;
-
-  res = crsql_asIdentifierListStr(tc1, 3, ',');
-
-  assert(strcmp(res, "\"one\",\"two\",\"three\"") == 0);
-  assert(strlen(res) == 19);
-  sqlite3_free(res);
-
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
-
 static char *join2map(const char *in) {
   return sqlite3_mprintf("foo %s bar", in);
 }
@@ -176,168 +108,13 @@ static void testJoin2() {
   printf("\t\e[0;32mSuccess\e[0m\n");
 }
 
-void testSiteIdCmp() {
-  printf("SiteIdCmp\n");
-
-  char left[1] = {0x00};
-  char right[1] = {0x00};
-
-  assert(crsql_siteIdCmp(left, 1, right, 1) == 0);
-
-  left[0] = 0x0a;
-  assert(crsql_siteIdCmp(left, 1, right, 1) == 1);
-
-  right[0] = 0x10;
-  assert(crsql_siteIdCmp(left, 1, right, 1) == -1);
-
-  char left2[2] = {0x00, 0x00};
-  right[0] = 0x00;
-  assert(crsql_siteIdCmp(left2, 2, right, 1) == 1);
-
-  char right2[2] = {0x00, 0x00};
-  left[0] = 0x00;
-  assert(crsql_siteIdCmp(left, 1, right2, 2) == -1);
-
-  left[0] = 0x0a;
-  assert(crsql_siteIdCmp(left, 1, right2, 2) == 1);
-
-  right[0] = 0x11;
-  assert(crsql_siteIdCmp(left2, 2, right, 1) == -1);
-
-  printf("\t\e[0;32mSuccess\e[0m\n");
-}
-
-#define FREE_PARTS(L)           \
-  for (int i = 0; i < L; ++i) { \
-    sqlite3_free(parts[i]);     \
-  }                             \
-  sqlite3_free(parts);
-
-void testSplitQuoteConcat() {
-  // test NULL
-  char **parts = crsql_splitQuoteConcat("NULL", 1);
-  assert(strcmp(parts[0], "NULL") == 0);
-  FREE_PARTS(1)
-
-  // test num
-  parts = crsql_splitQuoteConcat("1.23", 1);
-  assert(strcmp(parts[0], "1.23") == 0);
-  FREE_PARTS(1)
-
-  // test empty string
-  parts = crsql_splitQuoteConcat("''", 1);
-  assert(strcmp(parts[0], "''") == 0);
-  FREE_PARTS(1)
-
-  // test string
-  parts = crsql_splitQuoteConcat("'this is a''string'''", 1);
-  assert(strcmp(parts[0], "'this is a''string'''") == 0);
-  FREE_PARTS(1)
-
-  parts = crsql_splitQuoteConcat("'this is another'", 1);
-  assert(strcmp(parts[0], "'this is another'") == 0);
-  FREE_PARTS(1)
-
-  // test hex
-  parts = crsql_splitQuoteConcat("X'aa'", 1);
-  assert(strcmp(parts[0], "X'aa'") == 0);
-  FREE_PARTS(1)
-
-  // test many nulls
-  parts = crsql_splitQuoteConcat("NULL|NULL|NULL", 3);
-  assert(strcmp(parts[0], "NULL") == 0);
-  assert(strcmp(parts[1], "NULL") == 0);
-  assert(strcmp(parts[2], "NULL") == 0);
-  FREE_PARTS(3)
-
-  // test many nums
-  parts = crsql_splitQuoteConcat("12|23324|2.2", 3);
-  assert(strcmp(parts[0], "12") == 0);
-  assert(strcmp(parts[1], "23324") == 0);
-  assert(strcmp(parts[2], "2.2") == 0);
-  FREE_PARTS(3)
-
-  // test many empty strings
-  parts = crsql_splitQuoteConcat("''|''|''", 3);
-  assert(strcmp(parts[0], "''") == 0);
-  assert(strcmp(parts[1], "''") == 0);
-  assert(strcmp(parts[2], "''") == 0);
-  FREE_PARTS(3)
-
-  // test many hex
-  parts = crsql_splitQuoteConcat("X'aa'|X'ff'|X'cc'", 3);
-  assert(strcmp(parts[0], "X'aa'") == 0);
-  assert(strcmp(parts[1], "X'ff'") == 0);
-  assert(strcmp(parts[2], "X'cc'") == 0);
-  FREE_PARTS(3)
-
-  // test many strings
-  parts = crsql_splitQuoteConcat("'foo'|'bar'|'ba''z'", 3);
-  assert(strcmp(parts[0], "'foo'") == 0);
-  assert(strcmp(parts[1], "'bar'") == 0);
-  assert(strcmp(parts[2], "'ba''z'") == 0);
-  FREE_PARTS(3)
-
-  // test not enough parts
-  parts = crsql_splitQuoteConcat("'foo'|'bar'", 3);
-  assert(parts == 0);
-
-  // test too many parts
-  parts = crsql_splitQuoteConcat("'foo'|'bar'|1", 2);
-  assert(parts == 0);
-
-  // test combinations of types
-  parts = crsql_splitQuoteConcat("'foo'|'bar'|1", 3);
-  assert(strcmp(parts[0], "'foo'") == 0);
-  assert(strcmp(parts[1], "'bar'") == 0);
-  assert(strcmp(parts[2], "1") == 0);
-  FREE_PARTS(3)
-
-  parts = crsql_splitQuoteConcat("X'foo'|123|NULL", 3);
-  assert(strcmp(parts[0], "X'foo'") == 0);
-  assert(strcmp(parts[1], "123") == 0);
-  assert(strcmp(parts[2], "NULL") == 0);
-  FREE_PARTS(3)
-
-  // test incorrectly escaped string
-  parts = crsql_splitQuoteConcat("'dude''", 1);
-  assert(parts == 0);
-  parts = crsql_splitQuoteConcat("'du'de'", 1);
-  assert(parts == 0);
-
-  // test unquoted string
-  parts = crsql_splitQuoteConcat("s", 1);
-  assert(parts == 0);
-
-  // test digits with chars
-  parts = crsql_splitQuoteConcat("12s", 1);
-  assert(parts == 0);
-
-  // test X str
-  parts = crsql_splitQuoteConcat("Xs", 1);
-  assert(parts == 0);
-  parts = crsql_splitQuoteConcat("X's", 1);
-  assert(parts == 0);
-  parts = crsql_splitQuoteConcat("X's''", 1);
-  assert(parts == 0);
-
-  // test string missing end quote
-  parts = crsql_splitQuoteConcat("'s", 1);
-  assert(parts == 0);
-}
-
 void crsqlUtilTestSuite() {
   printf("\e[47m\e[1;30mSuite: crsql_util\e[0m\n");
 
   testGetVersionUnionQuery();
-  testDoesTableExist();
   testGetCount();
   testJoinWith();
-  testGetIndexedCols();
-  testAsIdentifierListStr();
   testJoin2();
-  testSiteIdCmp();
-  testSplitQuoteConcat();
 
   // TODO: test pk pulling and correct sorting of pks
   // TODO: create a fn to create test tables for all tests.

libsql-sqlite3/src/sqlite3ext.h

@@ -370,6 +370,7 @@ struct libsql_api_routines {
   struct libsql_wal_methods *(*wal_methods_find)(const char *);
   int (*wal_methods_register)(struct libsql_wal_methods*);
   int (*wal_methods_unregister)(struct libsql_wal_methods*);
+  /* libSQL 0.2.3 */
   void *(*close_hook)(sqlite3*, void(*)(void*,sqlite3*), void *pArg);
 };