Merge tag 'version-3.44.0' into HEAD

libsql-sqlite3/ext/wasm/api/sqlite3-api-glue.js

@@ -888,9 +888,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
      consistency with non-special-case wrappings.
   */
   const __dbArgcMismatch = (pDb,f,n)=>{
-    return sqlite3.util.sqlite3_wasm_db_error(pDb, capi.SQLITE_MISUSE,
-                                              f+"() requires "+n+" argument"+
-                                              (1===n?"":'s')+".");
+    return util.sqlite3_wasm_db_error(pDb, capi.SQLITE_MISUSE,
+                                      f+"() requires "+n+" argument"+
+                                      (1===n?"":'s')+".");
   };
 
   /** Code duplication reducer for functions which take an encoding

libsql-sqlite3/ext/wasm/api/sqlite3-api-prologue.js

@@ -772,8 +772,43 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     isSharedTypedArray,
     toss: function(...args){throw new Error(args.join(' '))},
     toss3,
-    typedArrayPart
-  };
+    typedArrayPart,
+    /**
+       Given a byte array or ArrayBuffer, this function throws if the
+       lead bytes of that buffer do not hold a SQLite3 database header,
+       else it returns without side effects.
+
+       Added in 3.44.
+    */
+    affirmDbHeader: function(bytes){
+      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      const header = "SQLite format 3";
+      if( header.length > bytes.byteLength ){
+        toss3("Input does not contain an SQLite3 database header.");
+      }
+      for(let i = 0; i < header.length; ++i){
+        if( header.charCodeAt(i) !== bytes[i] ){
+          toss3("Input does not contain an SQLite3 database header.");
+        }
+      }
+    },
+    /**
+       Given a byte array or ArrayBuffer, this function throws if the
+       database does not, at a cursory glance, appear to be an SQLite3
+       database. It only examines the size and header, but further
+       checks may be added in the future.
+
+       Added in 3.44.
+    */
+    affirmIsDb: function(bytes){
+      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      const n = bytes.byteLength;
+      if(n<512 || n%512!==0) {
+        toss3("Byte array size",n,"is invalid for an SQLite3 db.");
+      }
+      util.affirmDbHeader(bytes);
+    }
+  }/*util*/;
 
   Object.assign(wasm, {
     /**
@@ -1100,7 +1135,23 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
       return 1===n
         ? wasm.pstack.alloc(safePtrSize ? 8 : wasm.ptrSizeof)
         : wasm.pstack.allocChunks(n, safePtrSize ? 8 : wasm.ptrSizeof);
+    },
+
+    /**
+       Records the current pstack position, calls the given function,
+       passing it the sqlite3 object, then restores the pstack
+       regardless of whether the function throws. Returns the result
+       of the call or propagates an exception on error.
+
+       Added in 3.44.
+    */
+    call: function(f){
+      const stackPos = wasm.pstack.pointer;
+      try{ return f(sqlite3) } finally{
+        wasm.pstack.restore(stackPos);
+      }
     }
+
   })/*wasm.pstack*/;
   Object.defineProperties(wasm.pstack, {
     /**
@@ -1508,6 +1559,26 @@ globalThis.sqlite3ApiBootstrap = function sqlite3ApiBootstrap(
     }
   };
 
+  /**
+     Converts SQL input from a variety of convenient formats
+     to plain strings.
+
+     If v is a string, it is returned as-is. If it is-a Array, its
+     join("") result is returned.  If is is a Uint8Array, Int8Array,
+     or ArrayBuffer, it is assumed to hold UTF-8-encoded text and is
+     decoded to a string. If it looks like a WASM pointer,
+     wasm.cstrToJs(sql) is returned. Else undefined is returned.
+
+     Added in 3.44
+  */
+  capi.sqlite3_js_sql_to_string = (sql)=>{
+    if('string' === typeof sql){
+      return sql;
+    }
+    const x = flexibleString(v);
+    return x===v ? undefined : x;
+  }
+
   if( util.isUIThread() ){
     /* Features specific to the main window thread... */
 

libsql-sqlite3/ext/wasm/api/sqlite3-api-worker1.js

@@ -333,7 +333,6 @@ sqlite3.initWorker1API = function(){
   if(!(globalThis.WorkerGlobalScope instanceof Function)){
     toss("initWorker1API() must be run from a Worker thread.");
   }
-  const self = this.self;
   const sqlite3 = this.sqlite3 || toss("Missing this.sqlite3 object.");
   const DB = sqlite3.oo1.DB;
 
@@ -657,5 +656,5 @@ sqlite3.initWorker1API = function(){
     }, wState.xfer);
   };
   globalThis.postMessage({type:'sqlite3-api',result:'worker1-ready'});
-}.bind({self, sqlite3});
+}.bind({sqlite3});
 });

libsql-sqlite3/ext/wasm/api/sqlite3-vfs-opfs-sahpool.c-pp.js

@@ -59,6 +59,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
   const toss3 = sqlite3.util.toss3;
   const initPromises = Object.create(null);
   const capi = sqlite3.capi;
+  const util = sqlite3.util;
   const wasm = sqlite3.wasm;
   // Config opts for the VFS...
   const SECTOR_SIZE = 4096;
@@ -154,8 +155,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
             pool.deletePath(file.path);
           }
         }catch(e){
-          pool.storeErr(e);
-          return capi.SQLITE_IOERR;
+          return pool.storeErr(e, capi.SQLITE_IOERR);
         }
       }
       return 0;
@@ -199,8 +199,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
         }
         return 0;
       }catch(e){
-        pool.storeErr(e);
-        return capi.SQLITE_IOERR;
+        return pool.storeErr(e, capi.SQLITE_IOERR);
       }
     },
     xSectorSize: function(pFile){
@@ -216,8 +215,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
         file.sah.flush();
         return 0;
       }catch(e){
-        pool.storeErr(e);
-        return capi.SQLITE_IOERR;
+        return pool.storeErr(e, capi.SQLITE_IOERR);
       }
     },
     xTruncate: function(pFile,sz64){
@@ -230,8 +228,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
         file.sah.truncate(HEADER_OFFSET_DATA + Number(sz64));
         return 0;
       }catch(e){
-        pool.storeErr(e);
-        return capi.SQLITE_IOERR;
+        return pool.storeErr(e, capi.SQLITE_IOERR);
       }
     },
     xUnlock: function(pFile,lockType){
@@ -251,10 +248,9 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
           wasm.heap8u().subarray(pSrc, pSrc+n),
           { at: HEADER_OFFSET_DATA + Number(offset64) }
         );
-        return nBytes === n ? 0 : capi.SQLITE_IOERR;
+        return n===nBytes ? 0 : toss("Unknown write() failure.");
       }catch(e){
-        pool.storeErr(e);
-        return capi.SQLITE_IOERR;
+        return pool.storeErr(e, capi.SQLITE_IOERR);
       }
     }
   }/*ioMethods*/;
@@ -313,8 +309,8 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
     },
     xGetLastError: function(pVfs,nOut,pOut){
       const pool = getPoolForVfs(pVfs);
-      pool.log(`xGetLastError ${nOut}`);
       const e = pool.popErr();
+      pool.log(`xGetLastError ${nOut} e =`,e);
       if(e){
         const scope = wasm.scopedAllocPush();
         try{
@@ -327,7 +323,7 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
           wasm.scopedAllocPop(scope);
         }
       }
-      return 0;
+      return e ? (e.sqlite3Rc || capi.SQLITE_IOERR) : 0;
     },
     //xSleep is optionally defined below
     xOpen: function f(pVfs, zName, pFile, flags, pOutFlags){
@@ -761,12 +757,20 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
     }
 
     /**
-       Sets e as this object's current error. Pass a falsy
-       (or no) value to clear it.
+       Sets e (an Error object) as this object's current error. Pass a
+       falsy (or no) value to clear it. If code is truthy it is
+       assumed to be an SQLITE_xxx result code, defaulting to
+       SQLITE_IOERR if code is falsy.
+
+       Returns the 2nd argument.
     */
-    storeErr(e){
-      if(e) this.error(e);
-      return this.$error = e;
+    storeErr(e,code){
+      if(e){
+        e.sqlite3Rc = code || capi.SQLITE_IOERR;
+        this.error(e);
+      }
+      this.$error = e;
+      return code;
     }
     /**
        Pops this object's Error object and returns
@@ -869,9 +873,49 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
       return b;
     }
 
+    //! Impl for importDb() when its 2nd arg is a function.
+    async importDbChunked(name, callback){
+      const sah = this.#mapFilenameToSAH.get(name)
+            || this.nextAvailableSAH()
+            || toss("No available handles to import to.");
+      sah.truncate(0);
+      let nWrote = 0, chunk, checkedHeader = false, err = false;
+      try{
+        while( undefined !== (chunk = await callback()) ){
+          if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
+          if( 0===nWrote && chunk.byteLength>=15 ){
+            util.affirmDbHeader(chunk);
+            checkedHeader = true;
+          }
+          sah.write(chunk, {at:  HEADER_OFFSET_DATA + nWrote});
+          nWrote += chunk.byteLength;
+        }
+        if( nWrote < 512 || 0!==nWrote % 512 ){
+          toss("Input size",nWrote,"is not correct for an SQLite database.");
+        }
+        if( !checkedHeader ){
+          const header = new Uint8Array(20);
+          sah.read( header, {at: 0} );
+          util.affirmDbHeader( header );
+        }
+        sah.write(new Uint8Array([1,1]), {
+          at: HEADER_OFFSET_DATA + 18
+        }/*force db out of WAL mode*/);
+      }catch(e){
+        this.setAssociatedPath(sah, '', 0);
+        throw e;
+      }
+      this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
+      return nWrote;
+    }
+
     //! Documented elsewhere in this file.
     importDb(name, bytes){
-      if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      if( bytes instanceof ArrayBuffer ) bytes = new Uint8Array(bytes);
+      else if( bytes instanceof Function ) return this.importDbChunked(name, bytes);
+      const sah = this.#mapFilenameToSAH.get(name)
+            || this.nextAvailableSAH()
+            || toss("No available handles to import to.");
       const n = bytes.byteLength;
       if(n<512 || n%512!=0){
         toss("Byte array size is invalid for an SQLite db.");
@@ -882,16 +926,16 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
           toss("Input does not contain an SQLite database header.");
         }
       }
-      const sah = this.#mapFilenameToSAH.get(name)
-            || this.nextAvailableSAH()
-            || toss("No available handles to import to.");
       const nWrote = sah.write(bytes, {at: HEADER_OFFSET_DATA});
       if(nWrote != n){
         this.setAssociatedPath(sah, '', 0);
         toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
       }else{
+        sah.write(new Uint8Array([1,1]), {at: HEADER_OFFSET_DATA+18}
+                   /* force db out of WAL mode */);
         this.setAssociatedPath(sah, name, capi.SQLITE_OPEN_MAIN_DB);
       }
+      return nWrote;
     }
 
   }/*class OpfsSAHPool*/;
@@ -1098,6 +1142,19 @@ globalThis.sqlite3ApiBootstrap.initializers.push(function(sqlite3){
      automatically clean up any non-database files so importing them
      is pointless.
 
+     If passed a function for its second argument, its behavior
+     changes to asynchronous and it imports its data in chunks fed to
+     it by the given callback function. It calls the callback (which
+     may be async) repeatedly, expecting either a Uint8Array or
+     ArrayBuffer (to denote new input) or undefined (to denote
+     EOF). For so long as the callback continues to return
+     non-undefined, it will append incoming data to the given
+     VFS-hosted database file. The result of the resolved Promise when
+     called this way is the size of the resulting database.
+
+     On succes this routine rewrites the database header bytes in the
+     output file (not the input array) to force disabling of WAL mode.
+
      On a write error, the handle is removed from the pool and made
      available for re-use.
 

libsql-sqlite3/ext/wasm/api/sqlite3-vfs-opfs.c-pp.js

@@ -136,6 +136,7 @@ const installOpfsVfs = function callee(options){
     const error =  (...args)=>logImpl(0, ...args);
     const toss = sqlite3.util.toss;
     const capi = sqlite3.capi;
+    const util = sqlite3.util;
     const wasm = sqlite3.wasm;
     const sqlite3_vfs = capi.sqlite3_vfs;
     const sqlite3_file = capi.sqlite3_file;
@@ -1168,40 +1169,101 @@ const installOpfsVfs = function callee(options){
       doDir(opt.directory, 0);
     };
 
+    /**
+       impl of importDb() when it's given a function as its second
+       argument.
+    */
+    const importDbChunked = async function(filename, callback){
+      const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
+      const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+      let sah = await hFile.createSyncAccessHandle();
+      let nWrote = 0, chunk, checkedHeader = false, err = false;
+      try{
+        sah.truncate(0);
+        while( undefined !== (chunk = await callback()) ){
+          if(chunk instanceof ArrayBuffer) chunk = new Uint8Array(chunk);
+          if( 0===nWrote && chunk.byteLength>=15 ){
+            util.affirmDbHeader(chunk);
+            checkedHeader = true;
+          }
+          sah.write(chunk, {at: nWrote});
+          nWrote += chunk.byteLength;
+        }
+        if( nWrote < 512 || 0!==nWrote % 512 ){
+          toss("Input size",nWrote,"is not correct for an SQLite database.");
+        }
+        if( !checkedHeader ){
+          const header = new Uint8Array(20);
+          sah.read( header, {at: 0} );
+          util.affirmDbHeader( header );
+        }
+        sah.write(new Uint8Array([1,1]), {at: 18}/*force db out of WAL mode*/);
+        return nWrote;
+      }catch(e){
+        await sah.close();
+        sah = undefined;
+        await hDir.removeEntry( fnamePart ).catch(()=>{});
+        throw e;
+      }finally {
+        if( sah ) await sah.close();
+      }
+    };
+
     /**
        Asynchronously imports the given bytes (a byte array or
        ArrayBuffer) into the given database file.
 
+       If passed a function for its second argument, its behaviour
+       changes to async and it imports its data in chunks fed to it by
+       the given callback function. It calls the callback (which may
+       be async) repeatedly, expecting either a Uint8Array or
+       ArrayBuffer (to denote new input) or undefined (to denote
+       EOF). For so long as the callback continues to return
+       non-undefined, it will append incoming data to the given
+       VFS-hosted database file. When called this way, the resolved
+       value of the returned Promise is the number of bytes written to
+       the target file.
+
        It very specifically requires the input to be an SQLite3
        database and throws if that's not the case.  It does so in
        order to prevent this function from taking on a larger scope
        than it is specifically intended to. i.e. we do not want it to
        become a convenience for importing arbitrary files into OPFS.
 
-       Throws on error. Resolves to the number of bytes written.
+       This routine rewrites the database header bytes in the output
+       file (not the input array) to force disabling of WAL mode.
+
+       On error this throws and the state of the input file is
+       undefined (it depends on where the exception was triggered).
+
+       On success, resolves to the number of bytes written.
     */
     opfsUtil.importDb = async function(filename, bytes){
+      if( bytes instanceof Function ){
+        return importDbChunked(filename, bytes);
+      }
       if(bytes instanceof ArrayBuffer) bytes = new Uint8Array(bytes);
+      util.affirmIsDb(bytes);
       const n = bytes.byteLength;
-      if(n<512 || n%512!=0){
-        toss("Byte array size is invalid for an SQLite db.");
-      }
-      const header = "SQLite format 3";
-      for(let i = 0; i < header.length; ++i){
-        if( header.charCodeAt(i) !== bytes[i] ){
-          toss("Input does not contain an SQLite database header.");
-        }
-      }
       const [hDir, fnamePart] = await opfsUtil.getDirForFilename(filename, true);
-      const hFile = await hDir.getFileHandle(fnamePart, {create:true});
-      const sah = await hFile.createSyncAccessHandle();
-      sah.truncate(0);
-      const nWrote = sah.write(bytes, {at: 0});
-      sah.close();
-      if(nWrote != n){
-        toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+      let sah, err, nWrote = 0;
+      try {
+        const hFile = await hDir.getFileHandle(fnamePart, {create:true});
+        sah = await hFile.createSyncAccessHandle();
+        sah.truncate(0);
+        nWrote = sah.write(bytes, {at: 0});
+        if(nWrote != n){
+          toss("Expected to write "+n+" bytes but wrote "+nWrote+".");
+        }
+        sah.write(new Uint8Array([1,1]), {at: 18}) /* force db out of WAL mode */;
+        return nWrote;
+      }catch(e){
+        if( sah ){ await sah.close(); sah = undefined; }
+        await hDir.removeEntry( fnamePart ).catch(()=>{});
+        throw e;
+      }finally{
+        if( sah ) await sah.close();
       }
-      return nWrote;
     };
 
     if(sqlite3.oo1){

libsql-sqlite3/ext/wasm/api/sqlite3-wasm.c

@@ -84,6 +84,14 @@
 
 /**********************************************************************/
 /* SQLITE_ENABLE_... */
+/*
+** Unconditionally enable API_ARMOR in the WASM build. It ensures that
+** public APIs behave predictable in the face of passing illegal NULLs
+** or ranges which might otherwise invoke undefined behavior.
+*/
+#undef SQLITE_ENABLE_API_ARMOR
+#define SQLITE_ENABLE_API_ARMOR 1
+
 #ifndef SQLITE_ENABLE_BYTECODE_VTAB
 #  define SQLITE_ENABLE_BYTECODE_VTAB 1
 #endif
@@ -121,12 +129,6 @@
 #  define SQLITE_ENABLE_UNKNOWN_SQL_FUNCTION
 #endif
 
-/**********************************************************************/
-/* SQLITE_M... */
-#ifndef SQLITE_MAX_ALLOCATION_SIZE
-# define SQLITE_MAX_ALLOCATION_SIZE 0x1fffffff
-#endif
-
 /**********************************************************************/
 /* SQLITE_O... */
 #ifndef SQLITE_OMIT_DEPRECATED
@@ -352,7 +354,9 @@ int sqlite3_wasm_db_error(sqlite3*db, int err_code, const char *zMsg){
   if( db!=0 ){
     if( 0!=zMsg ){
       const int nMsg = sqlite3Strlen30(zMsg);
+      sqlite3_mutex_enter(sqlite3_db_mutex(db));
       sqlite3ErrorWithMsg(db, err_code, "%.*s", nMsg, zMsg);
+      sqlite3_mutex_leave(sqlite3_db_mutex(db));
     }else{
       sqlite3ErrorWithMsg(db, err_code, NULL);
     }
@@ -1799,6 +1803,118 @@ char * sqlite3_wasm_test_str_hello(int fail){
   }
   return s;
 }
+
+/*
+** For testing using SQLTester scripts.
+**
+** Return non-zero if string z matches glob pattern zGlob and zero if the
+** pattern does not match.
+**
+** To repeat:
+**
+**         zero == no match
+**     non-zero == match
+**
+** Globbing rules:
+**
+**      '*'       Matches any sequence of zero or more characters.
+**
+**      '?'       Matches exactly one character.
+**
+**     [...]      Matches one character from the enclosed list of
+**                characters.
+**
+**     [^...]     Matches one character not in the enclosed list.
+**
+**      '#'       Matches any sequence of one or more digits with an
+**                optional + or - sign in front, or a hexadecimal
+**                literal of the form 0x...
+*/
+static int sqlite3_wasm_SQLTester_strnotglob(const char *zGlob, const char *z){
+  int c, c2;
+  int invert;
+  int seen;
+  typedef int (*recurse_f)(const char *,const char *);
+  static const recurse_f recurse = sqlite3_wasm_SQLTester_strnotglob;
+
+  while( (c = (*(zGlob++)))!=0 ){
+    if( c=='*' ){
+      while( (c=(*(zGlob++))) == '*' || c=='?' ){
+        if( c=='?' && (*(z++))==0 ) return 0;
+      }
+      if( c==0 ){
+        return 1;
+      }else if( c=='[' ){
+        while( *z && recurse(zGlob-1,z)==0 ){
+          z++;
+        }
+        return (*z)!=0;
+      }
+      while( (c2 = (*(z++)))!=0 ){
+        while( c2!=c ){
+          c2 = *(z++);
+          if( c2==0 ) return 0;
+        }
+        if( recurse(zGlob,z) ) return 1;
+      }
+      return 0;
+    }else if( c=='?' ){
+      if( (*(z++))==0 ) return 0;
+    }else if( c=='[' ){
+      int prior_c = 0;
+      seen = 0;
+      invert = 0;
+      c = *(z++);
+      if( c==0 ) return 0;
+      c2 = *(zGlob++);
+      if( c2=='^' ){
+        invert = 1;
+        c2 = *(zGlob++);
+      }
+      if( c2==']' ){
+        if( c==']' ) seen = 1;
+        c2 = *(zGlob++);
+      }
+      while( c2 && c2!=']' ){
+        if( c2=='-' && zGlob[0]!=']' && zGlob[0]!=0 && prior_c>0 ){
+          c2 = *(zGlob++);
+          if( c>=prior_c && c<=c2 ) seen = 1;
+          prior_c = 0;
+        }else{
+          if( c==c2 ){
+            seen = 1;
+          }
+          prior_c = c2;
+        }
+        c2 = *(zGlob++);
+      }
+      if( c2==0 || (seen ^ invert)==0 ) return 0;
+    }else if( c=='#' ){
+      if( z[0]=='0'
+       && (z[1]=='x' || z[1]=='X')
+       && sqlite3Isxdigit(z[2])
+      ){
+        z += 3;
+        while( sqlite3Isxdigit(z[0]) ){ z++; }
+      }else{
+        if( (z[0]=='-' || z[0]=='+') && sqlite3Isdigit(z[1]) ) z++;
+        if( !sqlite3Isdigit(z[0]) ) return 0;
+        z++;
+        while( sqlite3Isdigit(z[0]) ){ z++; }
+      }
+    }else{
+      if( c!=(*(z++)) ) return 0;
+    }
+  }
+  return *z==0;
+}
+
+SQLITE_WASM_EXPORT
+int sqlite3_wasm_SQLTester_strglob(const char *zGlob, const char *z){
+ return !sqlite3_wasm_SQLTester_strnotglob(zGlob, z);
+}
+
+
 #endif /* SQLITE_WASM_TESTS */
 
 #undef SQLITE_WASM_EXPORT

libsql-sqlite3/ext/wasm/api/sqlite3-worker1.c-pp.js

@@ -37,7 +37,7 @@ import {default as sqlite3InitModule} from './sqlite3-bundler-friendly.mjs';
 "use strict";
 {
   const urlParams = globalThis.location
-        ? new URL(self.location.href).searchParams
+        ? new URL(globalThis.location.href).searchParams
         : new URLSearchParams();
   let theJs = 'sqlite3.js';
   if(urlParams.has('sqlite3.dir')){