idbfs+repl

cibuild.sh

@@ -181,6 +181,7 @@ if echo "$*"|grep " quack"
 then
     echo "================================================="
     PG_LINK=em++ echo WIP
+    ./quack.sh
 fi
 
 # in pg git test mode we pull pglite instead

cibuild/linkweb.sh

@@ -68,6 +68,9 @@ PG_L="-L../../src/port -L../../src/common \
  ../../src/common/libpgcommon_srv.a ../../src/port/libpgport_srv.a"
 
 PG_L="$PG_L -L../../src/interfaces/ecpg/ecpglib ../../src/interfaces/ecpg/ecpglib/libecpg.so"
+
+# /tmp/pglite/lib/postgresql/libduckdb.so"
+
 ## \
 # /opt/python-wasm-sdk/devices/emsdk/usr/lib/libxml2.a \
 # /opt/python-wasm-sdk/devices/emsdk/usr/lib/libgeos.a \
@@ -127,7 +130,7 @@ emcc $EMCC_WEB -fPIC -sMAIN_MODULE=1 \
   $MODULE -sERROR_ON_UNDEFINED_SYMBOLS -sASSERTIONS=0 \
  -lnodefs.js -lidbfs.js \
  -sEXPORTED_RUNTIME_METHODS=FS,setValue,getValue,stringToNewUTF8,stringToUTF8OnStack,ccall,cwrap,callMain \
- -sEXPORTED_FUNCTIONS=_main,_getenv,_setenv,_interactive_one,_interactive_write,_interactive_read,_pg_initdb \
+ -sEXPORTED_FUNCTIONS=_main,_getenv,_setenv,_interactive_one,_interactive_write,_interactive_read,_pg_initdb,_pg_shutdown \
  --preload-file ${PGROOT}/share/postgresql@${PGROOT}/share/postgresql \
  --preload-file ${PGROOT}/lib/postgresql@${PGROOT}/lib/postgresql \
  --preload-file ${PGROOT}/password@${PGROOT}/password \

cibuild/pglite-ts.sh

@@ -78,6 +78,8 @@ END
             npm pack
             packed=$(echo -n electric-sql-pglite-*.tgz)
             mv $packed /tmp/sdk/pg${PGVERSION}-${packed}
+            # for repl demo
+            cp -r ${PGLITE}/dist ${WEBROOT}/repl/
         else
             mkdir -p ${WEBROOT}/node_modules/@electric-sql/pglite
             cp -r ${PGLITE}/{../../LICENSE,package.json,README.md} ${PGLITE}/dist ${WEBROOT}/node_modules/@electric-sql/pglite/

packages/pglite/src/fs/idbfs.ts

@@ -10,12 +10,9 @@ export class IdbFs extends FilesystemBase {
       preRun: [
         (mod: any) => {
           const idbfs = mod.FS.filesystems.IDBFS;
-          // Mount the idbfs to the users dataDir
-          // then symlink the PGDATA to the idbfs mount
-          mod.FS.mkdir(`/pglite`);
-          mod.FS.mkdir(`/pglite${this.dataDir}`);
-          mod.FS.mount(idbfs, {}, `/pglite${this.dataDir}`);
-          mod.FS.symlink(`/pglite${this.dataDir}`, `/tmp/pglite/base`);
+          // Mount the idbfs to PGDATA in auto commit mode
+          mod.FS.mkdir(`/tmp/pglite/${this.dataDir}`);
+          mod.FS.mount(idbfs, {autoPersist: true}, `/tmp/pglite/${this.dataDir}`);
         },
       ],
     };
@@ -40,7 +37,7 @@ export class IdbFs extends FilesystemBase {
 
   syncToFs(fs: FS) {
     return new Promise<void>((resolve, reject) => {
-      fs.syncfs((err: any) => {
+      fs.syncfs(false, (err: any) => {
         if (err) {
           reject(err);
         } else {

packages/pglite/src/fs/index.ts

@@ -4,7 +4,7 @@ import { MemoryFS } from "./memoryfs.js";
 
 export type * from "./types.js";
 
-export const PGDATA = "/pgdata";
+export const PGDATA = "/tmp/pglite";
 
 export function parseDataDir(dataDir?: string) {
   let fsType: FsType;
@@ -18,12 +18,10 @@ export function parseDataDir(dataDir?: string) {
   } else if (dataDir?.startsWith("idb://")) {
     // Remove the idb:// prefix, and use indexeddb filesystem
     dataDir = dataDir.slice(6);
-    if (!dataDir.startsWith("/")) {
-      dataDir = "/" + dataDir;
-    }
     if (dataDir.length <= 1) {
       throw new Error("Invalid dataDir, path required for idbfs");
     }
+    console.warn
     fsType = "idbfs";
   } else if (!dataDir || dataDir?.startsWith("memory://")) {
     // Use in-memory filesystem

packages/pglite/src/pglite.ts

@@ -93,6 +93,9 @@ export class PGlite implements PGliteInterface {
    * @returns A promise that resolves when the database is ready
    */
   async #init(options: PGliteOptions) {
+
+    console.log("options :", options)
+
     if (options.fs) {
       this.fs = options.fs;
     } else {
@@ -118,21 +121,29 @@ export class PGlite implements PGliteInterface {
     }
 
     emscriptenOpts = await this.fs!.emscriptenOpts(emscriptenOpts);
-    console.log("emscriptenOpts:", emscriptenOpts);
+    //console.log("emscriptenOpts:", emscriptenOpts);
 
     // init pg core engine done only using MEMFS
     this.emp = await EmPostgresFactory(emscriptenOpts);
 
     // if ok, NOW:
     //   all pg c-api is avail. including exported sym
 
+    console.warn("idbfs: mounting");
+/*          this.emp.FS.mkdir("/tmp/pglite/base");
+          this.emp.FS.mount(this.emp.FS.filesystems.IDBFS, {autoPersist: false}, '/tmp/pglite/base');
+*/
+
 
     // finalize FS states needed before initdb.
     // maybe start extra FS/initdata async .
 
-    console.error("syncing fs ....");
+    console.error("syncing fs (idbfs->memfs)");
     await this.fs!.initialSyncFs(this.emp.FS);
 
+    console.warn("idbfs: mounted");
+
+
     // start compiling dynamic extensions present in FS.
 
     console.log("database engine is ready (but not yet system/user databases or extensions)");
@@ -505,6 +516,7 @@ export class PGlite implements PGliteInterface {
       await this.#fsSyncMutex.runExclusive(async () => {
         this.#fsSyncScheduled = false;
         await this.fs!.syncToFs(this.emp.FS);
+        console.warn("FS synced");
       });
     };