replace gob with protobuffer

.gitignore

@@ -1,4 +1,5 @@
 tmp/*
 data/*
+benchmarks/*
 *.txt
 *.log

OuroborosDB_test.go

@@ -125,7 +125,7 @@ func setupDBWithData(t testing.TB, conf setupDBConfig) (ou *OuroborosDB.Ouroboro
 
 	// Build index
 	if !conf.notBuildIndex {
-		_, err = ou.Index.RebuildIndex()
+		_, err := ou.Index.RebuildIndex()
 		if err != nil {
 			t.Errorf("RebuildIndex failed with error: %v", err)
 		}
@@ -135,23 +135,27 @@ func setupDBWithData(t testing.TB, conf setupDBConfig) (ou *OuroborosDB.Ouroboro
 }
 
 func Test_Index_RebuildIndex(t *testing.T) {
-	testEvs := 1000
-	ou, _ := setupDBWithData(t, setupDBConfig{})
+	testRows := 10000
+	ou, _ := setupDBWithData(t, setupDBConfig{
+		totalEvents:   testRows,
+		notBuildIndex: true,
+	})
 
 	// Rebuild the index
 	indexedRows, err := ou.Index.RebuildIndex()
 	if err != nil {
 		t.Errorf("RebuildIndex failed with error: %v", err)
 	}
 
-	if indexedRows != uint64(testEvs) {
-		t.Errorf("RebuildIndex failed, expected %d, got %d", testEvs, indexedRows)
+	if indexedRows != uint64(testRows) {
+		t.Errorf("RebuildIndex failed, expected %d, got %d", testRows, indexedRows)
 	}
 }
 
 func Benchmark_Index_RebuildingIndex(b *testing.B) {
 	ou, _ := setupDBWithData(b, setupDBConfig{
-		totalEvents: 10000,
+		totalEvents:   10000,
+		notBuildIndex: true,
 	})
 
 	b.Run("RebuildIndex", func(b *testing.B) {
@@ -201,8 +205,9 @@ func Benchmark_Index_GetDirectChildrenOfEvent(b *testing.B) {
 
 func Test_Index_GetChildrenHashesOfEvent(t *testing.T) {
 	ou, evs := setupDBWithData(t, setupDBConfig{
-		totalEvents:        100,
-		returnRandomEvents: 10,
+		totalEvents:        1000,
+		returnRandomEvents: 100,
+		eventLevels:        5,
 	})
 
 	var childrenCount int // todo this is not a good test

README.md

@@ -9,39 +9,33 @@ A versioned, distributed key-value store designed for data integrity against bit
 
 ## OuroborosDB Development TODO List (has to change)
 
-- [ ] **WASM Networking Tests**
-  - [ ] Set up the Go environment for WebAssembly compilation.
-  - [ ] Create a basic WASM module in Go.
-  - [ ] Implement networking tests for the WASM module.
-
-- [ ] **Core Database Features**
-  - [ ] Define data structures for key-value pairs and metadata.
-  - [ ] Implement CRUD operations for the key-value store.
-  - [ ] Integrate CRC checks for data integrity.
-  - [ ] Design and implement a chain of hashes for history tracking.
-
-- [ ] **Data Merging and Versioning**
-  - [ ] Implement data versioning mechanisms.
-  - [ ] Create functions for merging divergent datasets.
-  - [ ] Add conflict resolution strategies for data merges.
-
-- [ ] **Private-Public Key Security**
-  - [ ] Design the encryption module.
-  - [ ] Integrate a private-public key system for encryption/decryption.
-  - [ ] Implement secure key storage mechanisms.
-
-- [ ] **Deployment & Integration**
-  - [ ] Configure Docker for Linux deployment.
-  - [ ] Create a browser-friendly version using WASM.
-  - [ ] Document deployment processes for both platforms.
-
-- [ ] **Miscellaneous**
-  - [ ] Add logging and monitoring functionalities.
-  - [ ] Design a user-friendly CLI or UI.
-  - [ ] Write comprehensive unit tests for all functionalities.
-  - [ ] Draft detailed documentation and usage guidelines.
-
 
+## OuroborosDB Performance Version Differences
+```bash
+goos: linux
+goarch: arm64
+pkg: OuroborosDB
+                                                           │ ./benchmarks/v0.0.2 │         ./benchmarks/v0.0.3         │
+                                                           │       sec/op        │    sec/op     vs base               │
+_Index_RebuildingIndex/RebuildIndex-8                              397.79m ±  2%   19.40m ±  2%  -95.12% (p=0.002 n=6)
+_Index_GetDirectChildrenOfEvent/GetChildrenOfEvent-8               35.559µ ±  3%   4.727µ ±  3%  -86.71% (p=0.002 n=6)
+_Index_GetChildrenHashesOfEvent/GetChildrenHashesOfEvent-8          77.27n ±  2%   75.79n ±  1%   -1.91% (p=0.004 n=6)
+_DB_StoreFile/StoreFile-8                                           311.7µ ±  4%   188.4µ ±  4%  -39.57% (p=0.002 n=6)
+_DB_GetFile/GetFile-8                                               3.695µ ±  4%   3.422µ ±  4%   -7.40% (p=0.004 n=6)
+_DB_GetEvent/GetEvent-8                                            45.636µ ±  2%   6.220µ ±  2%  -86.37% (p=0.002 n=6)
+_DB_GetMetadata/GetMetadata-8                                       4.040µ ±  4%   3.972µ ±  9%        ~ (p=0.132 n=6)
+_DB_GetAllRootEvents/GetAllRootEvents-8                            121.18m ±  5%   19.46m ±  2%  -83.94% (p=0.002 n=6)
+_DB_GetRootIndex/GetRootIndex-8                                     2.465m ±  5%   2.477m ±  3%        ~ (p=1.000 n=6)
+_DB_GetRootEventsWithTitle/GetRootEventsWithTitle-8                 53.55µ ±  2%   12.24µ ±  3%  -77.13% (p=0.002 n=6)
+_DB_CreateRootEvent/CreateRootEvent-8                               158.1µ ± 11%   129.1µ ± 11%  -18.34% (p=0.002 n=6)
+_DB_CreateNewEvent/CreateNewEvent-8                                135.91µ ±  6%   49.64µ ± 15%  -63.48% (p=0.002 n=6)
+geomean                                                             144.0µ         52.30µ        -63.69%
+```
+
+## OuroborosDB Performance Changelog
+
+- **v0.0.3** - Switch from `gob` to `protobuf` for serialization
+- **v0.0.2** - Create tests and benchmarks
 
 
 ## Name and Logo

bench.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Declare an array of version tags
+declare -a versions=("v0.0.2" "v0.0.3")  # Add your arbitrary versions here
+
+# Create a directory to hold the benchmark results
+mkdir -p benchmarks
+
+# Run benchmarks for each version
+for version in "${versions[@]}"; do
+    echo "Checking out version: $version"
+    git checkout $version
+
+    echo "Running benchmarks for version: $version"
+    go test -run='^$' -bench=. -count=6 > "benchmarks/${version}"
+done
+
+# Checkout the main branch again
+git checkout main
+
+# Generate the benchstat comparison command with all the .txt files
+benchstat_cmd="benchstat "
+for version in "${versions[@]}"; do
+    benchstat_cmd+=" benchmarks/${version}"
+done
+
+# Execute the comparison command and save it as a CSV
+echo "Comparing all versions with benchstat..."
+eval "$benchstat_cmd" > "benchmarks/combined_benchmarks_comparison"

internal/storage/binaryCoding.go

@@ -0,0 +1,56 @@
+package storage
+
+// Convert an Event to an EventProto
+func convertToProtoEvent(ev Event) *EventProto {
+	return &EventProto{
+		Key:               ev.Key,
+		EventHash:         ev.EventHash[:],
+		Level:             ev.Level,
+		ContentHashes:     hashesToBytes(ev.ContentHashes),
+		MetadataHashes:    hashesToBytes(ev.MetadataHashes),
+		HashOfParentEvent: ev.HashOfParentEvent[:],
+		HashOfRootEvent:   ev.HashOfRootEvent[:],
+		Temporary:         ev.Temporary,
+		FullTextSearch:    ev.FullTextSearch,
+	}
+}
+
+// Convert an EventProto to an Event
+func convertFromProtoEvent(pbEvent *EventProto) Event {
+	return Event{
+		Key:               pbEvent.GetKey(),
+		EventHash:         bytesToHash(pbEvent.GetEventHash()),
+		Level:             pbEvent.GetLevel(),
+		ContentHashes:     bytesToHashes(pbEvent.GetContentHashes()),
+		MetadataHashes:    bytesToHashes(pbEvent.GetMetadataHashes()),
+		HashOfParentEvent: bytesToHash(pbEvent.GetHashOfParentEvent()),
+		HashOfRootEvent:   bytesToHash(pbEvent.GetHashOfRootEvent()),
+		Temporary:         pbEvent.GetTemporary(),
+		FullTextSearch:    pbEvent.GetFullTextSearch(),
+	}
+}
+
+// Convert a slice of [64]byte hashes to a slice of byte slices
+func hashesToBytes(hashes [][64]byte) [][]byte {
+	result := make([][]byte, len(hashes))
+	for i, hash := range hashes {
+		result[i] = hash[:]
+	}
+	return result
+}
+
+// Convert a slice of byte slices to a slice of [64]byte hashes
+func bytesToHashes(byteSlices [][]byte) [][64]byte {
+	hashes := make([][64]byte, len(byteSlices))
+	for i, bytes := range byteSlices {
+		copy(hashes[i][:], bytes)
+	}
+	return hashes
+}
+
+// Convert a byte slice to a [64]byte hash
+func bytesToHash(b []byte) [64]byte {
+	var hash [64]byte
+	copy(hash[:], b)
+	return hash
+}