From 5a6de5be79495c358a14fb8a6370da436ad4c9d3 Mon Sep 17 00:00:00 2001 From: i5heu Date: Thu, 16 May 2024 15:04:01 +0200 Subject: [PATCH] chore: Refactor OuroborosDB to use storage.StorageService instead of *storage.Storage and build index test as well as ChildToParent Index --- go.mod | 5 + go.sum | 6 +- ouroboros.go | 4 +- pkg/index/ChildrenToParents.go | 39 ++++ pkg/index/ParentsToChildren.go | 39 ++++ pkg/index/index.go | 45 +---- pkg/index/index_test.go | 133 +++++++++++++ pkg/mocks/StorageService.go | 344 +++++++++++++++++++++++++++++++++ pkg/storage/normalEvent.go | 6 +- pkg/storage/rootEvents.go | 8 +- pkg/storage/storage.go | 180 +++++++++++++++++ pkg/storage/storageService.go | 193 ++---------------- 12 files changed, 778 insertions(+), 224 deletions(-) create mode 100644 pkg/index/ChildrenToParents.go create mode 100644 pkg/index/ParentsToChildren.go create mode 100644 pkg/index/index_test.go create mode 100644 pkg/mocks/StorageService.go create mode 100644 pkg/storage/storage.go diff --git a/go.mod b/go.mod index f18ae26..212cdb7 100644 --- a/go.mod +++ b/go.mod @@ -5,12 +5,14 @@ go 1.21.5 require ( github.com/ipfs/boxo v0.19.0 github.com/shirou/gopsutil v3.21.11+incompatible + github.com/stretchr/testify v1.9.0 google.golang.org/protobuf v1.34.0 gopkg.in/yaml.v2 v2.4.0 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -23,9 +25,12 @@ require ( github.com/klauspost/compress v1.17.8 // indirect github.com/kr/pretty v0.2.0 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/stretchr/objx v0.5.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect go.opencensus.io v0.24.0 // indirect golang.org/x/net v0.25.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index 61eac5f..b5e4254 100644 --- a/go.sum +++ b/go.sum @@ -115,14 +115,16 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/ouroboros.go b/ouroboros.go index b53d626..54bcb9f 100644 --- a/ouroboros.go +++ b/ouroboros.go @@ -34,7 +34,7 @@ import ( var log *logrus.Logger type OuroborosDB struct { - DB *storage.Storage + DB storage.StorageService Index *index.Index config Config log *logrus.Logger @@ -81,7 +81,7 @@ func NewOuroborosDB(conf Config) (*OuroborosDB, error) { return nil, fmt.Errorf("error creating KeyValStore: %w", err) } - ss := storage.CreateStorage(kvStore) + ss := storage.NewStorage(kvStore) index := index.NewIndex(ss) ou := &OuroborosDB{ diff --git a/pkg/index/ChildrenToParents.go b/pkg/index/ChildrenToParents.go new file mode 100644 index 0000000..13d8a97 --- /dev/null +++ b/pkg/index/ChildrenToParents.go @@ -0,0 +1,39 @@ +package index + +import "github.com/i5heu/ouroboros-db/pkg/types" + +func (i *Index) RebuildChildrenToParents(allEvents []types.Event) error { + i.evChildToParentLock.Lock() + defer i.evChildToParentLock.Unlock() + + // Clear the existing mapping + clear(i.evChildToParent) + + // Rebuild the mapping + for _, event := range allEvents { + i.evChildToParent[event.EventHash] = event.HashOfParentEvent + } + + return nil +} + +func (i *Index) GetParentHashOfEvent(eventHash [64]byte) ([64]byte, bool) { + i.evChildToParentLock.RLock() + defer i.evChildToParentLock.RUnlock() + parentHash, exists := i.evChildToParent[eventHash] + return parentHash, exists +} + +func (i *Index) GetDirectParentOfEvent(eventHash [64]byte) (*types.Event, error) { + parentHash, exists := i.GetParentHashOfEvent(eventHash) + if !exists { + return nil, nil // No parent found + } + + parentEvent, err := i.ss.GetEvent(parentHash) + if err != nil { + return nil, err + } + + return &parentEvent, nil +} diff --git a/pkg/index/ParentsToChildren.go b/pkg/index/ParentsToChildren.go new file mode 100644 index 0000000..de7f368 --- /dev/null +++ b/pkg/index/ParentsToChildren.go @@ -0,0 +1,39 @@ +package index + +import "github.com/i5heu/ouroboros-db/pkg/types" + +func (i *Index) RebuildParentsToChildren(allEvents []types.Event) error { + i.evParentToChildLock.Lock() + defer i.evParentToChildLock.Unlock() + + clear(i.evParentToChild) + + for _, event := range allEvents { + i.evParentToChild[event.HashOfParentEvent] = append(i.evParentToChild[event.HashOfParentEvent], event.EventHash) + } + + return nil +} + +func (i *Index) GetChildrenHashesOfEvent(eventHash [64]byte) [][64]byte { + i.evParentToChildLock.RLock() + defer i.evParentToChildLock.RUnlock() + return i.evParentToChild[eventHash] +} + +func (i *Index) GetDirectChildrenOfEvent(eventHash [64]byte) ([]types.Event, error) { + childrenHashes := i.GetChildrenHashesOfEvent(eventHash) + children := make([]types.Event, 0) + + for _, childHash := range childrenHashes { + + child, err := i.ss.GetEvent(childHash) + if err != nil { + return nil, err + } + + children = append(children, child) + } + + return children, nil +} diff --git a/pkg/index/index.go b/pkg/index/index.go index dbc0570..3786df2 100644 --- a/pkg/index/index.go +++ b/pkg/index/index.go @@ -4,21 +4,24 @@ import ( "sync" "github.com/i5heu/ouroboros-db/pkg/storage" - "github.com/i5heu/ouroboros-db/pkg/types" ) type Index struct { - ss *storage.Storage + ss storage.StorageService evParentToChild map[[64]byte][][64]byte evParentToChildLock sync.RWMutex + evChildToParent map[[64]byte][64]byte + evChildToParentLock sync.RWMutex } -func NewIndex(ss *storage.Storage) *Index { +func NewIndex(ss storage.StorageService) *Index { i := &Index{ ss: ss, evParentToChildLock: sync.RWMutex{}, evParentToChild: make(map[[64]byte][][64]byte), + evChildToParentLock: sync.RWMutex{}, + evChildToParent: make(map[[64]byte][64]byte), } return i @@ -26,44 +29,14 @@ func NewIndex(ss *storage.Storage) *Index { func (i *Index) RebuildIndex() (uint64, error) { // get every event + // TODO we might need to optimize memory usage here events, err := i.ss.GetAllEvents() if err != nil { return 0, err } - i.evParentToChildLock.Lock() - defer i.evParentToChildLock.Unlock() - - // clear the map - clear(i.evParentToChild) - - // create a map of parent to children - for _, event := range events { - i.evParentToChild[event.HashOfParentEvent] = append(i.evParentToChild[event.HashOfParentEvent], event.EventHash) - } + i.RebuildParentsToChildren(events) + i.RebuildChildrenToParents(events) return uint64(len(events)), nil } - -func (i *Index) GetChildrenHashesOfEvent(eventHash [64]byte) [][64]byte { - i.evParentToChildLock.RLock() - defer i.evParentToChildLock.RUnlock() - return i.evParentToChild[eventHash] -} - -func (i *Index) GetDirectChildrenOfEvent(eventHash [64]byte) ([]types.Event, error) { - childrenHashes := i.GetChildrenHashesOfEvent(eventHash) - children := make([]types.Event, 0) - - for _, childHash := range childrenHashes { - - child, err := i.ss.GetEvent(childHash) - if err != nil { - return nil, err - } - - children = append(children, child) - } - - return children, nil -} diff --git a/pkg/index/index_test.go b/pkg/index/index_test.go new file mode 100644 index 0000000..a7324b6 --- /dev/null +++ b/pkg/index/index_test.go @@ -0,0 +1,133 @@ +package index + +import ( + "testing" + + "github.com/i5heu/ouroboros-db/pkg/mocks" + "github.com/i5heu/ouroboros-db/pkg/types" + "github.com/stretchr/testify/assert" +) + +func TestIndex_RebuildIndex(t *testing.T) { + mockStorageService := new(mocks.StorageService) + + events := []types.Event{ + {EventHash: [64]byte{1}, HashOfParentEvent: [64]byte{0}}, + {EventHash: [64]byte{2}, HashOfParentEvent: [64]byte{1}}, + } + + mockStorageService.On("GetAllEvents").Return(events, nil) + + index := NewIndex(mockStorageService) + + count, err := index.RebuildIndex() + + assert.NoError(t, err) + assert.Equal(t, uint64(2), count) + mockStorageService.AssertExpectations(t) +} + +func TestIndex_RebuildParentsToChildren(t *testing.T) { + mockStorageService := new(mocks.StorageService) + index := NewIndex(mockStorageService) + + parentHash := [64]byte{1} + childHash1 := [64]byte{2} + childHash2 := [64]byte{3} + + events := []types.Event{ + {EventHash: childHash1, HashOfParentEvent: parentHash}, + {EventHash: childHash2, HashOfParentEvent: parentHash}, + } + + err := index.RebuildParentsToChildren(events) + assert.NoError(t, err) + + index.evParentToChildLock.RLock() + defer index.evParentToChildLock.RUnlock() + assert.Contains(t, index.evParentToChild[parentHash], childHash1) + assert.Contains(t, index.evParentToChild[parentHash], childHash2) +} + +func TestIndex_GetChildrenHashesOfEvent(t *testing.T) { + index := &Index{ + evParentToChild: make(map[[64]byte][][64]byte), + } + + parentHash := [64]byte{1} + childHash := [64]byte{2} + index.evParentToChild[parentHash] = append(index.evParentToChild[parentHash], childHash) + + retrievedChildren := index.GetChildrenHashesOfEvent(parentHash) + assert.Contains(t, retrievedChildren, childHash) +} + +func TestIndex_GetDirectChildrenOfEvent(t *testing.T) { + mockStorageService := new(mocks.StorageService) + index := NewIndex(mockStorageService) + + parentHash := [64]byte{1} + childHash := [64]byte{2} + index.evParentToChild[parentHash] = append(index.evParentToChild[parentHash], childHash) + + childEvent := types.Event{EventHash: childHash} + mockStorageService.On("GetEvent", childHash).Return(childEvent, nil) + + retrievedChildren, err := index.GetDirectChildrenOfEvent(parentHash) + assert.NoError(t, err) + assert.Contains(t, retrievedChildren, childEvent) + mockStorageService.AssertExpectations(t) +} + +func TestIndex_RebuildChildrenToParents(t *testing.T) { + mockStorageService := new(mocks.StorageService) + index := NewIndex(mockStorageService) + + parentHash := [64]byte{1} + childHash1 := [64]byte{2} + childHash2 := [64]byte{3} + + events := []types.Event{ + {EventHash: childHash1, HashOfParentEvent: parentHash}, + {EventHash: childHash2, HashOfParentEvent: parentHash}, + } + + err := index.RebuildChildrenToParents(events) + assert.NoError(t, err) + + index.evChildToParentLock.RLock() + defer index.evChildToParentLock.RUnlock() + assert.Equal(t, parentHash, index.evChildToParent[childHash1]) + assert.Equal(t, parentHash, index.evChildToParent[childHash2]) +} + +func TestIndex_GetParentHashOfEvent(t *testing.T) { + index := &Index{ + evChildToParent: make(map[[64]byte][64]byte), + } + + parentHash := [64]byte{1} + childHash := [64]byte{2} + index.evChildToParent[childHash] = parentHash + + retrievedParentHash, exists := index.GetParentHashOfEvent(childHash) + assert.True(t, exists) + assert.Equal(t, parentHash, retrievedParentHash) +} + +func TestIndex_GetDirectParentOfEvent(t *testing.T) { + mockStorageService := new(mocks.StorageService) + index := NewIndex(mockStorageService) + + parentHash := [64]byte{1} + childHash := [64]byte{2} + index.evChildToParent[childHash] = parentHash + + parentEvent := types.Event{EventHash: parentHash} + mockStorageService.On("GetEvent", parentHash).Return(parentEvent, nil) + + retrievedParentEvent, err := index.GetDirectParentOfEvent(childHash) + assert.NoError(t, err) + assert.Equal(t, &parentEvent, retrievedParentEvent) + mockStorageService.AssertExpectations(t) +} diff --git a/pkg/mocks/StorageService.go b/pkg/mocks/StorageService.go new file mode 100644 index 0000000..ef514aa --- /dev/null +++ b/pkg/mocks/StorageService.go @@ -0,0 +1,344 @@ +// Code generated by mockery v2.43.0. DO NOT EDIT. + +package mocks + +import ( + storage "github.com/i5heu/ouroboros-db/pkg/storage" + mock "github.com/stretchr/testify/mock" + + types "github.com/i5heu/ouroboros-db/pkg/types" +) + +// StorageService is an autogenerated mock type for the StorageService type +type StorageService struct { + mock.Mock +} + +// Close provides a mock function with given fields: +func (_m *StorageService) Close() { + _m.Called() +} + +// CreateNewEvent provides a mock function with given fields: options +func (_m *StorageService) CreateNewEvent(options storage.EventOptions) (types.Event, error) { + ret := _m.Called(options) + + if len(ret) == 0 { + panic("no return value specified for CreateNewEvent") + } + + var r0 types.Event + var r1 error + if rf, ok := ret.Get(0).(func(storage.EventOptions) (types.Event, error)); ok { + return rf(options) + } + if rf, ok := ret.Get(0).(func(storage.EventOptions) types.Event); ok { + r0 = rf(options) + } else { + r0 = ret.Get(0).(types.Event) + } + + if rf, ok := ret.Get(1).(func(storage.EventOptions) error); ok { + r1 = rf(options) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// CreateRootEvent provides a mock function with given fields: title +func (_m *StorageService) CreateRootEvent(title string) (types.Event, error) { + ret := _m.Called(title) + + if len(ret) == 0 { + panic("no return value specified for CreateRootEvent") + } + + var r0 types.Event + var r1 error + if rf, ok := ret.Get(0).(func(string) (types.Event, error)); ok { + return rf(title) + } + if rf, ok := ret.Get(0).(func(string) types.Event); ok { + r0 = rf(title) + } else { + r0 = ret.Get(0).(types.Event) + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(title) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GarbageCollection provides a mock function with given fields: +func (_m *StorageService) GarbageCollection() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GarbageCollection") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// GetAllEvents provides a mock function with given fields: +func (_m *StorageService) GetAllEvents() ([]types.Event, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetAllEvents") + } + + var r0 []types.Event + var r1 error + if rf, ok := ret.Get(0).(func() ([]types.Event, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []types.Event); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.Event) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetAllRootEvents provides a mock function with given fields: +func (_m *StorageService) GetAllRootEvents() ([]types.Event, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetAllRootEvents") + } + + var r0 []types.Event + var r1 error + if rf, ok := ret.Get(0).(func() ([]types.Event, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []types.Event); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.Event) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetEvent provides a mock function with given fields: hashOfEvent +func (_m *StorageService) GetEvent(hashOfEvent [64]byte) (types.Event, error) { + ret := _m.Called(hashOfEvent) + + if len(ret) == 0 { + panic("no return value specified for GetEvent") + } + + var r0 types.Event + var r1 error + if rf, ok := ret.Get(0).(func([64]byte) (types.Event, error)); ok { + return rf(hashOfEvent) + } + if rf, ok := ret.Get(0).(func([64]byte) types.Event); ok { + r0 = rf(hashOfEvent) + } else { + r0 = ret.Get(0).(types.Event) + } + + if rf, ok := ret.Get(1).(func([64]byte) error); ok { + r1 = rf(hashOfEvent) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetFile provides a mock function with given fields: eventOfFile +func (_m *StorageService) GetFile(eventOfFile types.Event) ([]byte, error) { + ret := _m.Called(eventOfFile) + + if len(ret) == 0 { + panic("no return value specified for GetFile") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(types.Event) ([]byte, error)); ok { + return rf(eventOfFile) + } + if rf, ok := ret.Get(0).(func(types.Event) []byte); ok { + r0 = rf(eventOfFile) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(types.Event) error); ok { + r1 = rf(eventOfFile) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetMetadata provides a mock function with given fields: eventOfFile +func (_m *StorageService) GetMetadata(eventOfFile types.Event) ([]byte, error) { + ret := _m.Called(eventOfFile) + + if len(ret) == 0 { + panic("no return value specified for GetMetadata") + } + + var r0 []byte + var r1 error + if rf, ok := ret.Get(0).(func(types.Event) ([]byte, error)); ok { + return rf(eventOfFile) + } + if rf, ok := ret.Get(0).(func(types.Event) []byte); ok { + r0 = rf(eventOfFile) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]byte) + } + } + + if rf, ok := ret.Get(1).(func(types.Event) error); ok { + r1 = rf(eventOfFile) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetRootEventsWithTitle provides a mock function with given fields: title +func (_m *StorageService) GetRootEventsWithTitle(title string) ([]types.Event, error) { + ret := _m.Called(title) + + if len(ret) == 0 { + panic("no return value specified for GetRootEventsWithTitle") + } + + var r0 []types.Event + var r1 error + if rf, ok := ret.Get(0).(func(string) ([]types.Event, error)); ok { + return rf(title) + } + if rf, ok := ret.Get(0).(func(string) []types.Event); ok { + r0 = rf(title) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.Event) + } + } + + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(title) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// GetRootIndex provides a mock function with given fields: +func (_m *StorageService) GetRootIndex() ([]types.RootEventsIndex, error) { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for GetRootIndex") + } + + var r0 []types.RootEventsIndex + var r1 error + if rf, ok := ret.Get(0).(func() ([]types.RootEventsIndex, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []types.RootEventsIndex); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]types.RootEventsIndex) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// StoreFile provides a mock function with given fields: options +func (_m *StorageService) StoreFile(options storage.StoreFileOptions) (types.Event, error) { + ret := _m.Called(options) + + if len(ret) == 0 { + panic("no return value specified for StoreFile") + } + + var r0 types.Event + var r1 error + if rf, ok := ret.Get(0).(func(storage.StoreFileOptions) (types.Event, error)); ok { + return rf(options) + } + if rf, ok := ret.Get(0).(func(storage.StoreFileOptions) types.Event); ok { + r0 = rf(options) + } else { + r0 = ret.Get(0).(types.Event) + } + + if rf, ok := ret.Get(1).(func(storage.StoreFileOptions) error); ok { + r1 = rf(options) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// NewStorageService creates a new instance of StorageService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewStorageService(t interface { + mock.TestingT + Cleanup(func()) +}) *StorageService { + mock := &StorageService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/storage/normalEvent.go b/pkg/storage/normalEvent.go index 6106ddd..0e0cbc8 100644 --- a/pkg/storage/normalEvent.go +++ b/pkg/storage/normalEvent.go @@ -23,7 +23,7 @@ type EventOptions struct { FullTextSearch bool // optional } -func (ss *Storage) CreateNewEvent(options EventOptions) (types.Event, error) { +func (ss *storage) CreateNewEvent(options EventOptions) (types.Event, error) { // Create a new Event item := types.Event{ Key: []byte{}, @@ -87,7 +87,7 @@ func (ss *Storage) CreateNewEvent(options EventOptions) (types.Event, error) { return item, err } -func (ss *Storage) GetEvent(hashOfEvent [64]byte) (types.Event, error) { +func (ss *storage) GetEvent(hashOfEvent [64]byte) (types.Event, error) { // Read the EventChainItem from the keyValStore value, err := ss.kv.Read(GenerateKeyFromPrefixAndHash("Event:", hashOfEvent)) if err != nil { @@ -99,7 +99,7 @@ func (ss *Storage) GetEvent(hashOfEvent [64]byte) (types.Event, error) { return binaryCoder.ByteToEvent(value) } -func (ss *Storage) GetAllEvents() ([]types.Event, error) { +func (ss *storage) GetAllEvents() ([]types.Event, error) { items, err := ss.kv.GetItemsWithPrefix([]byte("Event:")) if err != nil { return nil, err diff --git a/pkg/storage/rootEvents.go b/pkg/storage/rootEvents.go index 0d9ddcf..75b02a0 100644 --- a/pkg/storage/rootEvents.go +++ b/pkg/storage/rootEvents.go @@ -21,7 +21,7 @@ func init() { // Same as Event struct in storageService.go but without some unnecessary fields -func (ss *Storage) CreateRootEvent(title string) (types.Event, error) { +func (ss *storage) CreateRootEvent(title string) (types.Event, error) { // Create a new IndexEvent item := types.Event{ Key: []byte{}, @@ -70,7 +70,7 @@ func (ss *Storage) CreateRootEvent(title string) (types.Event, error) { return item, err } -func (ss *Storage) GetAllRootEvents() ([]types.Event, error) { +func (ss *storage) GetAllRootEvents() ([]types.Event, error) { // Get all keys from the keyValStore rootIndex, err := ss.GetRootIndex() if err != nil { @@ -97,7 +97,7 @@ func (ss *Storage) GetAllRootEvents() ([]types.Event, error) { return rootEvents, nil } -func (ss *Storage) GetRootIndex() ([]types.RootEventsIndex, error) { +func (ss *storage) GetRootIndex() ([]types.RootEventsIndex, error) { // Get all keys from the keyValStore rootIndex, err := ss.kv.GetItemsWithPrefix([]byte("RootEvent:")) if err != nil { @@ -117,7 +117,7 @@ func (ss *Storage) GetRootIndex() ([]types.RootEventsIndex, error) { return revi, nil } -func (ss *Storage) GetRootEventsWithTitle(title string) ([]types.Event, error) { +func (ss *storage) GetRootEventsWithTitle(title string) ([]types.Event, error) { rootIndex, err := ss.kv.GetItemsWithPrefix([]byte("RootEvent:" + title + ":")) if err != nil { log.Fatalf("Error getting keys: %v", err) diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go new file mode 100644 index 0000000..6bf15cd --- /dev/null +++ b/pkg/storage/storage.go @@ -0,0 +1,180 @@ +package storage + +import ( + "fmt" + "log" + "sync" + + "github.com/i5heu/ouroboros-db/internal/keyValStore" + "github.com/i5heu/ouroboros-db/pkg/buzhashChunker" + "github.com/i5heu/ouroboros-db/pkg/types" +) + +type storage struct { + kv *keyValStore.KeyValStore +} + +type StoreFileOptions struct { + EventToAppendTo types.Event + Metadata []byte + File []byte + Temporary bool + FullTextSearch bool +} + +func NewStorage(kv *keyValStore.KeyValStore) StorageService { + return &storage{ + kv: kv, + } +} + +func (ss *storage) Close() { + ss.kv.Close() +} + +// will store the file in the chunkStore and create new Event as child of given event +func (ss *storage) StoreFile(options StoreFileOptions) (types.Event, error) { + // Validate options before proceeding + err := options.ValidateOptions() + if err != nil { + log.Fatalf("Error validating options: %v", err) + return types.Event{}, err + } + + // Create channels to handle asynchronous results and errors + fileChunkKeysChan := make(chan [][64]byte, 1) + metadataChunkKeysChan := make(chan [][64]byte, 1) + errorChan := make(chan error, 2) // buffer for two possible errors + + var wg sync.WaitGroup + wg.Add(2) + + // Store file data in chunk store asynchronously + go func() { + defer wg.Done() + keys, err := ss.storeDataInChunkStore(options.File) + if err != nil { + errorChan <- err + return + } + fileChunkKeysChan <- keys + }() + + // Store metadata in chunk store asynchronously + go func() { + defer wg.Done() + keys, err := ss.storeDataInChunkStore(options.Metadata) + if err != nil { + errorChan <- err + return + } + metadataChunkKeysChan <- keys + }() + + // Wait for both goroutines to complete + wg.Wait() + close(errorChan) + close(fileChunkKeysChan) + close(metadataChunkKeysChan) + + // Check for errors + for err := range errorChan { + log.Printf("Error in storing data: %v", err) + return types.Event{}, err + } + + // Retrieve results from channels + fileChunkKeys := <-fileChunkKeysChan + metadataChunkKeys := <-metadataChunkKeysChan + + // Create a new event + newEvent, err := ss.CreateNewEvent(EventOptions{ + ContentHashes: fileChunkKeys, + MetadataHashes: metadataChunkKeys, + HashOfParentEvent: options.EventToAppendTo.EventHash, + Temporary: options.Temporary, + FullTextSearch: options.FullTextSearch, + }) + if err != nil { + log.Fatalf("Error creating new event: %v", err) + return types.Event{}, err + } + + return newEvent, nil +} + +func (options *StoreFileOptions) ValidateOptions() error { + if options.EventToAppendTo.EventHash == [64]byte{} { + return fmt.Errorf("Error storing file: Parent event was not defined") + } + + if len(options.File) == 0 && len(options.Metadata) == 0 { + return fmt.Errorf("Error storing file: Both file and metadata are empty") + } + + if !options.Temporary && options.EventToAppendTo.Temporary { + return fmt.Errorf("Error storing file: Parent event is Temporary and can not have non-Temporary children") + } + + return nil +} + +func (ss *storage) GetFile(eventOfFile types.Event) ([]byte, error) { + file := []byte{} + + for _, hash := range eventOfFile.ContentHashes { + chunk, err := ss.kv.Read(hash[:]) + if err != nil { + return nil, fmt.Errorf("Error reading chunk from GetFile: %v", err) + } + + file = append(file, chunk...) + } + + return file, nil +} + +func (ss *storage) GetMetadata(eventOfFile types.Event) ([]byte, error) { + metadata := []byte{} + + for _, hash := range eventOfFile.MetadataHashes { + chunk, err := ss.kv.Read(hash[:]) + if err != nil { + return nil, fmt.Errorf("Error reading chunk from GetMetadata: %v", err) + } + + metadata = append(metadata, chunk...) + } + + return metadata, nil +} + +func (ss *storage) storeDataInChunkStore(data []byte) ([][64]byte, error) { + if len(data) == 0 { + return nil, fmt.Errorf("Error storing data: Data is empty") + } + + chunks, err := buzhashChunker.ChunkBytes(data) + if err != nil { + log.Fatalf("Error chunking data: %v", err) + return nil, err + } + + var keys [][64]byte + + for _, chunk := range chunks { + keys = append(keys, chunk.Hash) + } + + err = ss.kv.BatchWriteNonExistingChunks(chunks) + if err != nil { + log.Fatalf("Error writing chunks: %v", err) + return nil, err + } + + return keys, nil +} + +func (ss *storage) GarbageCollection() error { + return ss.kv.Clean() +} diff --git a/pkg/storage/storageService.go b/pkg/storage/storageService.go index b998c81..ce081f9 100644 --- a/pkg/storage/storageService.go +++ b/pkg/storage/storageService.go @@ -1,180 +1,19 @@ package storage -import ( - "fmt" - "log" - "sync" - - "github.com/i5heu/ouroboros-db/internal/keyValStore" - "github.com/i5heu/ouroboros-db/pkg/buzhashChunker" - "github.com/i5heu/ouroboros-db/pkg/types" -) - -type Storage struct { - kv *keyValStore.KeyValStore -} - -type StoreFileOptions struct { - EventToAppendTo types.Event - Metadata []byte - File []byte - Temporary bool - FullTextSearch bool -} - -func CreateStorage(kv *keyValStore.KeyValStore) *Storage { - return &Storage{ - kv: kv, - } -} - -func (ss *Storage) Close() { - ss.kv.Close() -} - -// will store the file in the chunkStore and create new Event as child of given event -func (ss *Storage) StoreFile(options StoreFileOptions) (types.Event, error) { - // Validate options before proceeding - err := options.ValidateOptions() - if err != nil { - log.Fatalf("Error validating options: %v", err) - return types.Event{}, err - } - - // Create channels to handle asynchronous results and errors - fileChunkKeysChan := make(chan [][64]byte, 1) - metadataChunkKeysChan := make(chan [][64]byte, 1) - errorChan := make(chan error, 2) // buffer for two possible errors - - var wg sync.WaitGroup - wg.Add(2) - - // Store file data in chunk store asynchronously - go func() { - defer wg.Done() - keys, err := ss.storeDataInChunkStore(options.File) - if err != nil { - errorChan <- err - return - } - fileChunkKeysChan <- keys - }() - - // Store metadata in chunk store asynchronously - go func() { - defer wg.Done() - keys, err := ss.storeDataInChunkStore(options.Metadata) - if err != nil { - errorChan <- err - return - } - metadataChunkKeysChan <- keys - }() - - // Wait for both goroutines to complete - wg.Wait() - close(errorChan) - close(fileChunkKeysChan) - close(metadataChunkKeysChan) - - // Check for errors - for err := range errorChan { - log.Printf("Error in storing data: %v", err) - return types.Event{}, err - } - - // Retrieve results from channels - fileChunkKeys := <-fileChunkKeysChan - metadataChunkKeys := <-metadataChunkKeysChan - - // Create a new event - newEvent, err := ss.CreateNewEvent(EventOptions{ - ContentHashes: fileChunkKeys, - MetadataHashes: metadataChunkKeys, - HashOfParentEvent: options.EventToAppendTo.EventHash, - Temporary: options.Temporary, - FullTextSearch: options.FullTextSearch, - }) - if err != nil { - log.Fatalf("Error creating new event: %v", err) - return types.Event{}, err - } - - return newEvent, nil -} - -func (options *StoreFileOptions) ValidateOptions() error { - if options.EventToAppendTo.EventHash == [64]byte{} { - return fmt.Errorf("Error storing file: Parent event was not defined") - } - - if len(options.File) == 0 && len(options.Metadata) == 0 { - return fmt.Errorf("Error storing file: Both file and metadata are empty") - } - - if !options.Temporary && options.EventToAppendTo.Temporary { - return fmt.Errorf("Error storing file: Parent event is Temporary and can not have non-Temporary children") - } - - return nil -} - -func (ss *Storage) GetFile(eventOfFile types.Event) ([]byte, error) { - file := []byte{} - - for _, hash := range eventOfFile.ContentHashes { - chunk, err := ss.kv.Read(hash[:]) - if err != nil { - return nil, fmt.Errorf("Error reading chunk from GetFile: %v", err) - } - - file = append(file, chunk...) - } - - return file, nil -} - -func (ss *Storage) GetMetadata(eventOfFile types.Event) ([]byte, error) { - metadata := []byte{} - - for _, hash := range eventOfFile.MetadataHashes { - chunk, err := ss.kv.Read(hash[:]) - if err != nil { - return nil, fmt.Errorf("Error reading chunk from GetMetadata: %v", err) - } - - metadata = append(metadata, chunk...) - } - - return metadata, nil -} - -func (ss *Storage) storeDataInChunkStore(data []byte) ([][64]byte, error) { - if len(data) == 0 { - return nil, fmt.Errorf("Error storing data: Data is empty") - } - - chunks, err := buzhashChunker.ChunkBytes(data) - if err != nil { - log.Fatalf("Error chunking data: %v", err) - return nil, err - } - - var keys [][64]byte - - for _, chunk := range chunks { - keys = append(keys, chunk.Hash) - } - - err = ss.kv.BatchWriteNonExistingChunks(chunks) - if err != nil { - log.Fatalf("Error writing chunks: %v", err) - return nil, err - } - - return keys, nil -} - -func (ss *Storage) GarbageCollection() error { - return ss.kv.Clean() +import "github.com/i5heu/ouroboros-db/pkg/types" + +// StorageService defines the methods for the storage service +type StorageService interface { + CreateRootEvent(title string) (types.Event, error) + GetAllRootEvents() ([]types.Event, error) + GetRootIndex() ([]types.RootEventsIndex, error) + GetRootEventsWithTitle(title string) ([]types.Event, error) + CreateNewEvent(options EventOptions) (types.Event, error) + GetEvent(hashOfEvent [64]byte) (types.Event, error) + StoreFile(options StoreFileOptions) (types.Event, error) + GetFile(eventOfFile types.Event) ([]byte, error) + GetMetadata(eventOfFile types.Event) ([]byte, error) + GarbageCollection() error + GetAllEvents() ([]types.Event, error) + Close() }