Skip to content

Commit

Permalink
chore: Refactor OuroborosDB to use storage.StorageService instead of …
Browse files Browse the repository at this point in the history
…*storage.Storage and build index test as well as ChildToParent Index
  • Loading branch information
i5heu committed May 16, 2024
1 parent 56773b0 commit 5a6de5b
Show file tree
Hide file tree
Showing 12 changed files with 778 additions and 224 deletions.
5 changes: 5 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ go 1.21.5
require (
github.com/ipfs/boxo v0.19.0
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/stretchr/testify v1.9.0
google.golang.org/protobuf v1.34.0
gopkg.in/yaml.v2 v2.4.0
)

require (
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/ristretto v0.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
Expand All @@ -23,9 +25,12 @@ require (
github.com/klauspost/compress v1.17.8 // indirect
github.com/kr/pretty v0.2.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/objx v0.5.2 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
go.opencensus.io v0.24.0 // indirect
golang.org/x/net v0.25.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)

require (
Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -115,14 +115,16 @@ github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E=
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
Expand Down
4 changes: 2 additions & 2 deletions ouroboros.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ import (
var log *logrus.Logger

type OuroborosDB struct {
DB *storage.Storage
DB storage.StorageService
Index *index.Index
config Config
log *logrus.Logger
Expand Down Expand Up @@ -81,7 +81,7 @@ func NewOuroborosDB(conf Config) (*OuroborosDB, error) {
return nil, fmt.Errorf("error creating KeyValStore: %w", err)
}

ss := storage.CreateStorage(kvStore)
ss := storage.NewStorage(kvStore)
index := index.NewIndex(ss)

ou := &OuroborosDB{
Expand Down
39 changes: 39 additions & 0 deletions pkg/index/ChildrenToParents.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package index

import "github.com/i5heu/ouroboros-db/pkg/types"

func (i *Index) RebuildChildrenToParents(allEvents []types.Event) error {
i.evChildToParentLock.Lock()
defer i.evChildToParentLock.Unlock()

// Clear the existing mapping
clear(i.evChildToParent)

// Rebuild the mapping
for _, event := range allEvents {
i.evChildToParent[event.EventHash] = event.HashOfParentEvent
}

return nil
}

func (i *Index) GetParentHashOfEvent(eventHash [64]byte) ([64]byte, bool) {
i.evChildToParentLock.RLock()
defer i.evChildToParentLock.RUnlock()
parentHash, exists := i.evChildToParent[eventHash]
return parentHash, exists
}

func (i *Index) GetDirectParentOfEvent(eventHash [64]byte) (*types.Event, error) {
parentHash, exists := i.GetParentHashOfEvent(eventHash)
if !exists {
return nil, nil // No parent found
}

parentEvent, err := i.ss.GetEvent(parentHash)
if err != nil {
return nil, err
}

return &parentEvent, nil
}
39 changes: 39 additions & 0 deletions pkg/index/ParentsToChildren.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package index

import "github.com/i5heu/ouroboros-db/pkg/types"

func (i *Index) RebuildParentsToChildren(allEvents []types.Event) error {
i.evParentToChildLock.Lock()
defer i.evParentToChildLock.Unlock()

clear(i.evParentToChild)

for _, event := range allEvents {
i.evParentToChild[event.HashOfParentEvent] = append(i.evParentToChild[event.HashOfParentEvent], event.EventHash)
}

return nil
}

func (i *Index) GetChildrenHashesOfEvent(eventHash [64]byte) [][64]byte {
i.evParentToChildLock.RLock()
defer i.evParentToChildLock.RUnlock()
return i.evParentToChild[eventHash]
}

func (i *Index) GetDirectChildrenOfEvent(eventHash [64]byte) ([]types.Event, error) {
childrenHashes := i.GetChildrenHashesOfEvent(eventHash)
children := make([]types.Event, 0)

for _, childHash := range childrenHashes {

child, err := i.ss.GetEvent(childHash)
if err != nil {
return nil, err
}

children = append(children, child)
}

return children, nil
}
45 changes: 9 additions & 36 deletions pkg/index/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,66 +4,39 @@ import (
"sync"

"github.com/i5heu/ouroboros-db/pkg/storage"
"github.com/i5heu/ouroboros-db/pkg/types"
)

type Index struct {
ss *storage.Storage
ss storage.StorageService
evParentToChild map[[64]byte][][64]byte
evParentToChildLock sync.RWMutex
evChildToParent map[[64]byte][64]byte
evChildToParentLock sync.RWMutex
}

func NewIndex(ss *storage.Storage) *Index {
func NewIndex(ss storage.StorageService) *Index {

i := &Index{
ss: ss,
evParentToChildLock: sync.RWMutex{},
evParentToChild: make(map[[64]byte][][64]byte),
evChildToParentLock: sync.RWMutex{},
evChildToParent: make(map[[64]byte][64]byte),
}

return i
}

func (i *Index) RebuildIndex() (uint64, error) {
// get every event
// TODO we might need to optimize memory usage here
events, err := i.ss.GetAllEvents()
if err != nil {
return 0, err
}

i.evParentToChildLock.Lock()
defer i.evParentToChildLock.Unlock()

// clear the map
clear(i.evParentToChild)

// create a map of parent to children
for _, event := range events {
i.evParentToChild[event.HashOfParentEvent] = append(i.evParentToChild[event.HashOfParentEvent], event.EventHash)
}
i.RebuildParentsToChildren(events)
i.RebuildChildrenToParents(events)

return uint64(len(events)), nil
}

func (i *Index) GetChildrenHashesOfEvent(eventHash [64]byte) [][64]byte {
i.evParentToChildLock.RLock()
defer i.evParentToChildLock.RUnlock()
return i.evParentToChild[eventHash]
}

func (i *Index) GetDirectChildrenOfEvent(eventHash [64]byte) ([]types.Event, error) {
childrenHashes := i.GetChildrenHashesOfEvent(eventHash)
children := make([]types.Event, 0)

for _, childHash := range childrenHashes {

child, err := i.ss.GetEvent(childHash)
if err != nil {
return nil, err
}

children = append(children, child)
}

return children, nil
}
133 changes: 133 additions & 0 deletions pkg/index/index_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package index

import (
"testing"

"github.com/i5heu/ouroboros-db/pkg/mocks"
"github.com/i5heu/ouroboros-db/pkg/types"
"github.com/stretchr/testify/assert"
)

func TestIndex_RebuildIndex(t *testing.T) {
mockStorageService := new(mocks.StorageService)

events := []types.Event{
{EventHash: [64]byte{1}, HashOfParentEvent: [64]byte{0}},
{EventHash: [64]byte{2}, HashOfParentEvent: [64]byte{1}},
}

mockStorageService.On("GetAllEvents").Return(events, nil)

index := NewIndex(mockStorageService)

count, err := index.RebuildIndex()

assert.NoError(t, err)
assert.Equal(t, uint64(2), count)
mockStorageService.AssertExpectations(t)
}

func TestIndex_RebuildParentsToChildren(t *testing.T) {
mockStorageService := new(mocks.StorageService)
index := NewIndex(mockStorageService)

parentHash := [64]byte{1}
childHash1 := [64]byte{2}
childHash2 := [64]byte{3}

events := []types.Event{
{EventHash: childHash1, HashOfParentEvent: parentHash},
{EventHash: childHash2, HashOfParentEvent: parentHash},
}

err := index.RebuildParentsToChildren(events)
assert.NoError(t, err)

index.evParentToChildLock.RLock()
defer index.evParentToChildLock.RUnlock()
assert.Contains(t, index.evParentToChild[parentHash], childHash1)
assert.Contains(t, index.evParentToChild[parentHash], childHash2)
}

func TestIndex_GetChildrenHashesOfEvent(t *testing.T) {
index := &Index{
evParentToChild: make(map[[64]byte][][64]byte),
}

parentHash := [64]byte{1}
childHash := [64]byte{2}
index.evParentToChild[parentHash] = append(index.evParentToChild[parentHash], childHash)

retrievedChildren := index.GetChildrenHashesOfEvent(parentHash)
assert.Contains(t, retrievedChildren, childHash)
}

func TestIndex_GetDirectChildrenOfEvent(t *testing.T) {
mockStorageService := new(mocks.StorageService)
index := NewIndex(mockStorageService)

parentHash := [64]byte{1}
childHash := [64]byte{2}
index.evParentToChild[parentHash] = append(index.evParentToChild[parentHash], childHash)

childEvent := types.Event{EventHash: childHash}
mockStorageService.On("GetEvent", childHash).Return(childEvent, nil)

retrievedChildren, err := index.GetDirectChildrenOfEvent(parentHash)
assert.NoError(t, err)
assert.Contains(t, retrievedChildren, childEvent)
mockStorageService.AssertExpectations(t)
}

func TestIndex_RebuildChildrenToParents(t *testing.T) {
mockStorageService := new(mocks.StorageService)
index := NewIndex(mockStorageService)

parentHash := [64]byte{1}
childHash1 := [64]byte{2}
childHash2 := [64]byte{3}

events := []types.Event{
{EventHash: childHash1, HashOfParentEvent: parentHash},
{EventHash: childHash2, HashOfParentEvent: parentHash},
}

err := index.RebuildChildrenToParents(events)
assert.NoError(t, err)

index.evChildToParentLock.RLock()
defer index.evChildToParentLock.RUnlock()
assert.Equal(t, parentHash, index.evChildToParent[childHash1])
assert.Equal(t, parentHash, index.evChildToParent[childHash2])
}

func TestIndex_GetParentHashOfEvent(t *testing.T) {
index := &Index{
evChildToParent: make(map[[64]byte][64]byte),
}

parentHash := [64]byte{1}
childHash := [64]byte{2}
index.evChildToParent[childHash] = parentHash

retrievedParentHash, exists := index.GetParentHashOfEvent(childHash)
assert.True(t, exists)
assert.Equal(t, parentHash, retrievedParentHash)
}

func TestIndex_GetDirectParentOfEvent(t *testing.T) {
mockStorageService := new(mocks.StorageService)
index := NewIndex(mockStorageService)

parentHash := [64]byte{1}
childHash := [64]byte{2}
index.evChildToParent[childHash] = parentHash

parentEvent := types.Event{EventHash: parentHash}
mockStorageService.On("GetEvent", parentHash).Return(parentEvent, nil)

retrievedParentEvent, err := index.GetDirectParentOfEvent(childHash)
assert.NoError(t, err)
assert.Equal(t, &parentEvent, retrievedParentEvent)
mockStorageService.AssertExpectations(t)
}
Loading

0 comments on commit 5a6de5b

Please sign in to comment.