From 393968237204be0e9e7973675b7a4e145c02a36a Mon Sep 17 00:00:00 2001 From: Lucas Molas Date: Mon, 20 Sep 2021 09:14:31 -0300 Subject: [PATCH] lock MFS root in node --- core/commands/files.go | 12 ++++++++++-- core/core.go | 1 + core/corerepo/gc.go | 8 ++++++-- core/node/core.go | 22 ++++++++++++++++++++++ core/node/groups.go | 1 + test-mfs-lock.bash | 13 +++++++++++++ 6 files changed, 53 insertions(+), 4 deletions(-) create mode 100755 test-mfs-lock.bash diff --git a/core/commands/files.go b/core/commands/files.go index 3f7e3e6b9a9..8abdff02a9c 100644 --- a/core/commands/files.go +++ b/core/commands/files.go @@ -9,6 +9,7 @@ import ( gopath "path" "sort" "strings" + "time" humanize "github.com/dustin/go-humanize" "github.com/ipfs/go-ipfs/core" @@ -768,6 +769,8 @@ stat' on the file or any of its ancestors. cmds.BoolOption(filesTruncateOptionName, "t", "Truncate the file to size zero before writing."), cmds.Int64Option(filesCountOptionName, "n", "Maximum number of bytes to read."), cmds.BoolOption(filesRawLeavesOptionName, "Use raw blocks for newly created leaf nodes. (experimental)"), + // FIXME: Fake lock for manual testing. Remove. + cmds.IntOption("lock-time", "lt", "[TESTING] timeout to hold the MFS lock while copying").WithDefault(0), cidVersionOption, hashOption, }, @@ -792,6 +795,11 @@ stat' on the file or any of its ancestors. if err != nil { return err } + filesRoot := nd.LockedFilesRoot.LockRoot() + defer nd.LockedFilesRoot.UnlockRoot() + // Keep the hold for a fake, arbitrary amount of time to test it manually. + timeout, _ := req.Options["lock-time"].(int) + defer time.Sleep(time.Duration(timeout) * time.Second) offset, _ := req.Options[filesOffsetOptionName].(int64) if offset < 0 { @@ -799,13 +807,13 @@ stat' on the file or any of its ancestors. } if mkParents { - err := ensureContainingDirectoryExists(nd.FilesRoot, path, prefix) + err := ensureContainingDirectoryExists(filesRoot, path, prefix) if err != nil { return err } } - fi, err := getFileHandle(nd.FilesRoot, path, create, prefix) + fi, err := getFileHandle(filesRoot, path, create, prefix) if err != nil { return err } diff --git a/core/core.go b/core/core.go index 888d3d78013..a7c0e2ea769 100644 --- a/core/core.go +++ b/core/core.go @@ -82,6 +82,7 @@ type IpfsNode struct { Reporter *metrics.BandwidthCounter `optional:"true"` Discovery mdns.Service `optional:"true"` FilesRoot *mfs.Root + LockedFilesRoot *node.LockedFilesRoot RecordValidator record.Validator // Online diff --git a/core/corerepo/gc.go b/core/corerepo/gc.go index 1fe888eb82c..9f01fb6d1ba 100644 --- a/core/corerepo/gc.go +++ b/core/corerepo/gc.go @@ -84,7 +84,9 @@ func BestEffortRoots(filesRoot *mfs.Root) ([]cid.Cid, error) { } func GarbageCollect(n *core.IpfsNode, ctx context.Context) error { - roots, err := BestEffortRoots(n.FilesRoot) + filesRoot := n.LockedFilesRoot.LockRoot() + defer n.LockedFilesRoot.UnlockRoot() + roots, err := BestEffortRoots(filesRoot) if err != nil { return err } @@ -148,7 +150,9 @@ func (e *MultiError) Error() string { } func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) <-chan gc.Result { - roots, err := BestEffortRoots(n.FilesRoot) + filesRoot := n.LockedFilesRoot.LockRoot() + defer n.LockedFilesRoot.UnlockRoot() + roots, err := BestEffortRoots(filesRoot) if err != nil { out := make(chan gc.Result) out <- gc.Result{Error: err} diff --git a/core/node/core.go b/core/node/core.go index cdb790bd22f..1243502accd 100644 --- a/core/node/core.go +++ b/core/node/core.go @@ -3,6 +3,7 @@ package node import ( "context" "fmt" + "sync" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" @@ -168,3 +169,24 @@ func Files(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo repo.Repo, dag format. return root, err } + +func LockedFiles(mctx helpers.MetricsCtx, lc fx.Lifecycle, root *mfs.Root) (*LockedFilesRoot, error) { + return &LockedFilesRoot{ + root: root, + lock: &sync.Mutex{}, + }, nil +} + +type LockedFilesRoot struct { + root *mfs.Root + lock *sync.Mutex +} + +func (lfr *LockedFilesRoot) LockRoot() *mfs.Root { + lfr.lock.Lock() + return lfr.root +} + +func (lfr *LockedFilesRoot) UnlockRoot() { + lfr.lock.Unlock() +} diff --git a/core/node/groups.go b/core/node/groups.go index 930a2151bef..cc7a3551955 100644 --- a/core/node/groups.go +++ b/core/node/groups.go @@ -296,6 +296,7 @@ var Core = fx.Options( fx.Provide(FetcherConfig), fx.Provide(Pinning), fx.Provide(Files), + fx.Provide(LockedFiles), ) func Networked(bcfg *BuildCfg, cfg *config.Config) fx.Option { diff --git a/test-mfs-lock.bash b/test-mfs-lock.bash new file mode 100755 index 00000000000..18573b19981 --- /dev/null +++ b/test-mfs-lock.bash @@ -0,0 +1,13 @@ +set -x +set -v + +# The daemon needs to be running, otherwise the commands will compete on their +# lock of the entire repo (not the MFS root) and fail. +ipfs swarm addrs > /dev/null || (echo "daemon not running" && exit 1) + +ipfs --version +ipfs files mkdir /test-lock/ +ipfs files rm /test-lock/ -r +((echo "content" | ./cmd/ipfs/ipfs files write --create --parents --truncate --lock-time 3 /test-lock/file) && echo "ipfs write lock released" &) +ipfs repo gc +# FIXME: This is a flaky test just to manually check the lock in ipfs write is blocking the GC.