From 8e373303ff19551aa1b5944745cd82b0ae6db557 Mon Sep 17 00:00:00 2001 From: Henrique Dias Date: Tue, 22 Aug 2023 13:24:00 +0200 Subject: [PATCH] fix: lazy load version, build correct multipart (#305) --- add.go | 18 +++++++++++++++--- dag.go | 10 ++++------ go.mod | 13 +++++++------ go.sum | 28 +++++++++++++++------------- key.go | 5 ++++- mfs.go | 5 ++++- shell.go | 51 ++++++++++++++++++++++++++++++++++++++++++++++----- shell_test.go | 5 ++++- 8 files changed, 99 insertions(+), 36 deletions(-) diff --git a/add.go b/add.go index f736cef54..d82b586e3 100644 --- a/add.go +++ b/add.go @@ -65,7 +65,11 @@ func CidVersion(version int) AddOpts { func (s *Shell) Add(r io.Reader, options ...AddOpts) (string, error) { fr := files.NewReaderFile(r) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)}) - fileReader := files.NewMultiFileReader(slf, true) + + fileReader, err := s.newMultiFileReader(slf) + if err != nil { + return "", err + } var out object rb := s.Request("add") @@ -90,7 +94,11 @@ func (s *Shell) AddWithOpts(r io.Reader, pin bool, rawLeaves bool) (string, erro func (s *Shell) AddLink(target string) (string, error) { link := files.NewLinkFile(target, nil) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", link)}) - reader := files.NewMultiFileReader(slf, true) + + reader, err := s.newMultiFileReader(slf) + if err != nil { + return "", err + } var out object return out.Hash, s.Request("add").Body(reader).Exec(context.Background(), &out) @@ -108,7 +116,11 @@ func (s *Shell) AddDir(dir string, options ...AddOpts) (string, error) { return "", err } slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry(filepath.Base(dir), sf)}) - reader := files.NewMultiFileReader(slf, true) + + reader, err := s.newMultiFileReader(slf) + if err != nil { + return "", err + } rb := s.Request("add").Option("recursive", true) for _, option := range options { diff --git a/dag.go b/dag.go index 37c8aa3df..e1ac73418 100644 --- a/dag.go +++ b/dag.go @@ -51,7 +51,7 @@ func (s *Shell) DagPutWithOpts(data interface{}, opts ...options.DagPutOption) ( return "", err } - fileReader, err := dagToFilesReader(data) + fileReader, err := s.dagToFilesReader(data) if err != nil { return "", err } @@ -80,7 +80,7 @@ func (s *Shell) DagImportWithOpts(data interface{}, opts ...options.DagImportOpt return nil, err } - fileReader, err := dagToFilesReader(data) + fileReader, err := s.dagToFilesReader(data) if err != nil { return nil, err } @@ -130,7 +130,7 @@ func (s *Shell) DagImportWithOpts(data interface{}, opts ...options.DagImportOpt return &out, err } -func dagToFilesReader(data interface{}) (*files.MultiFileReader, error) { +func (s *Shell) dagToFilesReader(data interface{}) (*files.MultiFileReader, error) { var r io.Reader switch data := data.(type) { case *files.MultiFileReader: @@ -147,7 +147,5 @@ func dagToFilesReader(data interface{}) (*files.MultiFileReader, error) { fr := files.NewReaderFile(r) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)}) - fileReader := files.NewMultiFileReader(slf, true) - - return fileReader, nil + return s.newMultiFileReader(slf) } diff --git a/go.mod b/go.mod index 898deca4e..23c2d4087 100644 --- a/go.mod +++ b/go.mod @@ -3,19 +3,20 @@ go 1.19 module github.com/ipfs/go-ipfs-api require ( + github.com/blang/semver/v4 v4.0.0 github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 - github.com/ipfs/boxo v0.8.0 + github.com/ipfs/boxo v0.12.0 github.com/libp2p/go-libp2p v0.26.3 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-multiaddr v0.8.0 - github.com/multiformats/go-multibase v0.1.1 + github.com/multiformats/go-multibase v0.2.0 ) require ( github.com/benbjohnson/clock v1.3.0 // indirect github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect - github.com/ipfs/go-cid v0.4.0 // indirect + github.com/ipfs/go-cid v0.4.1 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -23,13 +24,13 @@ require ( github.com/mr-tron/base58 v1.2.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multicodec v0.8.1 // indirect - github.com/multiformats/go-multihash v0.2.1 // indirect + github.com/multiformats/go-multicodec v0.9.0 // indirect + github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect golang.org/x/crypto v0.6.0 // indirect golang.org/x/sys v0.6.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/protobuf v1.30.0 // indirect lukechampine.com/blake3 v1.1.7 // indirect ) diff --git a/go.sum b/go.sum index 64222e734..4091cd7a1 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927 h1:SKI1/fuSdodxmNNyVBR8d7X/HuLnRpvvFO0AgyQk764= github.com/cheekybits/is v0.0.0-20150225183255-68e9c0620927/go.mod h1:h/aW8ynjgkuj+NQRlZcDbAbM1ORAbXjXX77sX7T289U= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= @@ -11,10 +13,10 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2U github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/ipfs/boxo v0.8.0 h1:UdjAJmHzQHo/j3g3b1bAcAXCj/GM6iTwvSlBDvPBNBs= -github.com/ipfs/boxo v0.8.0/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA= -github.com/ipfs/go-cid v0.4.0 h1:a4pdZq0sx6ZSxbCizebnKiMCx/xI/aBBFlB73IgH4rA= -github.com/ipfs/go-cid v0.4.0/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= +github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= @@ -37,12 +39,12 @@ github.com/multiformats/go-base36 v0.2.0 h1:lFsAbNOGeKtuKozrtBsAkSVhv1p9D0/qedU9 github.com/multiformats/go-base36 v0.2.0/go.mod h1:qvnKE++v+2MWCfePClUEjE78Z7P2a1UV0xHgWc0hkp4= github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= -github.com/multiformats/go-multicodec v0.8.1 h1:ycepHwavHafh3grIbR1jIXnKCsFm0fqsfEOsJ8NtKE8= -github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= -github.com/multiformats/go-multihash v0.2.1 h1:aem8ZT0VA2nCHHk7bPJ1BjUbHNciqZC/d16Vve9l108= -github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.7 h1:sWSGR+f/eu5ABZA2ZpYKBILXTTs9JWpdEM/nEGOHFS8= @@ -50,7 +52,7 @@ github.com/multiformats/go-varint v0.0.7/go.mod h1:r8PUYw/fD/SjBCiKOoDlGF6QawOEL github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -59,8 +61,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= diff --git a/key.go b/key.go index 1e8975b4b..7c496d9ee 100644 --- a/key.go +++ b/key.go @@ -118,7 +118,10 @@ func (keyImportOpt) AllowAnyKeyType(allow bool) KeyImportOpt { func (s *Shell) KeyImport(ctx context.Context, name string, key io.Reader, options ...KeyImportOpt) error { fr := files.NewReaderFile(key) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)}) - fileReader := files.NewMultiFileReader(slf, true) + fileReader, err := s.newMultiFileReader(slf) + if err != nil { + return err + } rb := s.Request("key/import", name) for _, opt := range options { diff --git a/mfs.go b/mfs.go index 260de072f..3b0394446 100644 --- a/mfs.go +++ b/mfs.go @@ -338,7 +338,10 @@ func (s *Shell) FilesStat(ctx context.Context, path string, options ...FilesOpt) func (s *Shell) FilesWrite(ctx context.Context, path string, data io.Reader, options ...FilesOpt) error { fr := files.NewReaderFile(data) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)}) - fileReader := files.NewMultiFileReader(slf, true) + fileReader, err := s.newMultiFileReader(slf) + if err != nil { + return err + } rb := s.Request("files/write", path) for _, opt := range options { diff --git a/shell.go b/shell.go index 84c582221..710eed02c 100644 --- a/shell.go +++ b/shell.go @@ -13,8 +13,10 @@ import ( "os" "path" "strings" + "sync" "time" + "github.com/blang/semver/v4" files "github.com/ipfs/boxo/files" tar "github.com/ipfs/boxo/tar" homedir "github.com/mitchellh/go-homedir" @@ -35,6 +37,9 @@ const ( type Shell struct { url string httpcli gohttp.Client + + versionOnce sync.Once + version *semver.Version } func NewLocalShell() *Shell { @@ -121,6 +126,39 @@ func NewShellWithClient(url string, client *gohttp.Client) *Shell { return &sh } +// encodedAbsolutePathVersion is the version from which the absolute path header in +// multipart requests is %-encoded. Before this version, its sent raw. +var encodedAbsolutePathVersion = semver.MustParse("0.23.0-dev") + +func (s *Shell) loadRemoteVersion() error { + if s.version == nil { + version, _, err := s.Version() + if err != nil { + return err + } + + remoteVersion, err := semver.New(version) + if err != nil { + return err + } + + s.versionOnce.Do(func() { + s.version = remoteVersion + }) + } + + return nil +} + +func (s *Shell) newMultiFileReader(dir files.Directory) (*files.MultiFileReader, error) { + err := s.loadRemoteVersion() + if err != nil { + return nil, err + } + + return files.NewMultiFileReader(dir, true, s.version.LT(encodedAbsolutePathVersion)), nil +} + func (s *Shell) SetTimeout(d time.Duration) { s.httpcli.Timeout = d } @@ -368,7 +406,7 @@ func (s *Shell) PatchData(root string, set bool, data interface{}) (string, erro fr := files.NewReaderFile(read) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)}) - fileReader := files.NewMultiFileReader(slf, true) + fileReader := files.NewMultiFileReader(slf, true, s.version.LT(encodedAbsolutePathVersion)) var out object return out.Hash, s.Request("object/patch/"+cmd, root). @@ -470,7 +508,7 @@ func (s *Shell) BlockPut(block []byte, format, mhtype string, mhlen int) (string fr := files.NewBytesFile(block) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)}) - fileReader := files.NewMultiFileReader(slf, true) + fileReader := files.NewMultiFileReader(slf, true, s.version.LT(encodedAbsolutePathVersion)) return out.Key, s.Request("block/put"). Option("mhtype", mhtype). @@ -507,7 +545,7 @@ func (s *Shell) ObjectPut(obj *IpfsObject) (string, error) { fr := files.NewReaderFile(&data) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)}) - fileReader := files.NewMultiFileReader(slf, true) + fileReader := files.NewMultiFileReader(slf, true, s.version.LT(encodedAbsolutePathVersion)) var out object return out.Hash, s.Request("object/put"). @@ -530,10 +568,13 @@ func (s *Shell) PubSubSubscribe(topic string) (*PubSubSubscription, error) { } func (s *Shell) PubSubPublish(topic, data string) (err error) { - fr := files.NewReaderFile(bytes.NewReader([]byte(data))) slf := files.NewSliceDirectory([]files.DirEntry{files.FileEntry("", fr)}) - fileReader := files.NewMultiFileReader(slf, true) + + fileReader, err := s.newMultiFileReader(slf) + if err != nil { + return err + } encoder, _ := mbase.EncoderByName("base64url") resp, err := s.Request("pubsub/pub", encoder.Encode([]byte(topic))). diff --git a/shell_test.go b/shell_test.go index 440edcbbc..2726a9de8 100644 --- a/shell_test.go +++ b/shell_test.go @@ -496,8 +496,11 @@ func TestDagImportMultipleCARs(t *testing.T) { files.FileEntry("", files.NewReaderFile(bytes.NewReader(carFile2))), }) + fileReader, err := s.newMultiFileReader(slf) + is.Nil(err) + dagImported, err := s.DagImportWithOpts( - files.NewMultiFileReader(slf, true), + fileReader, options.Dag.Stats(true), options.Dag.Silent(false), )