From 7eb118dfd0dc36cf44fcf53bc3f3558dac169eb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 14 Jan 2019 22:02:51 +0100 Subject: [PATCH 1/3] Enforce refs on files when using nocopy --- importer/helpers/dagbuilder.go | 12 ++++++++++-- importer/importer.go | 13 ++++++++++--- mod/dagmodifier.go | 6 +++++- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/importer/helpers/dagbuilder.go b/importer/helpers/dagbuilder.go index 891cdaa4d..a624217f8 100644 --- a/importer/helpers/dagbuilder.go +++ b/importer/helpers/dagbuilder.go @@ -2,6 +2,7 @@ package helpers import ( "context" + "errors" "io" "os" @@ -17,6 +18,8 @@ import ( ipld "github.com/ipfs/go-ipld-format" ) +var ErrMissingFsRef = errors.New("missing file path or URL, can't create filestore reference") + // DagBuilderHelper wraps together a bunch of objects needed to // efficiently create unixfs dag trees type DagBuilderHelper struct { @@ -71,7 +74,7 @@ type DagBuilderParams struct { // New generates a new DagBuilderHelper from the given params and a given // chunker.Splitter as data source. -func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper { +func (dbp *DagBuilderParams) New(spl chunker.Splitter) (*DagBuilderHelper, error) { db := &DagBuilderHelper{ dserv: dbp.Dagserv, spl: spl, @@ -87,7 +90,12 @@ func (dbp *DagBuilderParams) New(spl chunker.Splitter) *DagBuilderHelper { if dbp.URL != "" && dbp.NoCopy { db.fullPath = dbp.URL } - return db + + if dbp.NoCopy && db.fullPath == "" { // Enforce NoCopy + return nil, ErrMissingFsRef + } + + return db, nil } // prepareNext consumes the next item from the splitter and puts it diff --git a/importer/importer.go b/importer/importer.go index ecf016854..03f1c6048 100644 --- a/importer/importer.go +++ b/importer/importer.go @@ -18,8 +18,11 @@ func BuildDagFromReader(ds ipld.DAGService, spl chunker.Splitter) (ipld.Node, er Dagserv: ds, Maxlinks: h.DefaultLinksPerBlock, } - - return bal.Layout(dbp.New(spl)) + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + return bal.Layout(db) } // BuildTrickleDagFromReader creates a DAG given a DAGService and a Splitter @@ -30,5 +33,9 @@ func BuildTrickleDagFromReader(ds ipld.DAGService, spl chunker.Splitter) (ipld.N Maxlinks: h.DefaultLinksPerBlock, } - return trickle.Layout(dbp.New(spl)) + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + return trickle.Layout(db) } diff --git a/mod/dagmodifier.go b/mod/dagmodifier.go index a4c098052..1e2b2dcca 100644 --- a/mod/dagmodifier.go +++ b/mod/dagmodifier.go @@ -359,7 +359,11 @@ func (dm *DagModifier) appendData(nd ipld.Node, spl chunker.Splitter) (ipld.Node CidBuilder: dm.Prefix, RawLeaves: dm.RawLeaves, } - return trickle.Append(dm.ctx, nd, dbp.New(spl)) + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + return trickle.Append(dm.ctx, nd, db) default: return nil, ErrNotUnixfs } From 9e866f18e27d2c59de70057aab1f92afced0717e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 14 Jan 2019 22:14:06 +0100 Subject: [PATCH 2/3] fix tests after DagBuilder changes --- importer/balanced/balanced_test.go | 7 +++++- importer/trickle/trickle_test.go | 37 ++++++++++++++++++++++++++---- test/utils.go | 6 ++++- 3 files changed, 43 insertions(+), 7 deletions(-) diff --git a/importer/balanced/balanced_test.go b/importer/balanced/balanced_test.go index 1f135b781..b2069e3a9 100644 --- a/importer/balanced/balanced_test.go +++ b/importer/balanced/balanced_test.go @@ -27,7 +27,12 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter) (*dag.ProtoNode, err Maxlinks: h.DefaultLinksPerBlock, } - nd, err := Layout(dbp.New(spl)) + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + + nd, err := Layout(db) if err != nil { return nil, err } diff --git a/importer/trickle/trickle_test.go b/importer/trickle/trickle_test.go index 2067a24e3..2b6e0bd46 100644 --- a/importer/trickle/trickle_test.go +++ b/importer/trickle/trickle_test.go @@ -39,7 +39,12 @@ func buildTestDag(ds ipld.DAGService, spl chunker.Splitter, rawLeaves UseRawLeav RawLeaves: bool(rawLeaves), } - nd, err := Layout(dbp.New(spl)) + db, err := dbp.New(spl) + if err != nil { + return nil, err + } + + nd, err := Layout(db) if err != nil { return nil, err } @@ -503,7 +508,13 @@ func testAppend(t *testing.T, rawLeaves UseRawLeaves) { r := bytes.NewReader(should[nbytes/2:]) ctx := context.Background() - nnode, err := Append(ctx, nd, dbp.New(chunker.NewSizeSplitter(r, 500))) + + db, err := dbp.New(chunker.NewSizeSplitter(r, 500)) + if err != nil { + t.Fatal(err) + } + + nnode, err := Append(ctx, nd, db) if err != nil { t.Fatal(err) } @@ -564,7 +575,12 @@ func testMultipleAppends(t *testing.T, rawLeaves UseRawLeaves) { ctx := context.Background() for i := 0; i < len(should); i++ { - nnode, err := Append(ctx, nd, dbp.New(spl(bytes.NewReader(should[i:i+1])))) + db, err := dbp.New(spl(bytes.NewReader(should[i : i+1]))) + if err != nil { + t.Fatal(err) + } + + nnode, err := Append(ctx, nd, db) if err != nil { t.Fatal(err) } @@ -612,12 +628,23 @@ func TestAppendSingleBytesToEmpty(t *testing.T) { spl := chunker.SizeSplitterGen(500) ctx := context.Background() - nnode, err := Append(ctx, nd, dbp.New(spl(bytes.NewReader(data[:1])))) + + db, err := dbp.New(spl(bytes.NewReader(data[:1]))) + if err != nil { + t.Fatal(err) + } + + nnode, err := Append(ctx, nd, db) + if err != nil { + t.Fatal(err) + } + + db, err = dbp.New(spl(bytes.NewReader(data[1:]))) if err != nil { t.Fatal(err) } - nnode, err = Append(ctx, nnode, dbp.New(spl(bytes.NewReader(data[1:])))) + nnode, err = Append(ctx, nnode, db) if err != nil { t.Fatal(err) } diff --git a/test/utils.go b/test/utils.go index 98bce14cf..bb251bc11 100644 --- a/test/utils.go +++ b/test/utils.go @@ -67,7 +67,11 @@ func GetNode(t testing.TB, dserv ipld.DAGService, data []byte, opts NodeOpts) ip RawLeaves: opts.RawLeavesUsed, } - node, err := trickle.Layout(dbp.New(SizeSplitterGen(500)(in))) + db, err := dbp.New(SizeSplitterGen(500)(in)) + if err != nil { + t.Fatal(err) + } + node, err := trickle.Layout(db) if err != nil { t.Fatal(err) } From 4fcb63fa136afc699d8525383d66f64638049ae5 Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Wed, 23 Jan 2019 05:00:26 -0800 Subject: [PATCH 3/3] gx publish 1.2.13 --- .gx/lastpubver | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gx/lastpubver b/.gx/lastpubver index 43976c09a..3b5859fb3 100644 --- a/.gx/lastpubver +++ b/.gx/lastpubver @@ -1 +1 @@ -1.2.12: QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe +1.2.13: QmQReF8uaPBw6uKbDZQzaWyxCL4NuMMotcztXv1fKTSzkg diff --git a/package.json b/package.json index 694996eb4..557c3b991 100644 --- a/package.json +++ b/package.json @@ -79,6 +79,6 @@ "license": "", "name": "go-unixfs", "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", - "version": "1.2.12" + "version": "1.2.13" }