Merge pull request #51 from qmuntal/fs

Read/write external buffers from fs.FS

.github/workflows/test.yml

@@ -4,7 +4,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.14.x, 1.15.x]
+        go-version: [1.16.x, 1.17.x]
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:

decoder.go

@@ -8,34 +8,22 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"math"
+	"io/fs"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strings"
 	"unsafe"
 )
 
-const (
-	defaultMaxExternalBufferCount = 10
-	defaultMaxMemoryAllocation    = math.MaxUint32 // 4GB
-)
-
-// ReadHandler is the interface that wraps the ReadFullResource method.
-//
-// ReadFullResource should behaves as io.ReadFull in terms of reading the external resource.
-// The data already has the correct size so it can be used directly to store the read output.
-type ReadHandler interface {
-	ReadFullResource(uri string, data []byte) error
-}
-
 // Open will open a glTF or GLB file specified by name and return the Document.
 func Open(name string) (*Document, error) {
 	f, err := os.Open(name)
 	if err != nil {
 		return nil, err
 	}
 	defer f.Close()
-	dec := NewDecoder(f).WithReadHandler(&RelativeFileHandler{Dir: filepath.Dir(name)})
+	dec := NewDecoderFS(f, os.DirFS(filepath.Dir(name)))
 	doc := new(Document)
 	if err = dec.Decode(doc); err != nil {
 		doc = nil
@@ -44,29 +32,27 @@ func Open(name string) (*Document, error) {
 }
 
 // A Decoder reads and decodes glTF and GLB values from an input stream.
-// ReadHandler is called to read external resources.
+//
+// Only buffers with relative URIs will be read from Fsys.
+// Fsys is called to read external resources.
 type Decoder struct {
-	ReadHandler            ReadHandler
-	MaxExternalBufferCount int
-	MaxMemoryAllocation    uint64
-	r                      *bufio.Reader
+	Fsys fs.FS
+	r    *bufio.Reader
 }
 
-// NewDecoder returns a new decoder that reads from r
-// with relative external buffers support.
+// NewDecoder returns a new decoder that reads from r.
 func NewDecoder(r io.Reader) *Decoder {
 	return &Decoder{
-		ReadHandler:            new(RelativeFileHandler),
-		MaxExternalBufferCount: defaultMaxExternalBufferCount,
-		MaxMemoryAllocation:    defaultMaxMemoryAllocation,
-		r:                      bufio.NewReader(r),
+		r: bufio.NewReader(r),
 	}
 }
 
-// WithReadHandler sets the ReadHandler.
-func (d *Decoder) WithReadHandler(h ReadHandler) *Decoder {
-	d.ReadHandler = h
-	return d
+// NewDecoder returns a new decoder that reads from r.
+func NewDecoderFS(r io.Reader, fsys fs.FS) *Decoder {
+	return &Decoder{
+		Fsys: fsys,
+		r:    bufio.NewReader(r),
+	}
 }
 
 // Decode reads the next JSON-encoded value from its
@@ -92,27 +78,6 @@ func (d *Decoder) Decode(doc *Document) error {
 	return nil
 }
 
-func (d *Decoder) validateDocumentQuotas(doc *Document, isBinary bool) error {
-	var externalCount int
-	var allocs uint64
-	for _, b := range doc.Buffers {
-		allocs += uint64(b.ByteLength)
-		if !b.IsEmbeddedResource() {
-			externalCount++
-		}
-	}
-	if isBinary {
-		externalCount--
-	}
-	if externalCount > d.MaxExternalBufferCount {
-		return errors.New("gltf: External buffer count quota exceeded")
-	}
-	if allocs > d.MaxMemoryAllocation {
-		return errors.New("gltf: Memory allocation count quota exceeded")
-	}
-	return nil
-}
-
 func (d *Decoder) decodeDocument(doc *Document) (bool, error) {
 	glbHeader, err := d.readGLBHeader()
 	if err != nil {
@@ -131,9 +96,6 @@ func (d *Decoder) decodeDocument(doc *Document) (bool, error) {
 	}
 
 	err = jd.Decode(doc)
-	if err == nil {
-		err = d.validateDocumentQuotas(doc, isBinary)
-	}
 	return isBinary, err
 }
 
@@ -177,9 +139,17 @@ func (d *Decoder) decodeBuffer(buffer *Buffer) error {
 	var err error
 	if buffer.IsEmbeddedResource() {
 		buffer.Data, err = buffer.marshalData()
-	} else if err = validateBufferURI(buffer.URI); err == nil {
-		buffer.Data = make([]byte, buffer.ByteLength)
-		err = d.ReadHandler.ReadFullResource(buffer.URI, buffer.Data)
+	} else {
+		err = validateBufferURI(buffer.URI)
+		if err == nil && d.Fsys != nil {
+			uri, ok := sanitizeURI(buffer.URI)
+			if ok {
+				buffer.Data, err = fs.ReadFile(d.Fsys, uri)
+				if len(buffer.Data) > int(buffer.ByteLength) {
+					buffer.Data = buffer.Data[:buffer.ByteLength:buffer.ByteLength]
+				}
+			}
+		}
 	}
 	if err != nil {
 		buffer.Data = nil
@@ -216,3 +186,15 @@ func validateBufferURI(uri string) error {
 	}
 	return nil
 }
+
+func sanitizeURI(uri string) (string, bool) {
+	uri = strings.Replace(uri, "\\", "/", -1)
+	uri = strings.Replace(uri, "/./", "/", -1)
+	uri = strings.TrimPrefix(uri, "./")
+	u, err := url.Parse(uri)
+	if err != nil || u.IsAbs() {
+		// Only relative paths supported.
+		return "", false
+	}
+	return strings.TrimPrefix(u.RequestURI(), "/"), true
+}

decoder_test.go

@@ -3,10 +3,10 @@ package gltf
 import (
 	"bytes"
 	"encoding/json"
-	"io"
 	"io/ioutil"
 	"reflect"
 	"testing"
+	"testing/fstest"
 
 	"github.com/go-test/deep"
 )
@@ -140,29 +140,6 @@ func TestOpen(t *testing.T) {
 	}
 }
 
-type chunkedReader struct {
-	s []byte
-	n int
-}
-
-func (c *chunkedReader) Read(p []byte) (n int, err error) {
-	c.n++
-	if c.n == len(c.s)+1 {
-		return 0, io.EOF
-	}
-	p[0] = c.s[c.n-1 : c.n][0]
-	return 1, nil
-}
-
-type mockReadHandler struct {
-	Payload string
-}
-
-func (m mockReadHandler) ReadFullResource(uri string, data []byte) error {
-	copy(data, []byte(m.Payload))
-	return nil
-}
-
 func TestDecoder_decodeBuffer(t *testing.T) {
 	type args struct {
 		buffer *Buffer
@@ -177,8 +154,8 @@ func TestDecoder_decodeBuffer(t *testing.T) {
 		{"byteLength_0", &Decoder{}, args{&Buffer{ByteLength: 0, URI: "a.bin"}}, nil, true},
 		{"noURI", &Decoder{}, args{&Buffer{ByteLength: 1, URI: ""}}, nil, true},
 		{"invalidURI", &Decoder{}, args{&Buffer{ByteLength: 1, URI: "../a.bin"}}, nil, true},
-		{"noSchemeErr", NewDecoder(nil), args{&Buffer{ByteLength: 3, URI: "ftp://a.bin"}}, nil, true},
-		{"base", NewDecoder(nil).WithReadHandler(&mockReadHandler{"abcdfg"}), args{&Buffer{ByteLength: 6, URI: "a.bin"}}, []byte("abcdfg"), false},
+		{"noSchemeErr", NewDecoder(nil), args{&Buffer{ByteLength: 3, URI: "ftp://a.bin"}}, nil, false},
+		{"base", NewDecoderFS(nil, fstest.MapFS{"a.bin": &fstest.MapFile{Data: []byte("abcdfg")}}), args{&Buffer{ByteLength: 6, URI: "a.bin"}}, []byte("abcdfg"), false},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -237,9 +214,9 @@ func TestDecoder_Decode(t *testing.T) {
 		args    args
 		wantErr bool
 	}{
-		{"baseJSON", NewDecoder(bytes.NewBufferString("{\"buffers\": [{\"byteLength\": 1, \"URI\": \"a.bin\"}]}")).WithReadHandler(&mockReadHandler{"abcdfg"}), args{new(Document)}, false},
-		{"onlyGLBHeader", NewDecoder(bytes.NewBuffer([]byte{0x67, 0x6c, 0x54, 0x46, 0x02, 0x00, 0x00, 0x00, 0x40, 0x0b, 0x00, 0x00, 0x5c, 0x06, 0x00, 0x00, 0x4a, 0x53, 0x4f, 0x4e})).WithReadHandler(&mockReadHandler{"abcdfg"}), args{new(Document)}, true},
-		{"glbNoJSONChunk", NewDecoder(bytes.NewBuffer([]byte{0x67, 0x6c, 0x54, 0x46, 0x02, 0x00, 0x00, 0x00, 0x40, 0x0b, 0x00, 0x00, 0x5c, 0x06, 0x00, 0x00, 0x4a, 0x52, 0x4f, 0x4e})).WithReadHandler(&mockReadHandler{"abcdfg"}), args{new(Document)}, true},
+		{"baseJSON", NewDecoderFS(bytes.NewBufferString("{\"buffers\": [{\"byteLength\": 1, \"URI\": \"a.bin\"}]}"), fstest.MapFS{"a.bin": &fstest.MapFile{Data: []byte("abcdfg")}}), args{new(Document)}, false},
+		{"onlyGLBHeader", NewDecoderFS(bytes.NewBuffer([]byte{0x67, 0x6c, 0x54, 0x46, 0x02, 0x00, 0x00, 0x00, 0x40, 0x0b, 0x00, 0x00, 0x5c, 0x06, 0x00, 0x00, 0x4a, 0x53, 0x4f, 0x4e}), fstest.MapFS{"a.bin": &fstest.MapFile{Data: []byte("abcdfg")}}), args{new(Document)}, true},
+		{"glbNoJSONChunk", NewDecoderFS(bytes.NewBuffer([]byte{0x67, 0x6c, 0x54, 0x46, 0x02, 0x00, 0x00, 0x00, 0x40, 0x0b, 0x00, 0x00, 0x5c, 0x06, 0x00, 0x00, 0x4a, 0x52, 0x4f, 0x4e}), fstest.MapFS{"a.bin": &fstest.MapFile{Data: []byte("abcdfg")}}), args{new(Document)}, true},
 		{"empty", NewDecoder(bytes.NewBufferString("")), args{new(Document)}, true},
 		{"invalidJSON", NewDecoder(bytes.NewBufferString("{asset: {}}")), args{new(Document)}, true},
 		{"invalidBuffer", NewDecoder(bytes.NewBufferString("{\"buffers\": [{\"byteLength\": 0}]}")), args{new(Document)}, true},
@@ -253,40 +230,6 @@ func TestDecoder_Decode(t *testing.T) {
 	}
 }
 
-func TestDecoder_validateDocumentQuotas(t *testing.T) {
-	type args struct {
-		doc      *Document
-		isBinary bool
-	}
-	tests := []struct {
-		name    string
-		d       *Decoder
-		args    args
-		wantErr bool
-	}{
-		{
-			"exceedBuffers", &Decoder{MaxMemoryAllocation: 100000, MaxExternalBufferCount: 1},
-			args{&Document{Buffers: []*Buffer{{}, {}}}, false}, true,
-		}, {
-			"noExceedBuffers", &Decoder{MaxMemoryAllocation: 100000, MaxExternalBufferCount: 1},
-			args{&Document{Buffers: []*Buffer{{}, {}}}, true}, false,
-		}, {
-			"exceedAllocs", &Decoder{MaxMemoryAllocation: 10, MaxExternalBufferCount: 100},
-			args{&Document{Buffers: []*Buffer{{ByteLength: 11}}}, false}, true,
-		}, {
-			"noExceedAllocs", &Decoder{MaxMemoryAllocation: 11, MaxExternalBufferCount: 100},
-			args{&Document{Buffers: []*Buffer{{ByteLength: 11}}}, true}, false,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			if err := tt.d.validateDocumentQuotas(tt.args.doc, tt.args.isBinary); (err != nil) != tt.wantErr {
-				t.Errorf("Decoder.validateDocumentQuotas() error = %v, wantErr %v", err, tt.wantErr)
-			}
-		})
-	}
-}
-
 func TestSampler_Decode(t *testing.T) {
 
 	tests := []struct {

encode.go

@@ -5,15 +5,27 @@ import (
 	"encoding/json"
 	"errors"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 )
 
-// WriteHandler is the interface that wraps the Write method.
-//
-// WriteResource should behaves as io.Write in terms of reading the writing resource.
-type WriteHandler interface {
-	WriteResource(uri string, data []byte) error
+// A CreateFS provides access to a hierarchical file system.
+// Must follow the same naming convention as io/fs.FS.
+type CreateFS interface {
+	fs.FS
+	Create(name string) (io.WriteCloser, error)
+}
+
+// dirFS implements a file system (an fs.FS) for the tree of files rooted at the directory dir.
+type dirFS struct {
+	fs.FS
+	dir string
+}
+
+// Create creates or truncates the named file.
+func (d dirFS) Create(name string) (io.WriteCloser, error) {
+	return os.Create(d.dir + "/" + name)
 }
 
 // Save will save a document as a glTF with the specified by name.
@@ -31,7 +43,8 @@ func save(doc *Document, name string, asBinary bool) error {
 	if err != nil {
 		return err
 	}
-	e := NewEncoder(f).WithWriteHandler(&RelativeFileHandler{Dir: filepath.Dir(name)})
+	dir := filepath.Dir(name)
+	e := NewEncoderFS(f, dirFS{os.DirFS(dir), dir})
 	e.AsBinary = asBinary
 	if err := e.Encode(doc); err != nil {
 		f.Close()
@@ -40,27 +53,30 @@ func save(doc *Document, name string, asBinary bool) error {
 	return f.Close()
 }
 
-// An Encoder writes a GLTF to an output stream
-// with relative external buffers support.
+// An Encoder writes a glTF to an output stream.
+//
+// Only buffers with relative URIs will be written to Fsys.
 type Encoder struct {
-	AsBinary     bool
-	WriteHandler WriteHandler
-	w            io.Writer
+	AsBinary bool
+	Fsys     CreateFS
+	w        io.Writer
 }
 
 // NewEncoder returns a new encoder that writes to w as a normal glTF file.
 func NewEncoder(w io.Writer) *Encoder {
 	return &Encoder{
-		AsBinary:     true,
-		WriteHandler: new(RelativeFileHandler),
-		w:            w,
+		AsBinary: true,
+		w:        w,
 	}
 }
 
-// WithWriteHandler sets the WriteHandler.
-func (e *Encoder) WithWriteHandler(h WriteHandler) *Encoder {
-	e.WriteHandler = h
-	return e
+// NewEncoder returns a new encoder that writes to w as a normal glTF file.
+func NewEncoderFS(w io.Writer, fsys CreateFS) *Encoder {
+	return &Encoder{
+		AsBinary: true,
+		Fsys:     fsys,
+		w:        w,
+	}
 }
 
 // Encode writes the encoding of doc to the stream.
@@ -100,8 +116,22 @@ func (e *Encoder) encodeBuffer(buffer *Buffer) error {
 	if err := validateBufferURI(buffer.URI); err != nil {
 		return err
 	}
-
-	return e.WriteHandler.WriteResource(buffer.URI, buffer.Data)
+	if e.Fsys == nil {
+		return nil
+	}
+	uri, ok := sanitizeURI(buffer.URI)
+	if !ok {
+		return nil
+	}
+	w, err := e.Fsys.Create(uri)
+	if err != nil {
+		return err
+	}
+	_, err = w.Write(buffer.Data)
+	if err1 := w.Close(); err == nil {
+		err = err1
+	}
+	return err
 }
 
 func (e *Encoder) encodeBinary(doc *Document) (bool, error) {