From 6c677812cf71e9d78a6d6ca5971c11a94adbcdd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 24 Oct 2018 00:24:19 +0200 Subject: [PATCH 01/25] [WIP] Refactor filename - file relation --- file.go | 56 ++++++++----------- file_test.go | 43 ++++++++------- is_hidden.go | 5 +- is_hidden_windows.go | 9 +++- linkfile.go | 32 ++++++----- multifilereader.go | 32 +++++++---- multifilereader_test.go | 111 +++++++++++++++++++++++++++++--------- multipartfile.go | 117 ++++++++++++++++++++++++++++++---------- readerfile.go | 38 +++++++------ serialfile.go | 55 +++++++------------ slicefile.go | 41 ++++++-------- webfile.go | 17 ++---- 12 files changed, 325 insertions(+), 231 deletions(-) diff --git a/file.go b/file.go index c5e8203..ceb60dd 100644 --- a/file.go +++ b/file.go @@ -7,56 +7,46 @@ import ( ) var ( - ErrNotDirectory = errors.New("Couldn't call NextFile(), this isn't a directory") - ErrNotReader = errors.New("This file is a directory, can't use Reader functions") + ErrNotDirectory = errors.New("couldn't call NextFile(), this isn't a directory") + ErrNotReader = errors.New("this file is a directory, can't use Reader functions") + + ErrNotSupported = errors.New("operation not supported") ) // File is an interface that provides functionality for handling // files/directories as values that can be supplied to commands. For -// directories, child files are accessed serially by calling `NextFile()`. +// directories, child files are accessed serially by calling `Files()` +// or `Walk()`. +// +// Read/Seek/Close methods are only valid for files +// Files/Walk methods are only valid for directories type File interface { - // Files implement ReadCloser, but can only be read from or closed if - // they are not directories - io.ReadCloser - - // FileName returns a filename associated with this file - FileName() string + io.Reader + io.Closer + io.Seeker - // FullPath returns the full path used when adding with this file - FullPath() string + // Size returns size of the + Size() (int64, error) // IsDirectory returns true if the File is a directory (and therefore - // supports calling `NextFile`) and false if the File is a normal file - // (and therefor supports calling `Read` and `Close`) + // supports calling `Files`/`Walk`) and false if the File is a normal file + // (and therefore supports calling `Read`/`Close`/`Seek`) IsDirectory() bool // NextFile returns the next child file available (if the File is a - // directory). It will return (nil, io.EOF) if no more files are + // directory). It will return io.EOF if no more files are // available. If the file is a regular file (not a directory), NextFile // will return a non-nil error. - NextFile() (File, error) + NextFile() (string, File, error) } -type StatFile interface { - File - - Stat() os.FileInfo -} - -type PeekFile interface { - SizeFile - - Peek(n int) File - Length() int -} - -type SizeFile interface { +// FileInfo exposes information on files in local filesystem +type FileInfo interface { File - Size() (int64, error) -} - -type FileInfo interface { + // AbsPath returns full/real file path. AbsPath() string + + // Stat returns os.Stat of this file Stat() os.FileInfo } diff --git a/file_test.go b/file_test.go index 733a8c8..60d1bc4 100644 --- a/file_test.go +++ b/file_test.go @@ -9,15 +9,14 @@ import ( ) func TestSliceFiles(t *testing.T) { - name := "testname" - files := []File{ - NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), - NewReaderFile("beep.txt", "beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), - NewReaderFile("boop.txt", "boop.txt", ioutil.NopCloser(strings.NewReader("boop")), nil), + files := []FileEntry{ + {NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), ""}, + {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), ""}, + {NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil), ""}, } buf := make([]byte, 20) - sf := NewSliceFile(name, name, files) + sf := NewSliceFile(files) if !sf.IsDirectory() { t.Fatal("SliceFile should always be a directory") @@ -31,7 +30,7 @@ func TestSliceFiles(t *testing.T) { t.Fatal("Shouldn't be able to call `Close` on a SliceFile") } - file, err := sf.NextFile() + _, file, err := sf.NextFile() if file == nil || err != nil { t.Fatal("Expected a file and nil error") } @@ -40,16 +39,16 @@ func TestSliceFiles(t *testing.T) { t.Fatal("NextFile got a file in the wrong order") } - file, err = sf.NextFile() + _, file, err = sf.NextFile() if file == nil || err != nil { t.Fatal("Expected a file and nil error") } - file, err = sf.NextFile() + _, file, err = sf.NextFile() if file == nil || err != nil { t.Fatal("Expected a file and nil error") } - file, err = sf.NextFile() + _, file, err = sf.NextFile() if file != nil || err != io.EOF { t.Fatal("Expected a nil file and io.EOF") } @@ -57,13 +56,13 @@ func TestSliceFiles(t *testing.T) { func TestReaderFiles(t *testing.T) { message := "beep boop" - rf := NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader(message)), nil) + rf := NewReaderFile(ioutil.NopCloser(strings.NewReader(message)), nil) buf := make([]byte, len(message)) if rf.IsDirectory() { t.Fatal("ReaderFile should never be a directory") } - file, err := rf.NextFile() + _, file, err := rf.NextFile() if file != nil || err != ErrNotDirectory { t.Fatal("Expected a nil file and ErrNotDirectory") } @@ -114,17 +113,17 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err := NewFileFromPart(part) + mpname, mpf, err := newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if mpf.IsDirectory() { t.Fatal("Expected file to not be a directory") } - if mpf.FileName() != "name" { + if mpname != "name" { t.Fatal("Expected filename to be \"name\"") } - if file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory { + if _, file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory { t.Fatal("Expected a nil file and ErrNotDirectory") } if n, err := mpf.Read(buf); n != 4 || !(err == io.EOF || err == nil) { @@ -139,14 +138,14 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if !mpf.IsDirectory() { t.Fatal("Expected file to be a directory") } - if mpf.FileName() != "dir" { + if mpname != "dir" { t.Fatal("Expected filename to be \"dir\"") } if n, err := mpf.Read(buf); n > 0 || err != ErrNotReader { @@ -161,15 +160,15 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("dir/", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if mpf.IsDirectory() { t.Fatal("Expected file, got directory") } - if mpf.FileName() != "dir/nested" { - t.Fatalf("Expected filename to be \"nested\", got %s", mpf.FileName()) + if mpname != "nested" { + t.Fatalf("Expected filename to be \"nested\", got %s", mpname) } if n, err := mpf.Read(buf); n != 12 || !(err == nil || err == io.EOF) { t.Fatalf("expected to be able to read 12 bytes from file: %s (got %d)", err, n) @@ -183,14 +182,14 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("dir/", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if mpf.IsDirectory() { t.Fatal("Expected file to be a symlink") } - if mpf.FileName() != "dir/simlynk" { + if mpname != "simlynk" { t.Fatal("Expected filename to be \"dir/simlynk\"") } slink, ok := mpf.(*Symlink) diff --git a/is_hidden.go b/is_hidden.go index b036068..d5bc886 100644 --- a/is_hidden.go +++ b/is_hidden.go @@ -7,9 +7,8 @@ import ( "strings" ) -func IsHidden(f File) bool { - - fName := filepath.Base(f.FileName()) +func IsHidden(name string, f File) bool { + fName := filepath.Base(name) if strings.HasPrefix(fName, ".") && len(fName) > 1 { return true diff --git a/is_hidden_windows.go b/is_hidden_windows.go index 7679433..40f40ae 100644 --- a/is_hidden_windows.go +++ b/is_hidden_windows.go @@ -9,7 +9,7 @@ import ( windows "golang.org/x/sys/windows" ) -func IsHidden(f File) bool { +func IsHidden(name string, f File) bool { fName := filepath.Base(f.FileName()) @@ -17,7 +17,12 @@ func IsHidden(f File) bool { return true } - p, e := windows.UTF16PtrFromString(f.FullPath()) + fi, ok := f.(FileInfo) + if !ok { + return false + } + + p, e := windows.UTF16PtrFromString(fi.AbsPath()) if e != nil { return false } diff --git a/linkfile.go b/linkfile.go index 18466f4..0182d39 100644 --- a/linkfile.go +++ b/linkfile.go @@ -7,7 +7,6 @@ import ( ) type Symlink struct { - name string path string Target string stat os.FileInfo @@ -15,9 +14,8 @@ type Symlink struct { reader io.Reader } -func NewLinkFile(name, path, target string, stat os.FileInfo) File { +func NewLinkFile(path, target string, stat os.FileInfo) File { return &Symlink{ - name: name, path: path, Target: target, stat: stat, @@ -29,22 +27,30 @@ func (lf *Symlink) IsDirectory() bool { return false } -func (lf *Symlink) NextFile() (File, error) { - return nil, io.EOF +func (lf *Symlink) NextFile() (string, File, error) { + return "", nil, ErrNotDirectory } -func (f *Symlink) FileName() string { - return f.name -} +func (lf *Symlink) Close() error { + if c, ok := lf.reader.(io.Closer); ok { + return c.Close() + } -func (f *Symlink) Close() error { return nil } -func (f *Symlink) FullPath() string { - return f.path +func (lf *Symlink) Read(b []byte) (int, error) { + return lf.reader.Read(b) +} + +func (lf *Symlink) Seek(offset int64, whence int) (int64, error) { + if s, ok := lf.reader.(io.Seeker); ok { + return s.Seek(offset, whence) + } + + return 0, ErrNotSupported } -func (f *Symlink) Read(b []byte) (int, error) { - return f.reader.Read(b) +func (lf *Symlink) Size() (int64, error) { + return 0, ErrNotSupported } diff --git a/multifilereader.go b/multifilereader.go index 4833e8d..7863c9d 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -7,6 +7,7 @@ import ( "mime/multipart" "net/textproto" "net/url" + "path" "sync" ) @@ -15,7 +16,10 @@ import ( type MultiFileReader struct { io.Reader - files []File + // directory stack for NextFile + files []File + path []string + currentFile io.Reader buf bytes.Buffer mpWriter *multipart.Writer @@ -33,6 +37,7 @@ type MultiFileReader struct { func NewMultiFileReader(file File, form bool) *MultiFileReader { mfr := &MultiFileReader{ files: []File{file}, + path: []string{""}, form: form, mutex: &sync.Mutex{}, } @@ -53,6 +58,8 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // if the current file isn't set, advance to the next file if mfr.currentFile == nil { var file File + var name string + for file == nil { if len(mfr.files) == 0 { mfr.mpWriter.Close() @@ -60,40 +67,43 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { return mfr.buf.Read(buf) } - nextfile, err := mfr.files[len(mfr.files)-1].NextFile() + nextName, nextFile, err := mfr.files[len(mfr.files)-1].NextFile() if err == io.EOF { mfr.files = mfr.files[:len(mfr.files)-1] + mfr.path = mfr.path[:len(mfr.path)-1] continue } else if err != nil { return 0, err } - file = nextfile + file = nextFile + name = nextName } // handle starting a new file part if !mfr.closed { + mfr.currentFile = file + + // write the boundary and headers + header := make(textproto.MIMEHeader) + filename := url.QueryEscape(path.Join(path.Join(mfr.path...), name)) + header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) + var contentType string if _, ok := file.(*Symlink); ok { contentType = "application/symlink" } else if file.IsDirectory() { mfr.files = append(mfr.files, file) + mfr.path = append(mfr.path, name) contentType = "application/x-directory" } else { // otherwise, use the file as a reader to read its contents contentType = "application/octet-stream" } - mfr.currentFile = file - - // write the boundary and headers - header := make(textproto.MIMEHeader) - filename := url.QueryEscape(file.FileName()) - header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) - header.Set("Content-Type", contentType) - if rf, ok := file.(*ReaderFile); ok { + if rf, ok := file.(FileInfo); ok { header.Set("abspath", rf.AbsPath()) } diff --git a/multifilereader_test.go b/multifilereader_test.go index 3d2c978..18876ef 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -8,35 +8,93 @@ import ( "testing" ) -func TestOutput(t *testing.T) { - text := "Some text! :)" - fileset := []File{ - NewReaderFile("file.txt", "file.txt", ioutil.NopCloser(strings.NewReader(text)), nil), - NewSliceFile("boop", "boop", []File{ - NewReaderFile("boop/a.txt", "boop/a.txt", ioutil.NopCloser(strings.NewReader("bleep")), nil), - NewReaderFile("boop/b.txt", "boop/b.txt", ioutil.NopCloser(strings.NewReader("bloop")), nil), - }), - NewReaderFile("beep.txt", "beep.txt", ioutil.NopCloser(strings.NewReader("beep")), nil), - } - sf := NewSliceFile("", "", fileset) - buf := make([]byte, 20) +var text = "Some text! :)" + +func getTestMultiFileReader() *MultiFileReader { + fileset := []FileEntry{ + {NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil), "file.txt"}, + {NewSliceFile([]FileEntry{ + {NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil), "a.txt"}, + {NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil), "b.txt"}, + }), "boop"}, + {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), "beep.txt"}, + } + sf := NewSliceFile(fileset) // testing output by reading it with the go stdlib "mime/multipart" Reader - mfr := NewMultiFileReader(sf, true) + return NewMultiFileReader(sf, true) +} + +func TestMultiFileReaderToMultiFile(t *testing.T) { + mfr := getTestMultiFileReader() mpReader := multipart.NewReader(mfr, mfr.Boundary()) + mf, err := NewFileFromPartReader(mpReader, multipartFormdataType) + if err != nil { + t.Fatal(err) + } + + if !mf.IsDirectory() { + t.Fatal("Expected a directory") + } + + fn, f, err := mf.NextFile() + if fn != "file.txt" || f == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } + + dn, d, err := mf.NextFile() + if dn != "boop" || d == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } + + if !d.IsDirectory() { + t.Fatal("Expected a directory") + } + + cfn, cf, err := d.NextFile() + if cfn != "a.txt" || cf == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } + + cfn, cf, err = d.NextFile() + if cfn != "b.txt" || cf == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } + + cfn, cf, err = d.NextFile() + if cfn != "" || cf != nil || err != io.EOF { + t.Fatal("NextFile returned unexpected data") + } + + // try to break internal state + cfn, cf, err = d.NextFile() + if cfn != "" || cf != nil || err != io.EOF { + t.Fatal("NextFile returned unexpected data") + } + + fn, f, err = mf.NextFile() + if fn != "beep.txt" || f == nil || err != nil { + t.Fatal("NextFile returned unexpected data") + } +} + +func TestOutput(t *testing.T) { + mfr := getTestMultiFileReader() + mpReader := &peekReader{r: multipart.NewReader(mfr, mfr.Boundary())} + buf := make([]byte, 20) part, err := mpReader.NextPart() if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err := NewFileFromPart(part) + mpname, mpf, err := newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if mpf.IsDirectory() { t.Fatal("Expected file to not be a directory") } - if mpf.FileName() != "file.txt" { + if mpname != "file.txt" { t.Fatal("Expected filename to be \"file.txt\"") } if n, err := mpf.Read(buf); n != len(text) || err != nil { @@ -50,14 +108,14 @@ func TestOutput(t *testing.T) { if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } if !mpf.IsDirectory() { t.Fatal("Expected file to be a directory") } - if mpf.FileName() != "boop" { + if mpname != "boop" { t.Fatal("Expected filename to be \"boop\"") } @@ -65,33 +123,33 @@ func TestOutput(t *testing.T) { if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - child, err := NewFileFromPart(part) + cname, child, err := newFileFromPart("boop", part, mpReader) if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } if child.IsDirectory() { t.Fatal("Expected file to not be a directory") } - if child.FileName() != "boop/a.txt" { - t.Fatal("Expected filename to be \"some/file/path\"") + if cname != "a.txt" { + t.Fatal("Expected filename to be \"a.txt\"") } part, err = mpReader.NextPart() if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - child, err = NewFileFromPart(part) + cname, child, err = newFileFromPart("boop", part, mpReader) if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } if child.IsDirectory() { t.Fatal("Expected file to not be a directory") } - if child.FileName() != "boop/b.txt" { - t.Fatal("Expected filename to be \"some/file/path\"") + if cname != "b.txt" { + t.Fatal("Expected filename to be \"b.txt\"") } - child, err = mpf.NextFile() + cname, child, err = mpf.NextFile() if child != nil || err != io.EOF { t.Fatal("Expected to get (nil, io.EOF)") } @@ -100,10 +158,13 @@ func TestOutput(t *testing.T) { if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpf, err = NewFileFromPart(part) + mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } + if mpname != "beep.txt" { + t.Fatal("Expected filename to be \"b.txt\"") + } part, err = mpReader.NextPart() if part != nil || err != io.EOF { diff --git a/multipartfile.go b/multipartfile.go index 8a4b0b2..a304163 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -1,11 +1,13 @@ package files import ( + "errors" "io" "io/ioutil" "mime" "mime/multipart" "net/url" + "path" ) const ( @@ -19,19 +21,36 @@ const ( contentTypeHeader = "Content-Type" ) +var ErrPartOutsideParent = errors.New("file outside parent dir") + // MultipartFile implements File, and is created from a `multipart.Part`. // It can be either a directory or file (checked by calling `IsDirectory()`). type MultipartFile struct { File Part *multipart.Part - Reader *multipart.Reader + Reader PartReader Mediatype string } -func NewFileFromPart(part *multipart.Part) (File, error) { +func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, error) { + f := &MultipartFile{ + Reader: &peekReader{r: reader}, + Mediatype: mediatype, + } + + return f, nil +} + +func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (string, File, error) { f := &MultipartFile{ - Part: part, + Part: part, + Reader: reader, + } + + dir, base := path.Split(f.fileName()) + if path.Clean(dir) != path.Clean(parent) { + return "", nil, ErrPartOutsideParent } contentType := part.Header.Get(contentTypeHeader) @@ -39,54 +58,64 @@ func NewFileFromPart(part *multipart.Part) (File, error) { case applicationSymlink: out, err := ioutil.ReadAll(part) if err != nil { - return nil, err + return "", nil, err } - return &Symlink{ + return base, &Symlink{ Target: string(out), - name: f.FileName(), }, nil case "": // default to application/octet-stream fallthrough case applicationFile: - return &ReaderFile{ - reader: part, - filename: f.FileName(), - abspath: part.Header.Get("abspath"), - fullpath: f.FullPath(), + return base, &ReaderFile{ + reader: part, + abspath: part.Header.Get("abspath"), }, nil } var err error f.Mediatype, _, err = mime.ParseMediaType(contentType) if err != nil { - return nil, err + return "", nil, err } - return f, nil + return base, f, nil } func (f *MultipartFile) IsDirectory() bool { return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory } -func (f *MultipartFile) NextFile() (File, error) { +func (f *MultipartFile) NextFile() (string, File, error) { if !f.IsDirectory() { - return nil, ErrNotDirectory + return "", nil, ErrNotDirectory + } + if f.Reader == nil { + return "", nil, io.EOF + } + part, err := f.Reader.NextPart() + if err != nil { + return "", nil, err } - if f.Reader != nil { - part, err := f.Reader.NextPart() - if err != nil { - return nil, err - } - return NewFileFromPart(part) + name, cf, err := newFileFromPart(f.fileName(), part, f.Reader) + if err != ErrPartOutsideParent { + return name, cf, err } - return nil, io.EOF + // we read too much, try to fix this + pr, ok := f.Reader.(*peekReader) + if !ok { + return "", nil, errors.New("cannot undo NextPart") + } + + if err := pr.put(part); err != nil { + return "", nil, err + } + return "", nil, io.EOF } -func (f *MultipartFile) FileName() string { +func (f *MultipartFile) fileName() string { if f == nil || f.Part == nil { return "" } @@ -99,10 +128,6 @@ func (f *MultipartFile) FileName() string { return filename } -func (f *MultipartFile) FullPath() string { - return f.FileName() -} - func (f *MultipartFile) Read(p []byte) (int, error) { if f.IsDirectory() { return 0, ErrNotReader @@ -116,3 +141,41 @@ func (f *MultipartFile) Close() error { } return f.Part.Close() } + +func (f *MultipartFile) Seek(offset int64, whence int) (int64, error) { + if f.IsDirectory() { + return 0, ErrNotReader + } + return 0, ErrNotReader +} + +func (f *MultipartFile) Size() (int64, error) { + return 0, ErrNotReader +} + +type PartReader interface { + NextPart() (*multipart.Part, error) +} + +type peekReader struct { + r PartReader + next *multipart.Part +} + +func (pr *peekReader) NextPart() (*multipart.Part, error) { + if pr.next != nil { + p := pr.next + pr.next = nil + return p, nil + } + + return pr.r.NextPart() +} + +func (pr *peekReader) put(p *multipart.Part) error { + if pr.next != nil { + return errors.New("cannot put multiple parts") + } + pr.next = p + return nil +} diff --git a/readerfile.go b/readerfile.go index 8636414..e40032c 100644 --- a/readerfile.go +++ b/readerfile.go @@ -10,40 +10,30 @@ import ( // ReaderFile is a implementation of File created from an `io.Reader`. // ReaderFiles are never directories, and can be read from and closed. type ReaderFile struct { - filename string - fullpath string - abspath string - reader io.ReadCloser - stat os.FileInfo + abspath string + reader io.ReadCloser + stat os.FileInfo } -func NewReaderFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) *ReaderFile { - return &ReaderFile{filename, path, path, reader, stat} +func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) File { + return &ReaderFile{"", reader, stat} } -func NewReaderPathFile(filename, path string, reader io.ReadCloser, stat os.FileInfo) (*ReaderFile, error) { +func NewReaderPathFile(path string, reader io.ReadCloser, stat os.FileInfo) (*ReaderFile, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err } - return &ReaderFile{filename, path, abspath, reader, stat}, nil + return &ReaderFile{abspath, reader, stat}, nil } func (f *ReaderFile) IsDirectory() bool { return false } -func (f *ReaderFile) NextFile() (File, error) { - return nil, ErrNotDirectory -} - -func (f *ReaderFile) FileName() string { - return f.filename -} - -func (f *ReaderFile) FullPath() string { - return f.fullpath +func (f *ReaderFile) NextFile() (string, File, error) { + return "", nil, ErrNotDirectory } func (f *ReaderFile) AbsPath() string { @@ -64,7 +54,15 @@ func (f *ReaderFile) Stat() os.FileInfo { func (f *ReaderFile) Size() (int64, error) { if f.stat == nil { - return 0, errors.New("File size unknown") + return 0, errors.New("file size unknown") } return f.stat.Size(), nil } + +func (f *ReaderFile) Seek(offset int64, whence int) (int64, error) { + if s, ok := f.reader.(io.Seeker); ok { + return s.Seek(offset, whence) + } + + return 0, ErrNotSupported +} diff --git a/serialfile.go b/serialfile.go index 15e6c90..eb41976 100644 --- a/serialfile.go +++ b/serialfile.go @@ -7,29 +7,26 @@ import ( "os" "path/filepath" "strings" - "syscall" ) // serialFile implements File, and reads from a path on the OS filesystem. // No more than one file will be opened at a time (directories will advance // to the next file when NextFile() is called). type serialFile struct { - name string path string files []os.FileInfo stat os.FileInfo - current *File handleHiddenFiles bool } -func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, error) { +func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { switch mode := stat.Mode(); { case mode.IsRegular(): file, err := os.Open(path) if err != nil { return nil, err } - return NewReaderPathFile(name, path, file, stat) + return NewReaderPathFile(path, file, stat) case mode.IsDir(): // for directories, stat all of the contents first, so we know what files to // open when NextFile() is called @@ -37,15 +34,15 @@ func NewSerialFile(name, path string, hidden bool, stat os.FileInfo) (File, erro if err != nil { return nil, err } - return &serialFile{name, path, contents, stat, nil, hidden}, nil + return &serialFile{path, contents, stat, hidden}, nil case mode&os.ModeSymlink != 0: target, err := os.Readlink(path) if err != nil { return nil, err } - return NewLinkFile(name, path, target, stat), nil + return NewLinkFile(path, target, stat), nil default: - return nil, fmt.Errorf("Unrecognized file type for %s: %s", name, mode.String()) + return nil, fmt.Errorf("unrecognized file type for %s: %s", path, mode.String()) } } @@ -55,23 +52,23 @@ func (f *serialFile) IsDirectory() bool { return true } -func (f *serialFile) NextFile() (File, error) { +func (f *serialFile) NextFile() (string, File, error) { // if a file was opened previously, close it err := f.Close() if err != nil { switch err2 := err.(type) { case *os.PathError: if err2.Err != os.ErrClosed { - return nil, err + return "", nil, err } default: - return nil, err + return "", nil, err } } // if there aren't any files left in the root directory, we're done if len(f.files) == 0 { - return nil, io.EOF + return "", nil, io.EOF } stat := f.files[0] @@ -79,7 +76,7 @@ func (f *serialFile) NextFile() (File, error) { for !f.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") { if len(f.files) == 0 { - return nil, io.EOF + return "", nil, io.EOF } stat = f.files[0] @@ -87,28 +84,17 @@ func (f *serialFile) NextFile() (File, error) { } // open the next file - fileName := filepath.ToSlash(filepath.Join(f.name, stat.Name())) filePath := filepath.ToSlash(filepath.Join(f.path, stat.Name())) // recursively call the constructor on the next file // if it's a regular file, we will open it as a ReaderFile // if it's a directory, files in it will be opened serially - sf, err := NewSerialFile(fileName, filePath, f.handleHiddenFiles, stat) + sf, err := NewSerialFile(filePath, f.handleHiddenFiles, stat) if err != nil { - return nil, err + return "", nil, err } - f.current = &sf - - return sf, nil -} - -func (f *serialFile) FileName() string { - return f.name -} - -func (f *serialFile) FullPath() string { - return f.path + return stat.Name(), sf, nil } func (f *serialFile) Read(p []byte) (int, error) { @@ -116,18 +102,13 @@ func (f *serialFile) Read(p []byte) (int, error) { } func (f *serialFile) Close() error { - // close the current file if there is one - if f.current != nil { - err := (*f.current).Close() - // ignore EINVAL error, the file might have already been closed - if err != nil && err != syscall.EINVAL { - return err - } - } - return nil } +func (f *serialFile) Seek(offset int64, whence int) (int64, error) { + return 0, ErrNotReader +} + func (f *serialFile) Stat() os.FileInfo { return f.stat } @@ -138,7 +119,7 @@ func (f *serialFile) Size() (int64, error) { } var du int64 - err := filepath.Walk(f.FullPath(), func(p string, fi os.FileInfo, err error) error { + err := filepath.Walk(f.path, func(p string, fi os.FileInfo, err error) error { if err != nil { return err } diff --git a/slicefile.go b/slicefile.go index 8d18dca..d356465 100644 --- a/slicefile.go +++ b/slicefile.go @@ -1,43 +1,37 @@ package files import ( - "errors" "io" ) +type FileEntry struct { + File File + Name string +} + // SliceFile implements File, and provides simple directory handling. // It contains children files, and is created from a `[]File`. // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { - filename string - path string - files []File - n int + files []FileEntry + n int } -func NewSliceFile(filename, path string, files []File) *SliceFile { - return &SliceFile{filename, path, files, 0} +func NewSliceFile(files []FileEntry) File { + return &SliceFile{files, 0} } func (f *SliceFile) IsDirectory() bool { return true } -func (f *SliceFile) NextFile() (File, error) { +func (f *SliceFile) NextFile() (string, File, error) { if f.n >= len(f.files) { - return nil, io.EOF + return "", nil, io.EOF } file := f.files[f.n] f.n++ - return file, nil -} - -func (f *SliceFile) FileName() string { - return f.filename -} - -func (f *SliceFile) FullPath() string { - return f.path + return file.Name, file.File, nil } func (f *SliceFile) Read(p []byte) (int, error) { @@ -48,8 +42,8 @@ func (f *SliceFile) Close() error { return ErrNotReader } -func (f *SliceFile) Peek(n int) File { - return f.files[n] +func (f *SliceFile) Seek(offset int64, whence int) (int64, error) { + return 0, ErrNotReader } func (f *SliceFile) Length() int { @@ -60,12 +54,7 @@ func (f *SliceFile) Size() (int64, error) { var size int64 for _, file := range f.files { - sizeFile, ok := file.(SizeFile) - if !ok { - return 0, errors.New("Could not get size of child file") - } - - s, err := sizeFile.Size() + s, err := file.File.Size() if err != nil { return 0, err } diff --git a/webfile.go b/webfile.go index fcf4412..65d1c1e 100644 --- a/webfile.go +++ b/webfile.go @@ -4,7 +4,6 @@ import ( "io" "net/http" "net/url" - "path/filepath" ) // WebFile is an implementation of File which reads it @@ -46,17 +45,6 @@ func (wf *WebFile) Close() error { return wf.body.Close() } -// FullPath returns the "Host+Path" for this WebFile. -func (wf *WebFile) FullPath() string { - return wf.url.Host + wf.url.Path -} - -// FileName returns the last element of the URL -// path for this file. -func (wf *WebFile) FileName() string { - return filepath.Base(wf.url.Path) -} - // IsDirectory returns false. func (wf *WebFile) IsDirectory() bool { return false @@ -66,3 +54,8 @@ func (wf *WebFile) IsDirectory() bool { func (wf *WebFile) NextFile() (File, error) { return nil, ErrNotDirectory } + +// TODO: implement +func (wf *WebFile) Seek(offset int64, whence int) (int64, error) { + return 0, ErrNotSupported +} From dc3989c61ae9577632db1a6e81ebcf5e43d8dafe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 24 Oct 2018 00:43:36 +0200 Subject: [PATCH 02/25] Enable CI --- ci/Jenkinsfile | 1 + 1 file changed, 1 insertion(+) create mode 100644 ci/Jenkinsfile diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile new file mode 100644 index 0000000..b2067e6 --- /dev/null +++ b/ci/Jenkinsfile @@ -0,0 +1 @@ +golang() From cfdeccb3723af80fe98b9b3103eff1501527b6cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 24 Oct 2018 16:48:13 +0200 Subject: [PATCH 03/25] Allow Close on dirs, make sure errors make sense --- file.go | 19 ++++++++++++++----- file_test.go | 10 +++++----- multifilereader.go | 8 ++++++-- multipartfile.go | 7 ++----- readerfile.go | 3 +-- serialfile.go | 2 +- slicefile.go | 4 ++-- 7 files changed, 31 insertions(+), 22 deletions(-) diff --git a/file.go b/file.go index ceb60dd..daa30b8 100644 --- a/file.go +++ b/file.go @@ -15,11 +15,10 @@ var ( // File is an interface that provides functionality for handling // files/directories as values that can be supplied to commands. For -// directories, child files are accessed serially by calling `Files()` -// or `Walk()`. +// directories, child files are accessed serially by calling `NextFile()` // -// Read/Seek/Close methods are only valid for files -// Files/Walk methods are only valid for directories +// Read/Seek methods are only valid for files +// NextFile method is only valid for directories type File interface { io.Reader io.Closer @@ -37,6 +36,16 @@ type File interface { // directory). It will return io.EOF if no more files are // available. If the file is a regular file (not a directory), NextFile // will return a non-nil error. + // + // Note: + // - Some implementations may only allow reading in order - if a + // child directory is returned, you need to read all it's children + // first before calling NextFile on parent again. Before doing parallel + // reading or reading entire level at once, make sure the implementation + // you are using allows that + // - Returned files may not be sorted + // - Depending on implementation it may not be safe to iterate multiple + // children in parallel NextFile() (string, File, error) } @@ -44,7 +53,7 @@ type File interface { type FileInfo interface { File - // AbsPath returns full/real file path. + // AbsPath returns full real file path. AbsPath() string // Stat returns os.Stat of this file diff --git a/file_test.go b/file_test.go index 60d1bc4..e68540c 100644 --- a/file_test.go +++ b/file_test.go @@ -22,12 +22,12 @@ func TestSliceFiles(t *testing.T) { t.Fatal("SliceFile should always be a directory") } - if n, err := sf.Read(buf); n > 0 || err != io.EOF { + if n, err := sf.Read(buf); n > 0 || err != ErrNotReader { t.Fatal("Shouldn't be able to read data from a SliceFile") } - if err := sf.Close(); err != ErrNotReader { - t.Fatal("Shouldn't be able to call `Close` on a SliceFile") + if err := sf.Close(); err != nil { + t.Fatal("Should be able to call `Close` on a SliceFile") } _, file, err := sf.NextFile() @@ -151,8 +151,8 @@ anotherfile if n, err := mpf.Read(buf); n > 0 || err != ErrNotReader { t.Fatal("Shouldn't be able to call `Read` on a directory") } - if err := mpf.Close(); err != ErrNotReader { - t.Fatal("Shouldn't be able to call `Close` on a directory") + if err := mpf.Close(); err != nil { + t.Fatal("Should be able to call `Close` on a directory") } // test properties of file created from third part (nested file) diff --git a/multifilereader.go b/multifilereader.go index 7863c9d..85e66e7 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -20,7 +20,7 @@ type MultiFileReader struct { files []File path []string - currentFile io.Reader + currentFile File buf bytes.Buffer mpWriter *multipart.Writer closed bool @@ -121,7 +121,11 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // otherwise, read from file data written, err = mfr.currentFile.Read(buf) - if err == io.EOF { + if err == io.EOF || err == ErrNotReader { + if err := mfr.currentFile.Close(); err != nil { + return written, err + } + mfr.currentFile = nil return written, nil } diff --git a/multipartfile.go b/multipartfile.go index a304163..cc94f17 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -136,9 +136,6 @@ func (f *MultipartFile) Read(p []byte) (int, error) { } func (f *MultipartFile) Close() error { - if f.IsDirectory() { - return ErrNotReader - } return f.Part.Close() } @@ -146,11 +143,11 @@ func (f *MultipartFile) Seek(offset int64, whence int) (int64, error) { if f.IsDirectory() { return 0, ErrNotReader } - return 0, ErrNotReader + return 0, ErrNotSupported } func (f *MultipartFile) Size() (int64, error) { - return 0, ErrNotReader + return 0, ErrNotSupported } type PartReader interface { diff --git a/readerfile.go b/readerfile.go index e40032c..070ee97 100644 --- a/readerfile.go +++ b/readerfile.go @@ -1,7 +1,6 @@ package files import ( - "errors" "io" "os" "path/filepath" @@ -54,7 +53,7 @@ func (f *ReaderFile) Stat() os.FileInfo { func (f *ReaderFile) Size() (int64, error) { if f.stat == nil { - return 0, errors.New("file size unknown") + return 0, ErrNotSupported } return f.stat.Size(), nil } diff --git a/serialfile.go b/serialfile.go index eb41976..71855d5 100644 --- a/serialfile.go +++ b/serialfile.go @@ -98,7 +98,7 @@ func (f *serialFile) NextFile() (string, File, error) { } func (f *serialFile) Read(p []byte) (int, error) { - return 0, io.EOF + return 0, ErrNotReader } func (f *serialFile) Close() error { diff --git a/slicefile.go b/slicefile.go index d356465..5fea5a6 100644 --- a/slicefile.go +++ b/slicefile.go @@ -35,11 +35,11 @@ func (f *SliceFile) NextFile() (string, File, error) { } func (f *SliceFile) Read(p []byte) (int, error) { - return 0, io.EOF + return 0, ErrNotReader } func (f *SliceFile) Close() error { - return ErrNotReader + return nil } func (f *SliceFile) Seek(offset int64, whence int) (int64, error) { From b0b422eb2d7f4defd547e4cb93fbc0fe2023c0b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 5 Nov 2018 17:05:53 +0100 Subject: [PATCH 04/25] fix IsHidden on windows --- is_hidden_windows.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/is_hidden_windows.go b/is_hidden_windows.go index 40f40ae..6f9568a 100644 --- a/is_hidden_windows.go +++ b/is_hidden_windows.go @@ -11,7 +11,7 @@ import ( func IsHidden(name string, f File) bool { - fName := filepath.Base(f.FileName()) + fName := filepath.Base(name) if strings.HasPrefix(fName, ".") && len(fName) > 1 { return true From 97852d87e92fd8484e9fcf0e9b2b1c942849e44d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Nov 2018 15:49:42 +0100 Subject: [PATCH 05/25] Separate file/directory types --- file.go | 29 +++++++++++------- file_test.go | 66 +++++++++++++++-------------------------- linkfile.go | 14 ++------- multifilereader.go | 36 +++++++++++++--------- multifilereader_test.go | 36 ++++++++++++---------- multipartfile.go | 38 ++++++++++-------------- readerfile.go | 13 +++----- serialfile.go | 18 ++--------- slicefile.go | 16 ++-------- webfile.go | 24 ++++++++------- 10 files changed, 127 insertions(+), 163 deletions(-) diff --git a/file.go b/file.go index daa30b8..b2d9712 100644 --- a/file.go +++ b/file.go @@ -7,8 +7,8 @@ import ( ) var ( - ErrNotDirectory = errors.New("couldn't call NextFile(), this isn't a directory") - ErrNotReader = errors.New("this file is a directory, can't use Reader functions") + ErrNotDirectory = errors.New("file isn't a directory") + ErrNotReader = errors.New("file isn't a regular file") ErrNotSupported = errors.New("operation not supported") ) @@ -20,22 +20,29 @@ var ( // Read/Seek methods are only valid for files // NextFile method is only valid for directories type File interface { - io.Reader io.Closer - io.Seeker - // Size returns size of the + // Size returns size of this file (if this file is a directory, total size of + // all files stored in the tree should be returned). Some implementations may + // choose not to implement this Size() (int64, error) +} + +// Regular represents the regular Unix file +type Regular interface { + File - // IsDirectory returns true if the File is a directory (and therefore - // supports calling `Files`/`Walk`) and false if the File is a normal file - // (and therefore supports calling `Read`/`Close`/`Seek`) - IsDirectory() bool + io.Reader + io.Seeker +} + +// Directory is a special file which can link to any number of files +type Directory interface { + File // NextFile returns the next child file available (if the File is a // directory). It will return io.EOF if no more files are - // available. If the file is a regular file (not a directory), NextFile - // will return a non-nil error. + // available. // // Note: // - Some implementations may only allow reading in order - if a diff --git a/file_test.go b/file_test.go index e68540c..9b13082 100644 --- a/file_test.go +++ b/file_test.go @@ -18,23 +18,15 @@ func TestSliceFiles(t *testing.T) { sf := NewSliceFile(files) - if !sf.IsDirectory() { - t.Fatal("SliceFile should always be a directory") - } - - if n, err := sf.Read(buf); n > 0 || err != ErrNotReader { - t.Fatal("Shouldn't be able to read data from a SliceFile") - } - - if err := sf.Close(); err != nil { - t.Fatal("Should be able to call `Close` on a SliceFile") - } - _, file, err := sf.NextFile() if file == nil || err != nil { t.Fatal("Expected a file and nil error") } - read, err := file.Read(buf) + rf, ok := file.(Regular) + if !ok { + t.Fatal("Expected a regular file") + } + read, err := rf.Read(buf) if read != 11 || err != nil { t.Fatal("NextFile got a file in the wrong order") } @@ -52,6 +44,10 @@ func TestSliceFiles(t *testing.T) { if file != nil || err != io.EOF { t.Fatal("Expected a nil file and io.EOF") } + + if err := sf.Close(); err != nil { + t.Fatal("Should be able to call `Close` on a SliceFile") + } } func TestReaderFiles(t *testing.T) { @@ -59,14 +55,6 @@ func TestReaderFiles(t *testing.T) { rf := NewReaderFile(ioutil.NopCloser(strings.NewReader(message)), nil) buf := make([]byte, len(message)) - if rf.IsDirectory() { - t.Fatal("ReaderFile should never be a directory") - } - _, file, err := rf.NextFile() - if file != nil || err != ErrNotDirectory { - t.Fatal("Expected a nil file and ErrNotDirectory") - } - if n, err := rf.Read(buf); n == 0 || err != nil { t.Fatal("Expected to be able to read") } @@ -117,19 +105,17 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpf.IsDirectory() { + mf, ok := mpf.(Regular) + if !ok { t.Fatal("Expected file to not be a directory") } if mpname != "name" { t.Fatal("Expected filename to be \"name\"") } - if _, file, err := mpf.NextFile(); file != nil || err != ErrNotDirectory { - t.Fatal("Expected a nil file and ErrNotDirectory") - } - if n, err := mpf.Read(buf); n != 4 || !(err == io.EOF || err == nil) { + if n, err := mf.Read(buf); n != 4 || !(err == io.EOF || err == nil) { t.Fatal("Expected to be able to read 4 bytes", n, err) } - if err := mpf.Close(); err != nil { + if err := mf.Close(); err != nil { t.Fatal("Expected to be able to close file") } @@ -142,16 +128,14 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if !mpf.IsDirectory() { + md, ok := mpf.(Directory) + if !ok { t.Fatal("Expected file to be a directory") } if mpname != "dir" { t.Fatal("Expected filename to be \"dir\"") } - if n, err := mpf.Read(buf); n > 0 || err != ErrNotReader { - t.Fatal("Shouldn't be able to call `Read` on a directory") - } - if err := mpf.Close(); err != nil { + if err := md.Close(); err != nil { t.Fatal("Should be able to call `Close` on a directory") } @@ -164,13 +148,14 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpf.IsDirectory() { - t.Fatal("Expected file, got directory") + mf, ok = mpf.(Regular) + if !ok { + t.Fatal("Expected file to not be a directory") } if mpname != "nested" { t.Fatalf("Expected filename to be \"nested\", got %s", mpname) } - if n, err := mpf.Read(buf); n != 12 || !(err == nil || err == io.EOF) { + if n, err := mf.Read(buf); n != 12 || !(err == nil || err == io.EOF) { t.Fatalf("expected to be able to read 12 bytes from file: %s (got %d)", err, n) } if err := mpf.Close(); err != nil { @@ -186,17 +171,14 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpf.IsDirectory() { - t.Fatal("Expected file to be a symlink") + ms, ok := mpf.(*Symlink) + if !ok { + t.Fatal("Expected file to not be a directory") } if mpname != "simlynk" { t.Fatal("Expected filename to be \"dir/simlynk\"") } - slink, ok := mpf.(*Symlink) - if !ok { - t.Fatalf("expected file to be a symlink") - } - if slink.Target != "anotherfile" { + if ms.Target != "anotherfile" { t.Fatal("expected link to point to anotherfile") } } diff --git a/linkfile.go b/linkfile.go index 0182d39..df334d5 100644 --- a/linkfile.go +++ b/linkfile.go @@ -7,30 +7,20 @@ import ( ) type Symlink struct { - path string Target string stat os.FileInfo reader io.Reader } -func NewLinkFile(path, target string, stat os.FileInfo) File { +func NewLinkFile(target string, stat os.FileInfo) Regular { return &Symlink{ - path: path, Target: target, stat: stat, reader: strings.NewReader(target), } } -func (lf *Symlink) IsDirectory() bool { - return false -} - -func (lf *Symlink) NextFile() (string, File, error) { - return "", nil, ErrNotDirectory -} - func (lf *Symlink) Close() error { if c, ok := lf.reader.(io.Closer); ok { return c.Close() @@ -54,3 +44,5 @@ func (lf *Symlink) Seek(offset int64, whence int) (int64, error) { func (lf *Symlink) Size() (int64, error) { return 0, ErrNotSupported } + +var _ Regular = &Symlink{} diff --git a/multifilereader.go b/multifilereader.go index 85e66e7..b0ce1d9 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -17,7 +17,7 @@ type MultiFileReader struct { io.Reader // directory stack for NextFile - files []File + files []Directory path []string currentFile File @@ -31,12 +31,12 @@ type MultiFileReader struct { form bool } -// NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.File`. +// NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.Directory`. // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. -func NewMultiFileReader(file File, form bool) *MultiFileReader { +func NewMultiFileReader(file Directory, form bool) *MultiFileReader { mfr := &MultiFileReader{ - files: []File{file}, + files: []Directory{file}, path: []string{""}, form: form, mutex: &sync.Mutex{}, @@ -91,15 +91,19 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) var contentType string - if _, ok := file.(*Symlink); ok { + + switch f := file.(type) { + case *Symlink: contentType = "application/symlink" - } else if file.IsDirectory() { - mfr.files = append(mfr.files, file) + case Directory: + mfr.files = append(mfr.files, f) mfr.path = append(mfr.path, name) contentType = "application/x-directory" - } else { + case Regular: // otherwise, use the file as a reader to read its contents contentType = "application/octet-stream" + default: + return 0, ErrNotSupported } header.Set("Content-Type", contentType) @@ -120,16 +124,20 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } // otherwise, read from file data - written, err = mfr.currentFile.Read(buf) - if err == io.EOF || err == ErrNotReader { - if err := mfr.currentFile.Close(); err != nil { + switch f := mfr.currentFile.(type) { + case Regular: + written, err = f.Read(buf) + if err != io.EOF { return written, err } + } - mfr.currentFile = nil - return written, nil + if err := mfr.currentFile.Close(); err != nil { + return written, err } - return written, err + + mfr.currentFile = nil + return written, nil } // Boundary returns the boundary string to be used to separate files in the multipart data diff --git a/multifilereader_test.go b/multifilereader_test.go index 18876ef..5f0cac5 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -33,46 +33,48 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal(err) } - if !mf.IsDirectory() { + md, ok := mf.(Directory) + if !ok { t.Fatal("Expected a directory") } - fn, f, err := mf.NextFile() + fn, f, err := md.NextFile() if fn != "file.txt" || f == nil || err != nil { t.Fatal("NextFile returned unexpected data") } - dn, d, err := mf.NextFile() + dn, d, err := md.NextFile() if dn != "boop" || d == nil || err != nil { t.Fatal("NextFile returned unexpected data") } - if !d.IsDirectory() { + df, ok := d.(Directory) + if !ok { t.Fatal("Expected a directory") } - cfn, cf, err := d.NextFile() + cfn, cf, err := df.NextFile() if cfn != "a.txt" || cf == nil || err != nil { t.Fatal("NextFile returned unexpected data") } - cfn, cf, err = d.NextFile() + cfn, cf, err = df.NextFile() if cfn != "b.txt" || cf == nil || err != nil { t.Fatal("NextFile returned unexpected data") } - cfn, cf, err = d.NextFile() + cfn, cf, err = df.NextFile() if cfn != "" || cf != nil || err != io.EOF { t.Fatal("NextFile returned unexpected data") } // try to break internal state - cfn, cf, err = d.NextFile() + cfn, cf, err = df.NextFile() if cfn != "" || cf != nil || err != io.EOF { t.Fatal("NextFile returned unexpected data") } - fn, f, err = mf.NextFile() + fn, f, err = md.NextFile() if fn != "beep.txt" || f == nil || err != nil { t.Fatal("NextFile returned unexpected data") } @@ -91,13 +93,14 @@ func TestOutput(t *testing.T) { if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpf.IsDirectory() { - t.Fatal("Expected file to not be a directory") + mpr, ok := mpf.(Regular) + if !ok { + t.Fatal("Expected file to be a regular file") } if mpname != "file.txt" { t.Fatal("Expected filename to be \"file.txt\"") } - if n, err := mpf.Read(buf); n != len(text) || err != nil { + if n, err := mpr.Read(buf); n != len(text) || err != nil { t.Fatal("Expected to read from file", n, err) } if string(buf[:len(text)]) != text { @@ -112,7 +115,8 @@ func TestOutput(t *testing.T) { if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if !mpf.IsDirectory() { + mpd, ok := mpf.(Directory) + if !ok { t.Fatal("Expected file to be a directory") } if mpname != "boop" { @@ -127,7 +131,7 @@ func TestOutput(t *testing.T) { if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } - if child.IsDirectory() { + if _, ok := child.(Regular); !ok { t.Fatal("Expected file to not be a directory") } if cname != "a.txt" { @@ -142,14 +146,14 @@ func TestOutput(t *testing.T) { if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } - if child.IsDirectory() { + if _, ok := child.(Regular); !ok { t.Fatal("Expected file to not be a directory") } if cname != "b.txt" { t.Fatal("Expected filename to be \"b.txt\"") } - cname, child, err = mpf.NextFile() + cname, child, err = mpd.NextFile() if child != nil || err != io.EOF { t.Fatal("Expected to get (nil, io.EOF)") } diff --git a/multipartfile.go b/multipartfile.go index cc94f17..5c62c2c 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -34,6 +34,10 @@ type MultipartFile struct { } func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, error) { + if !isDirectory(mediatype) { + return nil, ErrNotDirectory + } + f := &MultipartFile{ Reader: &peekReader{r: reader}, Mediatype: mediatype, @@ -61,9 +65,7 @@ func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (st return "", nil, err } - return base, &Symlink{ - Target: string(out), - }, nil + return base, NewLinkFile(string(out), nil), nil case "": // default to application/octet-stream fallthrough case applicationFile: @@ -79,17 +81,21 @@ func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (st return "", nil, err } + if !isDirectory(f.Mediatype) { + return base, &ReaderFile{ + reader: part, + abspath: part.Header.Get("abspath"), + }, nil + } + return base, f, nil } -func (f *MultipartFile) IsDirectory() bool { - return f.Mediatype == multipartFormdataType || f.Mediatype == applicationDirectory +func isDirectory(mediatype string) bool { + return mediatype == multipartFormdataType || mediatype == applicationDirectory } func (f *MultipartFile) NextFile() (string, File, error) { - if !f.IsDirectory() { - return "", nil, ErrNotDirectory - } if f.Reader == nil { return "", nil, io.EOF } @@ -128,24 +134,10 @@ func (f *MultipartFile) fileName() string { return filename } -func (f *MultipartFile) Read(p []byte) (int, error) { - if f.IsDirectory() { - return 0, ErrNotReader - } - return f.Part.Read(p) -} - func (f *MultipartFile) Close() error { return f.Part.Close() } -func (f *MultipartFile) Seek(offset int64, whence int) (int64, error) { - if f.IsDirectory() { - return 0, ErrNotReader - } - return 0, ErrNotSupported -} - func (f *MultipartFile) Size() (int64, error) { return 0, ErrNotSupported } @@ -176,3 +168,5 @@ func (pr *peekReader) put(p *multipart.Part) error { pr.next = p return nil } + +var _ Directory = &MultipartFile{} diff --git a/readerfile.go b/readerfile.go index 070ee97..7db7e96 100644 --- a/readerfile.go +++ b/readerfile.go @@ -14,7 +14,7 @@ type ReaderFile struct { stat os.FileInfo } -func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) File { +func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) Regular { return &ReaderFile{"", reader, stat} } @@ -27,14 +27,6 @@ func NewReaderPathFile(path string, reader io.ReadCloser, stat os.FileInfo) (*Re return &ReaderFile{abspath, reader, stat}, nil } -func (f *ReaderFile) IsDirectory() bool { - return false -} - -func (f *ReaderFile) NextFile() (string, File, error) { - return "", nil, ErrNotDirectory -} - func (f *ReaderFile) AbsPath() string { return f.abspath } @@ -65,3 +57,6 @@ func (f *ReaderFile) Seek(offset int64, whence int) (int64, error) { return 0, ErrNotSupported } + +var _ Regular = &ReaderFile{} +var _ FileInfo = &ReaderFile{} diff --git a/serialfile.go b/serialfile.go index 71855d5..666399d 100644 --- a/serialfile.go +++ b/serialfile.go @@ -40,18 +40,12 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { if err != nil { return nil, err } - return NewLinkFile(path, target, stat), nil + return NewLinkFile(target, stat), nil default: return nil, fmt.Errorf("unrecognized file type for %s: %s", path, mode.String()) } } -func (f *serialFile) IsDirectory() bool { - // non-directories get created as a ReaderFile, so serialFiles should only - // represent directories - return true -} - func (f *serialFile) NextFile() (string, File, error) { // if a file was opened previously, close it err := f.Close() @@ -97,18 +91,10 @@ func (f *serialFile) NextFile() (string, File, error) { return stat.Name(), sf, nil } -func (f *serialFile) Read(p []byte) (int, error) { - return 0, ErrNotReader -} - func (f *serialFile) Close() error { return nil } -func (f *serialFile) Seek(offset int64, whence int) (int64, error) { - return 0, ErrNotReader -} - func (f *serialFile) Stat() os.FileInfo { return f.stat } @@ -132,3 +118,5 @@ func (f *serialFile) Size() (int64, error) { return du, err } + +var _ Directory = &serialFile{} diff --git a/slicefile.go b/slicefile.go index 5fea5a6..0d78e16 100644 --- a/slicefile.go +++ b/slicefile.go @@ -17,14 +17,10 @@ type SliceFile struct { n int } -func NewSliceFile(files []FileEntry) File { +func NewSliceFile(files []FileEntry) Directory { return &SliceFile{files, 0} } -func (f *SliceFile) IsDirectory() bool { - return true -} - func (f *SliceFile) NextFile() (string, File, error) { if f.n >= len(f.files) { return "", nil, io.EOF @@ -34,18 +30,10 @@ func (f *SliceFile) NextFile() (string, File, error) { return file.Name, file.File, nil } -func (f *SliceFile) Read(p []byte) (int, error) { - return 0, ErrNotReader -} - func (f *SliceFile) Close() error { return nil } -func (f *SliceFile) Seek(offset int64, whence int) (int64, error) { - return 0, ErrNotReader -} - func (f *SliceFile) Length() int { return len(f.files) } @@ -63,3 +51,5 @@ func (f *SliceFile) Size() (int64, error) { return size, nil } + +var _ Directory = &SliceFile{} diff --git a/webfile.go b/webfile.go index 65d1c1e..5638f77 100644 --- a/webfile.go +++ b/webfile.go @@ -1,6 +1,7 @@ package files import ( + "github.com/pkg/errors" "io" "net/http" "net/url" @@ -12,6 +13,7 @@ import ( type WebFile struct { body io.ReadCloser url *url.URL + contentLength int64 } // NewWebFile creates a WebFile with the given URL, which @@ -33,6 +35,7 @@ func (wf *WebFile) Read(b []byte) (int, error) { return 0, err } wf.body = resp.Body + wf.contentLength = resp.ContentLength } return wf.body.Read(b) } @@ -45,17 +48,18 @@ func (wf *WebFile) Close() error { return wf.body.Close() } -// IsDirectory returns false. -func (wf *WebFile) IsDirectory() bool { - return false -} - -// NextFile always returns an ErrNotDirectory error. -func (wf *WebFile) NextFile() (File, error) { - return nil, ErrNotDirectory -} - // TODO: implement func (wf *WebFile) Seek(offset int64, whence int) (int64, error) { return 0, ErrNotSupported } + +func (wf *WebFile) Size() (int64, error) { + if wf.contentLength < 0 { + return -1, errors.New("Content-Length hearer was not set") + } + + return wf.contentLength, nil +} + + +var _ Regular = &WebFile{} From 2b34e278cd80cdb924b4b28bf434c4c084ab61fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 7 Nov 2018 18:53:12 +0100 Subject: [PATCH 06/25] Directory iterators --- file.go | 75 +++++++++++++++++++++++++------ file_test.go | 37 ++++++++-------- multifilereader.go | 46 +++++++++++-------- multifilereader_test.go | 77 ++++++++++++++++---------------- multipartfile.go | 69 ++++++++++++++++++++++------- serialfile.go | 98 ++++++++++++++++++++++++++++++++++++----- slicefile.go | 80 +++++++++++++++++++++++++-------- webfile.go | 5 +-- 8 files changed, 348 insertions(+), 139 deletions(-) diff --git a/file.go b/file.go index b2d9712..693baab 100644 --- a/file.go +++ b/file.go @@ -28,7 +28,7 @@ type File interface { Size() (int64, error) } -// Regular represents the regular Unix file +// Regular represents a regular Unix file type Regular interface { File @@ -36,24 +36,71 @@ type Regular interface { io.Seeker } -// Directory is a special file which can link to any number of files +// DirEntry exposes information about a directory entry +type DirEntry interface { + // Name returns the base name of this entry, which is the base name of + // the referenced file + Name() string + + // File returns the file referenced by this DirEntry + File() File + + // Regular is an alias for ent.File().(Regular). If the file isn't a regular + // file, nil value will be returned + Regular() Regular + + // Dir is an alias for ent.File().(directory). If the file isn't a directory, + // nil value will be returned + Dir() Directory +} + +// DirIterator is a iterator over directory entries. +// See Directory.Entries for more +type DirIterator interface { + // DirEntry holds information about current directory entry. + // Note that after creating new iterator you MUST call Next() at least once + // before accessing these methods. Calling these methods without prior calls + // to Next() and after Next() returned false may result in undefined behavior + DirEntry + + // Next advances the iterator to the next file. + Next() bool + + // Err may return an error after the previous call to Next() returned `false`. + // If the previous call to Next() returned `true`, Err() is guaranteed to + // return nil + Err() error +} + +// Directory is a special file which can link to any number of files. type Directory interface { File - // NextFile returns the next child file available (if the File is a - // directory). It will return io.EOF if no more files are - // available. + // Entries returns a stateful iterator over directory entries. + // + // Example usage: + // + // it := dir.Entries() + // for it.Next() { + // name := it.Name() + // file := it.File() + // [...] + // } + // if it.Err() != nil { + // return err + // } // // Note: - // - Some implementations may only allow reading in order - if a - // child directory is returned, you need to read all it's children - // first before calling NextFile on parent again. Before doing parallel - // reading or reading entire level at once, make sure the implementation - // you are using allows that - // - Returned files may not be sorted + // - Below limitations aren't applicable to all implementations, consult + // your implementations manual before using this interface in a way that + // doesn't meet these constraints + // - Some implementations may only allow reading in order - so if the iterator + // returns a directory you must iterate over it's entries first before + // calling Next again + // - Order is not guaranteed // - Depending on implementation it may not be safe to iterate multiple - // children in parallel - NextFile() (string, File, error) + // 'branches' in parallel + Entries() (DirIterator, error) } // FileInfo exposes information on files in local filesystem @@ -63,6 +110,6 @@ type FileInfo interface { // AbsPath returns full real file path. AbsPath() string - // Stat returns os.Stat of this file + // Stat returns os.Stat of this file, may be nil for some files Stat() os.FileInfo } diff --git a/file_test.go b/file_test.go index 9b13082..cbf227c 100644 --- a/file_test.go +++ b/file_test.go @@ -9,21 +9,24 @@ import ( ) func TestSliceFiles(t *testing.T) { - files := []FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil), ""}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), ""}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil), ""}, + files := []DirEntry{ + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil)), + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), + FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil)), } buf := make([]byte, 20) sf := NewSliceFile(files) + it, err := sf.Entries() + if err != nil { + t.Fatal(err) + } - _, file, err := sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - rf, ok := file.(Regular) - if !ok { + rf := it.Regular() + if rf == nil { t.Fatal("Expected a regular file") } read, err := rf.Read(buf) @@ -31,18 +34,14 @@ func TestSliceFiles(t *testing.T) { t.Fatal("NextFile got a file in the wrong order") } - _, file, err = sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - _, file, err = sf.NextFile() - if file == nil || err != nil { - t.Fatal("Expected a file and nil error") + if !it.Next() { + t.Fatal("Expected a file") } - - _, file, err = sf.NextFile() - if file != nil || err != io.EOF { - t.Fatal("Expected a nil file and io.EOF") + if it.Next() { + t.Fatal("Wild file appeared!") } if err := sf.Close(); err != nil { diff --git a/multifilereader.go b/multifilereader.go index b0ce1d9..8817ad8 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -17,7 +17,7 @@ type MultiFileReader struct { io.Reader // directory stack for NextFile - files []Directory + files []DirIterator path []string currentFile File @@ -34,16 +34,21 @@ type MultiFileReader struct { // NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.Directory`. // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. -func NewMultiFileReader(file Directory, form bool) *MultiFileReader { +func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { + it, err := file.Entries() + if err != nil { + return nil, err + } + mfr := &MultiFileReader{ - files: []Directory{file}, + files: []DirIterator{it}, path: []string{""}, form: form, mutex: &sync.Mutex{}, } mfr.mpWriter = multipart.NewWriter(&mfr.buf) - return mfr + return mfr, nil } func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { @@ -57,47 +62,50 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // if the current file isn't set, advance to the next file if mfr.currentFile == nil { - var file File - var name string + var entry DirEntry - for file == nil { + for entry == nil { if len(mfr.files) == 0 { mfr.mpWriter.Close() mfr.closed = true return mfr.buf.Read(buf) } - nextName, nextFile, err := mfr.files[len(mfr.files)-1].NextFile() - if err == io.EOF { + if !mfr.files[len(mfr.files)-1].Next() { mfr.files = mfr.files[:len(mfr.files)-1] mfr.path = mfr.path[:len(mfr.path)-1] continue - } else if err != nil { - return 0, err + } + if mfr.files[len(mfr.files)-1].Err() != nil { + return 0, mfr.files[len(mfr.files)-1].Err() } - file = nextFile - name = nextName + entry = mfr.files[len(mfr.files)-1] } // handle starting a new file part if !mfr.closed { - mfr.currentFile = file + mfr.currentFile = entry.File() // write the boundary and headers header := make(textproto.MIMEHeader) - filename := url.QueryEscape(path.Join(path.Join(mfr.path...), name)) + filename := url.QueryEscape(path.Join(path.Join(mfr.path...), entry.Name())) header.Set("Content-Disposition", fmt.Sprintf("file; filename=\"%s\"", filename)) var contentType string - switch f := file.(type) { + switch f := entry.File().(type) { case *Symlink: contentType = "application/symlink" case Directory: - mfr.files = append(mfr.files, f) - mfr.path = append(mfr.path, name) + newIt, err := f.Entries() + if err != nil { + return 0, err + } + + mfr.files = append(mfr.files, newIt) + mfr.path = append(mfr.path, entry.Name()) contentType = "application/x-directory" case Regular: // otherwise, use the file as a reader to read its contents @@ -107,7 +115,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } header.Set("Content-Type", contentType) - if rf, ok := file.(FileInfo); ok { + if rf, ok := entry.File().(FileInfo); ok { header.Set("abspath", rf.AbsPath()) } diff --git a/multifilereader_test.go b/multifilereader_test.go index 5f0cac5..dd7dac9 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -10,23 +10,27 @@ import ( var text = "Some text! :)" -func getTestMultiFileReader() *MultiFileReader { - fileset := []FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil), "file.txt"}, - {NewSliceFile([]FileEntry{ - {NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil), "a.txt"}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil), "b.txt"}, - }), "boop"}, - {NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil), "beep.txt"}, +func getTestMultiFileReader(t *testing.T) *MultiFileReader { + fileset := []DirEntry{ + FileEntry("file.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil)), + FileEntry("boop", NewSliceFile([]DirEntry{ + FileEntry("a.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil)), + FileEntry("b.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil)), + })), + FileEntry("beep.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), } sf := NewSliceFile(fileset) // testing output by reading it with the go stdlib "mime/multipart" Reader - return NewMultiFileReader(sf, true) + r, err := NewMultiFileReader(sf, true) + if err != nil { + t.Fatal(err) + } + return r } func TestMultiFileReaderToMultiFile(t *testing.T) { - mfr := getTestMultiFileReader() + mfr := getTestMultiFileReader(t) mpReader := multipart.NewReader(mfr, mfr.Boundary()) mf, err := NewFileFromPartReader(mpReader, multipartFormdataType) if err != nil { @@ -37,51 +41,48 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { if !ok { t.Fatal("Expected a directory") } + it, err := md.Entries() + if err != nil { + t.Fatal(err) + } - fn, f, err := md.NextFile() - if fn != "file.txt" || f == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "file.txt" { + t.Fatal("iterator didn't work as expected") } - dn, d, err := md.NextFile() - if dn != "boop" || d == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "boop" || it.Dir() == nil { + t.Fatal("iterator didn't work as expected") } - df, ok := d.(Directory) - if !ok { - t.Fatal("Expected a directory") + subIt, err := it.Dir().Entries() + if err != nil { + t.Fatal(err) } - cfn, cf, err := df.NextFile() - if cfn != "a.txt" || cf == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !subIt.Next() || subIt.Name() != "a.txt" || subIt.Dir() != nil { + t.Fatal("iterator didn't work as expected") } - cfn, cf, err = df.NextFile() - if cfn != "b.txt" || cf == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !subIt.Next() || subIt.Name() != "b.txt" || subIt.Dir() != nil { + t.Fatal("iterator didn't work as expected") } - cfn, cf, err = df.NextFile() - if cfn != "" || cf != nil || err != io.EOF { - t.Fatal("NextFile returned unexpected data") + if subIt.Next() { + t.Fatal("iterator didn't work as expected") } // try to break internal state - cfn, cf, err = df.NextFile() - if cfn != "" || cf != nil || err != io.EOF { - t.Fatal("NextFile returned unexpected data") + if subIt.Next() { + t.Fatal("iterator didn't work as expected") } - fn, f, err = md.NextFile() - if fn != "beep.txt" || f == nil || err != nil { - t.Fatal("NextFile returned unexpected data") + if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil { + t.Fatal("iterator didn't work as expected") } } func TestOutput(t *testing.T) { - mfr := getTestMultiFileReader() + mfr := getTestMultiFileReader(t) mpReader := &peekReader{r: multipart.NewReader(mfr, mfr.Boundary())} buf := make([]byte, 20) @@ -153,9 +154,9 @@ func TestOutput(t *testing.T) { t.Fatal("Expected filename to be \"b.txt\"") } - cname, child, err = mpd.NextFile() - if child != nil || err != io.EOF { - t.Fatal("Expected to get (nil, io.EOF)") + it, err := mpd.Entries() + if it.Next() { + t.Fatal("Expected to get false") } part, err = mpReader.NextPart() diff --git a/multipartfile.go b/multipartfile.go index 5c62c2c..be7e221 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -2,7 +2,6 @@ package files import ( "errors" - "io" "io/ioutil" "mime" "mime/multipart" @@ -39,7 +38,7 @@ func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, er } f := &MultipartFile{ - Reader: &peekReader{r: reader}, + Reader: &peekReader{r: reader}, Mediatype: mediatype, } @@ -95,30 +94,65 @@ func isDirectory(mediatype string) bool { return mediatype == multipartFormdataType || mediatype == applicationDirectory } -func (f *MultipartFile) NextFile() (string, File, error) { - if f.Reader == nil { - return "", nil, io.EOF +type multipartIterator struct { + f *MultipartFile + + curFile File + curName string + err error +} + +func (it *multipartIterator) Name() string { + return it.curName +} + +func (it *multipartIterator) File() File { + return it.curFile +} + +func (it *multipartIterator) Regular() Regular { + return castRegular(it.File()) +} + +func (it *multipartIterator) Dir() Directory { + return castDir(it.File()) +} + +func (it *multipartIterator) Next() bool { + if it.f.Reader == nil { + return false } - part, err := f.Reader.NextPart() + part, err := it.f.Reader.NextPart() if err != nil { - return "", nil, err + it.err = err + return false } - name, cf, err := newFileFromPart(f.fileName(), part, f.Reader) + name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) if err != ErrPartOutsideParent { - return name, cf, err + it.curFile = cf + it.curName = name + it.err = err + return err == nil } // we read too much, try to fix this - pr, ok := f.Reader.(*peekReader) + pr, ok := it.f.Reader.(*peekReader) if !ok { - return "", nil, errors.New("cannot undo NextPart") + it.err = errors.New("cannot undo NextPart") + return false } - if err := pr.put(part); err != nil { - return "", nil, err - } - return "", nil, io.EOF + it.err = pr.put(part) + return false +} + +func (it *multipartIterator) Err() error { + panic("implement me") +} + +func (f *MultipartFile) Entries() (DirIterator, error) { + return &multipartIterator{f: f}, nil } func (f *MultipartFile) fileName() string { @@ -135,7 +169,10 @@ func (f *MultipartFile) fileName() string { } func (f *MultipartFile) Close() error { - return f.Part.Close() + if f.Part != nil { + return f.Part.Close() + } + return nil } func (f *MultipartFile) Size() (int64, error) { diff --git a/serialfile.go b/serialfile.go index 666399d..77afafa 100644 --- a/serialfile.go +++ b/serialfile.go @@ -1,6 +1,7 @@ package files import ( + "errors" "fmt" "io" "io/ioutil" @@ -19,6 +20,18 @@ type serialFile struct { handleHiddenFiles bool } +type serialIterator struct { + files []os.FileInfo + handleHiddenFiles bool + path string + + curName string + curFile File + + err error +} + +// TODO: test/document limitations func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { switch mode := stat.Mode(); { case mode.IsRegular(): @@ -46,20 +59,69 @@ func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { } } -func (f *serialFile) NextFile() (string, File, error) { - // if a file was opened previously, close it - err := f.Close() - if err != nil { - switch err2 := err.(type) { - case *os.PathError: - if err2.Err != os.ErrClosed { - return "", nil, err - } - default: - return "", nil, err +func (it *serialIterator) Name() string { + return it.curName +} + +func (it *serialIterator) File() File { + return it.curFile +} + +func (it *serialIterator) Regular() Regular { + return castRegular(it.File()) +} + +func (it *serialIterator) Dir() Directory { + return castDir(it.File()) +} + +func (it *serialIterator) Next() bool { + // if there aren't any files left in the root directory, we're done + if len(it.files) == 0 { + return false + } + + stat := it.files[0] + it.files = it.files[1:] + for !it.handleHiddenFiles && strings.HasPrefix(stat.Name(), ".") { + if len(it.files) == 0 { + return false } + + stat = it.files[0] + it.files = it.files[1:] + } + + // open the next file + filePath := filepath.ToSlash(filepath.Join(it.path, stat.Name())) + + // recursively call the constructor on the next file + // if it's a regular file, we will open it as a ReaderFile + // if it's a directory, files in it will be opened serially + sf, err := NewSerialFile(filePath, it.handleHiddenFiles, stat) + if err != nil { + it.err = err + return false } + it.curName = stat.Name() + it.curFile = sf + return true +} + +func (it *serialIterator) Err() error { + return it.err +} + +func (f *serialFile) Entries() (DirIterator, error) { + return &serialIterator{ + path: f.path, + files: f.files, + handleHiddenFiles: f.handleHiddenFiles, + }, nil +} + +func (f *serialFile) NextFile() (string, File, error) { // if there aren't any files left in the root directory, we're done if len(f.files) == 0 { return "", nil, io.EOF @@ -101,7 +163,8 @@ func (f *serialFile) Stat() os.FileInfo { func (f *serialFile) Size() (int64, error) { if !f.stat.IsDir() { - return f.stat.Size(), nil + //something went terribly, terribly wrong + return 0, errors.New("serialFile is not a directory") } var du int64 @@ -119,4 +182,15 @@ func (f *serialFile) Size() (int64, error) { return du, err } +func castRegular(f File) Regular { + r, _ := f.(Regular) + return r +} + +func castDir(f File) Directory { + d, _ := f.(Directory) + return d +} + var _ Directory = &serialFile{} +var _ DirIterator = &serialIterator{} diff --git a/slicefile.go b/slicefile.go index 0d78e16..62c299f 100644 --- a/slicefile.go +++ b/slicefile.go @@ -1,33 +1,76 @@ package files -import ( - "io" -) +type fileEntry struct { + name string + file File +} + +func (e fileEntry) Name() string { + return e.name +} + +func (e fileEntry) File() File { + return e.file +} + +func (e fileEntry) Regular() Regular { + return castRegular(e.file) +} + +func (e fileEntry) Dir() Directory { + return castDir(e.file) +} + +func FileEntry(name string, file File) DirEntry { + return fileEntry{ + name: name, + file: file, + } +} + +type sliceIterator struct { + files []DirEntry + n int +} + +func (it *sliceIterator) Name() string { + return it.files[it.n].Name() +} + +func (it *sliceIterator) File() File { + return it.files[it.n].File() +} + +func (it *sliceIterator) Regular() Regular { + return it.files[it.n].Regular() +} -type FileEntry struct { - File File - Name string +func (it *sliceIterator) Dir() Directory { + return it.files[it.n].Dir() +} + +func (it *sliceIterator) Next() bool { + it.n++ + return it.n < len(it.files) +} + +func (it *sliceIterator) Err() error { + return nil } // SliceFile implements File, and provides simple directory handling. // It contains children files, and is created from a `[]File`. // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { - files []FileEntry - n int + files []DirEntry } -func NewSliceFile(files []FileEntry) Directory { - return &SliceFile{files, 0} +func NewSliceFile(files []DirEntry) Directory { + return &SliceFile{files} } -func (f *SliceFile) NextFile() (string, File, error) { - if f.n >= len(f.files) { - return "", nil, io.EOF - } - file := f.files[f.n] - f.n++ - return file.Name, file.File, nil +func (f *SliceFile) Entries() (DirIterator, error) { + return &sliceIterator{files: f.files, n: -1}, nil } func (f *SliceFile) Close() error { @@ -42,7 +85,7 @@ func (f *SliceFile) Size() (int64, error) { var size int64 for _, file := range f.files { - s, err := file.File.Size() + s, err := file.File().Size() if err != nil { return 0, err } @@ -53,3 +96,4 @@ func (f *SliceFile) Size() (int64, error) { } var _ Directory = &SliceFile{} +var _ DirEntry = fileEntry{} diff --git a/webfile.go b/webfile.go index 5638f77..c8e930a 100644 --- a/webfile.go +++ b/webfile.go @@ -11,8 +11,8 @@ import ( // from a Web URL (http). A GET request will be performed // against the source when calling Read(). type WebFile struct { - body io.ReadCloser - url *url.URL + body io.ReadCloser + url *url.URL contentLength int64 } @@ -61,5 +61,4 @@ func (wf *WebFile) Size() (int64, error) { return wf.contentLength, nil } - var _ Regular = &WebFile{} From 960ec1edab9061dc7b8d2ae9c86e2784706d748e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 10 Nov 2018 00:28:12 +0100 Subject: [PATCH 07/25] Rename File/Regular to Node/File --- file.go | 41 ++++++++++++++++++----------------------- file_test.go | 6 +++--- is_hidden.go | 2 +- is_hidden_windows.go | 2 +- linkfile.go | 4 ++-- multifilereader.go | 14 +++++++------- multifilereader_test.go | 6 +++--- multipartfile.go | 14 +++++++------- readerfile.go | 4 ++-- serialfile.go | 18 +++++++++--------- slicefile.go | 22 +++++++++++----------- webfile.go | 2 +- 12 files changed, 65 insertions(+), 70 deletions(-) diff --git a/file.go b/file.go index 693baab..6e6a249 100644 --- a/file.go +++ b/file.go @@ -13,13 +13,8 @@ var ( ErrNotSupported = errors.New("operation not supported") ) -// File is an interface that provides functionality for handling -// files/directories as values that can be supplied to commands. For -// directories, child files are accessed serially by calling `NextFile()` -// -// Read/Seek methods are only valid for files -// NextFile method is only valid for directories -type File interface { +// Node is a common interface for files, directories and other special files +type Node interface { io.Closer // Size returns size of this file (if this file is a directory, total size of @@ -28,9 +23,9 @@ type File interface { Size() (int64, error) } -// Regular represents a regular Unix file -type Regular interface { - File +// Node represents a regular Unix file +type File interface { + Node io.Reader io.Seeker @@ -38,18 +33,18 @@ type Regular interface { // DirEntry exposes information about a directory entry type DirEntry interface { - // Name returns the base name of this entry, which is the base name of - // the referenced file + // Name returns base name of this entry, which is the base name of referenced + // file Name() string - // File returns the file referenced by this DirEntry - File() File + // Node returns the file referenced by this DirEntry + Node() Node - // Regular is an alias for ent.File().(Regular). If the file isn't a regular + // File is an alias for ent.Node().(File). If the file isn't a regular // file, nil value will be returned - Regular() Regular + File() File - // Dir is an alias for ent.File().(directory). If the file isn't a directory, + // Dir is an alias for ent.Node().(directory). If the file isn't a directory, // nil value will be returned Dir() Directory } @@ -63,18 +58,18 @@ type DirIterator interface { // to Next() and after Next() returned false may result in undefined behavior DirEntry - // Next advances the iterator to the next file. + // Next advances iterator to the next file. Next() bool - // Err may return an error after the previous call to Next() returned `false`. - // If the previous call to Next() returned `true`, Err() is guaranteed to + // Err may return an error after previous call to Next() returned `false`. + // If previous call to Next() returned `true`, Err() is guaranteed to // return nil Err() error } // Directory is a special file which can link to any number of files. type Directory interface { - File + Node // Entries returns a stateful iterator over directory entries. // @@ -83,7 +78,7 @@ type Directory interface { // it := dir.Entries() // for it.Next() { // name := it.Name() - // file := it.File() + // file := it.Node() // [...] // } // if it.Err() != nil { @@ -105,7 +100,7 @@ type Directory interface { // FileInfo exposes information on files in local filesystem type FileInfo interface { - File + Node // AbsPath returns full real file path. AbsPath() string diff --git a/file_test.go b/file_test.go index cbf227c..8fa2a87 100644 --- a/file_test.go +++ b/file_test.go @@ -25,7 +25,7 @@ func TestSliceFiles(t *testing.T) { if !it.Next() { t.Fatal("Expected a file") } - rf := it.Regular() + rf := it.File() if rf == nil { t.Fatal("Expected a regular file") } @@ -104,7 +104,7 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - mf, ok := mpf.(Regular) + mf, ok := mpf.(File) if !ok { t.Fatal("Expected file to not be a directory") } @@ -147,7 +147,7 @@ anotherfile if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - mf, ok = mpf.(Regular) + mf, ok = mpf.(File) if !ok { t.Fatal("Expected file to not be a directory") } diff --git a/is_hidden.go b/is_hidden.go index d5bc886..4ebca60 100644 --- a/is_hidden.go +++ b/is_hidden.go @@ -7,7 +7,7 @@ import ( "strings" ) -func IsHidden(name string, f File) bool { +func IsHidden(name string, f Node) bool { fName := filepath.Base(name) if strings.HasPrefix(fName, ".") && len(fName) > 1 { diff --git a/is_hidden_windows.go b/is_hidden_windows.go index 6f9568a..7419f93 100644 --- a/is_hidden_windows.go +++ b/is_hidden_windows.go @@ -9,7 +9,7 @@ import ( windows "golang.org/x/sys/windows" ) -func IsHidden(name string, f File) bool { +func IsHidden(name string, f Node) bool { fName := filepath.Base(name) diff --git a/linkfile.go b/linkfile.go index df334d5..11f7c92 100644 --- a/linkfile.go +++ b/linkfile.go @@ -13,7 +13,7 @@ type Symlink struct { reader io.Reader } -func NewLinkFile(target string, stat os.FileInfo) Regular { +func NewLinkFile(target string, stat os.FileInfo) File { return &Symlink{ Target: target, stat: stat, @@ -45,4 +45,4 @@ func (lf *Symlink) Size() (int64, error) { return 0, ErrNotSupported } -var _ Regular = &Symlink{} +var _ File = &Symlink{} diff --git a/multifilereader.go b/multifilereader.go index 8817ad8..f5f35b4 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -11,7 +11,7 @@ import ( "sync" ) -// MultiFileReader reads from a `commands.File` (which can be a directory of files +// MultiFileReader reads from a `commands.Node` (which can be a directory of files // or a regular file) as HTTP multipart encoded data. type MultiFileReader struct { io.Reader @@ -20,7 +20,7 @@ type MultiFileReader struct { files []DirIterator path []string - currentFile File + currentFile Node buf bytes.Buffer mpWriter *multipart.Writer closed bool @@ -86,7 +86,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // handle starting a new file part if !mfr.closed { - mfr.currentFile = entry.File() + mfr.currentFile = entry.Node() // write the boundary and headers header := make(textproto.MIMEHeader) @@ -95,7 +95,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { var contentType string - switch f := entry.File().(type) { + switch f := entry.Node().(type) { case *Symlink: contentType = "application/symlink" case Directory: @@ -107,7 +107,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { mfr.files = append(mfr.files, newIt) mfr.path = append(mfr.path, entry.Name()) contentType = "application/x-directory" - case Regular: + case File: // otherwise, use the file as a reader to read its contents contentType = "application/octet-stream" default: @@ -115,7 +115,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } header.Set("Content-Type", contentType) - if rf, ok := entry.File().(FileInfo); ok { + if rf, ok := entry.Node().(FileInfo); ok { header.Set("abspath", rf.AbsPath()) } @@ -133,7 +133,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { // otherwise, read from file data switch f := mfr.currentFile.(type) { - case Regular: + case File: written, err = f.Read(buf) if err != io.EOF { return written, err diff --git a/multifilereader_test.go b/multifilereader_test.go index dd7dac9..a3cf1ee 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -94,7 +94,7 @@ func TestOutput(t *testing.T) { if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - mpr, ok := mpf.(Regular) + mpr, ok := mpf.(File) if !ok { t.Fatal("Expected file to be a regular file") } @@ -132,7 +132,7 @@ func TestOutput(t *testing.T) { if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } - if _, ok := child.(Regular); !ok { + if _, ok := child.(File); !ok { t.Fatal("Expected file to not be a directory") } if cname != "a.txt" { @@ -147,7 +147,7 @@ func TestOutput(t *testing.T) { if child == nil || err != nil { t.Fatal("Expected to be able to read a child file") } - if _, ok := child.(Regular); !ok { + if _, ok := child.(File); !ok { t.Fatal("Expected file to not be a directory") } if cname != "b.txt" { diff --git a/multipartfile.go b/multipartfile.go index be7e221..1552543 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -22,17 +22,17 @@ const ( var ErrPartOutsideParent = errors.New("file outside parent dir") -// MultipartFile implements File, and is created from a `multipart.Part`. +// MultipartFile implements Node, and is created from a `multipart.Part`. // It can be either a directory or file (checked by calling `IsDirectory()`). type MultipartFile struct { - File + Node Part *multipart.Part Reader PartReader Mediatype string } -func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, error) { +func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Node, error) { if !isDirectory(mediatype) { return nil, ErrNotDirectory } @@ -45,7 +45,7 @@ func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (File, er return f, nil } -func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (string, File, error) { +func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (string, Node, error) { f := &MultipartFile{ Part: part, Reader: reader, @@ -97,7 +97,7 @@ func isDirectory(mediatype string) bool { type multipartIterator struct { f *MultipartFile - curFile File + curFile Node curName string err error } @@ -106,11 +106,11 @@ func (it *multipartIterator) Name() string { return it.curName } -func (it *multipartIterator) File() File { +func (it *multipartIterator) File() Node { return it.curFile } -func (it *multipartIterator) Regular() Regular { +func (it *multipartIterator) Regular() File { return castRegular(it.File()) } diff --git a/readerfile.go b/readerfile.go index 7db7e96..261273c 100644 --- a/readerfile.go +++ b/readerfile.go @@ -14,7 +14,7 @@ type ReaderFile struct { stat os.FileInfo } -func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) Regular { +func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) File { return &ReaderFile{"", reader, stat} } @@ -58,5 +58,5 @@ func (f *ReaderFile) Seek(offset int64, whence int) (int64, error) { return 0, ErrNotSupported } -var _ Regular = &ReaderFile{} +var _ File = &ReaderFile{} var _ FileInfo = &ReaderFile{} diff --git a/serialfile.go b/serialfile.go index 77afafa..cbfecc2 100644 --- a/serialfile.go +++ b/serialfile.go @@ -10,7 +10,7 @@ import ( "strings" ) -// serialFile implements File, and reads from a path on the OS filesystem. +// serialFile implements Node, and reads from a path on the OS filesystem. // No more than one file will be opened at a time (directories will advance // to the next file when NextFile() is called). type serialFile struct { @@ -26,13 +26,13 @@ type serialIterator struct { path string curName string - curFile File + curFile Node err error } // TODO: test/document limitations -func NewSerialFile(path string, hidden bool, stat os.FileInfo) (File, error) { +func NewSerialFile(path string, hidden bool, stat os.FileInfo) (Node, error) { switch mode := stat.Mode(); { case mode.IsRegular(): file, err := os.Open(path) @@ -63,11 +63,11 @@ func (it *serialIterator) Name() string { return it.curName } -func (it *serialIterator) File() File { +func (it *serialIterator) File() Node { return it.curFile } -func (it *serialIterator) Regular() Regular { +func (it *serialIterator) Regular() File { return castRegular(it.File()) } @@ -121,7 +121,7 @@ func (f *serialFile) Entries() (DirIterator, error) { }, nil } -func (f *serialFile) NextFile() (string, File, error) { +func (f *serialFile) NextFile() (string, Node, error) { // if there aren't any files left in the root directory, we're done if len(f.files) == 0 { return "", nil, io.EOF @@ -182,12 +182,12 @@ func (f *serialFile) Size() (int64, error) { return du, err } -func castRegular(f File) Regular { - r, _ := f.(Regular) +func castRegular(f Node) File { + r, _ := f.(File) return r } -func castDir(f File) Directory { +func castDir(f Node) Directory { d, _ := f.(Directory) return d } diff --git a/slicefile.go b/slicefile.go index 62c299f..ddc2632 100644 --- a/slicefile.go +++ b/slicefile.go @@ -2,18 +2,18 @@ package files type fileEntry struct { name string - file File + file Node } func (e fileEntry) Name() string { return e.name } -func (e fileEntry) File() File { +func (e fileEntry) File() Node { return e.file } -func (e fileEntry) Regular() Regular { +func (e fileEntry) Regular() File { return castRegular(e.file) } @@ -21,7 +21,7 @@ func (e fileEntry) Dir() Directory { return castDir(e.file) } -func FileEntry(name string, file File) DirEntry { +func FileEntry(name string, file Node) DirEntry { return fileEntry{ name: name, file: file, @@ -37,12 +37,12 @@ func (it *sliceIterator) Name() string { return it.files[it.n].Name() } -func (it *sliceIterator) File() File { - return it.files[it.n].File() +func (it *sliceIterator) File() Node { + return it.files[it.n].Node() } -func (it *sliceIterator) Regular() Regular { - return it.files[it.n].Regular() +func (it *sliceIterator) Regular() File { + return it.files[it.n].File() } func (it *sliceIterator) Dir() Directory { @@ -58,8 +58,8 @@ func (it *sliceIterator) Err() error { return nil } -// SliceFile implements File, and provides simple directory handling. -// It contains children files, and is created from a `[]File`. +// SliceFile implements Node, and provides simple directory handling. +// It contains children files, and is created from a `[]Node`. // SliceFiles are always directories, and can't be read from or closed. type SliceFile struct { files []DirEntry @@ -85,7 +85,7 @@ func (f *SliceFile) Size() (int64, error) { var size int64 for _, file := range f.files { - s, err := file.File().Size() + s, err := file.Node().Size() if err != nil { return 0, err } diff --git a/webfile.go b/webfile.go index c8e930a..6c1cc3f 100644 --- a/webfile.go +++ b/webfile.go @@ -61,4 +61,4 @@ func (wf *WebFile) Size() (int64, error) { return wf.contentLength, nil } -var _ Regular = &WebFile{} +var _ File = &WebFile{} From 401cce734f3daaff69f97ebfd10460624dc3a6b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 14 Nov 2018 18:26:22 +0100 Subject: [PATCH 08/25] move the MultipartFile iterator note --- file.go | 11 ++--------- multipartfile.go | 7 +++++-- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/file.go b/file.go index 6e6a249..e416650 100644 --- a/file.go +++ b/file.go @@ -86,15 +86,8 @@ type Directory interface { // } // // Note: - // - Below limitations aren't applicable to all implementations, consult - // your implementations manual before using this interface in a way that - // doesn't meet these constraints - // - Some implementations may only allow reading in order - so if the iterator - // returns a directory you must iterate over it's entries first before - // calling Next again - // - Order is not guaranteed - // - Depending on implementation it may not be safe to iterate multiple - // 'branches' in parallel + // - Some implementations of this functions may define some constraints in how + // it can be used Entries() (DirIterator, error) } diff --git a/multipartfile.go b/multipartfile.go index 1552543..b79efa5 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -23,7 +23,10 @@ const ( var ErrPartOutsideParent = errors.New("file outside parent dir") // MultipartFile implements Node, and is created from a `multipart.Part`. -// It can be either a directory or file (checked by calling `IsDirectory()`). +// +// Note: iterating entries can be done only once and must be done in order, +// meaning that when iterator returns a directory, you MUST read all it's +// children before calling Next again type MultipartFile struct { Node @@ -32,7 +35,7 @@ type MultipartFile struct { Mediatype string } -func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Node, error) { +func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Directory, error) { if !isDirectory(mediatype) { return nil, ErrNotDirectory } From fbb870de8d52b2ea0d9388334c7ccd3603199319 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 16 Nov 2018 01:31:38 +0100 Subject: [PATCH 09/25] Fix some iterators after rename --- multipartfile.go | 8 ++++---- serialfile.go | 8 ++++---- slicefile.go | 8 ++++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/multipartfile.go b/multipartfile.go index b79efa5..fc5610b 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -109,16 +109,16 @@ func (it *multipartIterator) Name() string { return it.curName } -func (it *multipartIterator) File() Node { +func (it *multipartIterator) Node() Node { return it.curFile } -func (it *multipartIterator) Regular() File { - return castRegular(it.File()) +func (it *multipartIterator) File() File { + return castRegular(it.Node()) } func (it *multipartIterator) Dir() Directory { - return castDir(it.File()) + return castDir(it.Node()) } func (it *multipartIterator) Next() bool { diff --git a/serialfile.go b/serialfile.go index cbfecc2..f4e62b5 100644 --- a/serialfile.go +++ b/serialfile.go @@ -63,16 +63,16 @@ func (it *serialIterator) Name() string { return it.curName } -func (it *serialIterator) File() Node { +func (it *serialIterator) Node() Node { return it.curFile } -func (it *serialIterator) Regular() File { - return castRegular(it.File()) +func (it *serialIterator) File() File { + return castRegular(it.Node()) } func (it *serialIterator) Dir() Directory { - return castDir(it.File()) + return castDir(it.Node()) } func (it *serialIterator) Next() bool { diff --git a/slicefile.go b/slicefile.go index ddc2632..6400d47 100644 --- a/slicefile.go +++ b/slicefile.go @@ -9,11 +9,11 @@ func (e fileEntry) Name() string { return e.name } -func (e fileEntry) File() Node { +func (e fileEntry) Node() Node { return e.file } -func (e fileEntry) Regular() File { +func (e fileEntry) File() File { return castRegular(e.file) } @@ -37,11 +37,11 @@ func (it *sliceIterator) Name() string { return it.files[it.n].Name() } -func (it *sliceIterator) File() Node { +func (it *sliceIterator) Node() Node { return it.files[it.n].Node() } -func (it *sliceIterator) Regular() File { +func (it *sliceIterator) File() File { return it.files[it.n].File() } From a557fc4681d48dabfced24efedb2bd7456ba8c6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 19 Nov 2018 03:54:49 +0100 Subject: [PATCH 10/25] Fix errors import --- webfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webfile.go b/webfile.go index 6c1cc3f..dbc813b 100644 --- a/webfile.go +++ b/webfile.go @@ -1,7 +1,7 @@ package files import ( - "github.com/pkg/errors" + "errors" "io" "net/http" "net/url" From 89a3f6de696e8d4c170dfcc5157c8e0187f9cc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 24 Nov 2018 03:51:16 +0100 Subject: [PATCH 11/25] Don't return error from Entries --- file.go | 2 +- file_test.go | 5 +---- multifilereader.go | 17 +++++------------ multifilereader_test.go | 12 +++--------- multipartfile.go | 4 ++-- serialfile.go | 4 ++-- slicefile.go | 4 ++-- 7 files changed, 16 insertions(+), 32 deletions(-) diff --git a/file.go b/file.go index e416650..d975f11 100644 --- a/file.go +++ b/file.go @@ -88,7 +88,7 @@ type Directory interface { // Note: // - Some implementations of this functions may define some constraints in how // it can be used - Entries() (DirIterator, error) + Entries() DirIterator } // FileInfo exposes information on files in local filesystem diff --git a/file_test.go b/file_test.go index 8fa2a87..81bcabe 100644 --- a/file_test.go +++ b/file_test.go @@ -17,10 +17,7 @@ func TestSliceFiles(t *testing.T) { buf := make([]byte, 20) sf := NewSliceFile(files) - it, err := sf.Entries() - if err != nil { - t.Fatal(err) - } + it := sf.Entries() if !it.Next() { t.Fatal("Expected a file") diff --git a/multifilereader.go b/multifilereader.go index f5f35b4..52d9f4c 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -35,10 +35,7 @@ type MultiFileReader struct { // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { - it, err := file.Entries() - if err != nil { - return nil, err - } + it := file.Entries() mfr := &MultiFileReader{ files: []DirIterator{it}, @@ -72,13 +69,13 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { } if !mfr.files[len(mfr.files)-1].Next() { + if mfr.files[len(mfr.files)-1].Err() != nil { + return 0, err + } mfr.files = mfr.files[:len(mfr.files)-1] mfr.path = mfr.path[:len(mfr.path)-1] continue } - if mfr.files[len(mfr.files)-1].Err() != nil { - return 0, mfr.files[len(mfr.files)-1].Err() - } entry = mfr.files[len(mfr.files)-1] } @@ -99,11 +96,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { case *Symlink: contentType = "application/symlink" case Directory: - newIt, err := f.Entries() - if err != nil { - return 0, err - } - + newIt := f.Entries() mfr.files = append(mfr.files, newIt) mfr.path = append(mfr.path, entry.Name()) contentType = "application/x-directory" diff --git a/multifilereader_test.go b/multifilereader_test.go index a3cf1ee..8df39a8 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -41,10 +41,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { if !ok { t.Fatal("Expected a directory") } - it, err := md.Entries() - if err != nil { - t.Fatal(err) - } + it := md.Entries() if !it.Next() || it.Name() != "file.txt" { t.Fatal("iterator didn't work as expected") @@ -54,10 +51,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - subIt, err := it.Dir().Entries() - if err != nil { - t.Fatal(err) - } + subIt := it.Dir().Entries() if !subIt.Next() || subIt.Name() != "a.txt" || subIt.Dir() != nil { t.Fatal("iterator didn't work as expected") @@ -154,7 +148,7 @@ func TestOutput(t *testing.T) { t.Fatal("Expected filename to be \"b.txt\"") } - it, err := mpd.Entries() + it := mpd.Entries() if it.Next() { t.Fatal("Expected to get false") } diff --git a/multipartfile.go b/multipartfile.go index fc5610b..18bf12c 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -154,8 +154,8 @@ func (it *multipartIterator) Err() error { panic("implement me") } -func (f *MultipartFile) Entries() (DirIterator, error) { - return &multipartIterator{f: f}, nil +func (f *MultipartFile) Entries() DirIterator { + return &multipartIterator{f: f} } func (f *MultipartFile) fileName() string { diff --git a/serialfile.go b/serialfile.go index f4e62b5..e379574 100644 --- a/serialfile.go +++ b/serialfile.go @@ -113,12 +113,12 @@ func (it *serialIterator) Err() error { return it.err } -func (f *serialFile) Entries() (DirIterator, error) { +func (f *serialFile) Entries() DirIterator { return &serialIterator{ path: f.path, files: f.files, handleHiddenFiles: f.handleHiddenFiles, - }, nil + } } func (f *serialFile) NextFile() (string, Node, error) { diff --git a/slicefile.go b/slicefile.go index 6400d47..82ce223 100644 --- a/slicefile.go +++ b/slicefile.go @@ -69,8 +69,8 @@ func NewSliceFile(files []DirEntry) Directory { return &SliceFile{files} } -func (f *SliceFile) Entries() (DirIterator, error) { - return &sliceIterator{files: f.files, n: -1}, nil +func (f *SliceFile) Entries() DirIterator { + return &sliceIterator{files: f.files, n: -1} } func (f *SliceFile) Close() error { From 34b159078a3a709315c69c133f89638e8a7cbdf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 26 Nov 2018 09:10:51 +0100 Subject: [PATCH 12/25] Fix error return in multipartfile --- multipartfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/multipartfile.go b/multipartfile.go index 18bf12c..86cef0f 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -151,7 +151,7 @@ func (it *multipartIterator) Next() bool { } func (it *multipartIterator) Err() error { - panic("implement me") + return it.err } func (f *MultipartFile) Entries() DirIterator { From 23e95656f2e4e8325b03e72df7b9f0ba6197b45a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 29 Nov 2018 20:19:04 +0100 Subject: [PATCH 13/25] Don't expose io.EOF in multifilereader --- multifilereader.go | 2 +- multifilereader_test.go | 10 +++++++--- multipartfile.go | 14 +++++++++++++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/multifilereader.go b/multifilereader.go index 52d9f4c..758b501 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -70,7 +70,7 @@ func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { if !mfr.files[len(mfr.files)-1].Next() { if mfr.files[len(mfr.files)-1].Err() != nil { - return 0, err + return 0, mfr.files[len(mfr.files)-1].Err() } mfr.files = mfr.files[:len(mfr.files)-1] mfr.path = mfr.path[:len(mfr.path)-1] diff --git a/multifilereader_test.go b/multifilereader_test.go index 8df39a8..46efe0e 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -61,16 +61,20 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if subIt.Next() { + if subIt.Next() || it.Err() != nil { t.Fatal("iterator didn't work as expected") } // try to break internal state - if subIt.Next() { + if subIt.Next() || it.Err() != nil { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil { + if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil || it.Err() != nil { + t.Fatal("iterator didn't work as expected") + } + + if it.Next() || it.Err() != nil { t.Fatal("iterator didn't work as expected") } } diff --git a/multipartfile.go b/multipartfile.go index 86cef0f..651bfdb 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -2,6 +2,7 @@ package files import ( "errors" + "io" "io/ioutil" "mime" "mime/multipart" @@ -127,6 +128,9 @@ func (it *multipartIterator) Next() bool { } part, err := it.f.Reader.NextPart() if err != nil { + if err == io.EOF { + return false + } it.err = err return false } @@ -198,7 +202,15 @@ func (pr *peekReader) NextPart() (*multipart.Part, error) { return p, nil } - return pr.r.NextPart() + if pr.r == nil { + return nil, io.EOF + } + + p, err := pr.r.NextPart() + if err == io.EOF { + pr.r = nil + } + return p, err } func (pr *peekReader) put(p *multipart.Part) error { From 2936a34a266263beb3a9518ab8b44907840b6ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sat, 1 Dec 2018 00:35:08 +0100 Subject: [PATCH 14/25] gx publish 2.0.0 --- .gx/lastpubver | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gx/lastpubver b/.gx/lastpubver index 76d527b..cd21251 100644 --- a/.gx/lastpubver +++ b/.gx/lastpubver @@ -1 +1 @@ -1.0.1: QmZMWMvWMVKCbHetJ4RgndbuEF1io2UpUxwQwtNjtYPzSC +2.0.0: QmXy8ePeFSRHMUJMpjJ32fKS6qgbGWc2P6oJNa6bFPiKqN diff --git a/package.json b/package.json index 167621a..457913c 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,6 @@ "license": "", "name": "go-ipfs-files", "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", - "version": "1.0.1" + "version": "2.0.0" } From f732b393dcf123f84fdbceb88985e8e85466794a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 3 Dec 2018 14:20:03 +0100 Subject: [PATCH 15/25] files2.0: Add some convenience functions --- file.go | 8 ----- file_test.go | 16 ++++----- linkfile.go | 5 +++ multifilereader_test.go | 39 ++++++++++---------- multipartfile.go | 8 ----- serialfile.go | 18 ---------- slicefile.go | 16 --------- util.go | 80 +++++++++++++++++++++++++++++++++++++++++ 8 files changed, 110 insertions(+), 80 deletions(-) create mode 100644 util.go diff --git a/file.go b/file.go index d975f11..41545be 100644 --- a/file.go +++ b/file.go @@ -39,14 +39,6 @@ type DirEntry interface { // Node returns the file referenced by this DirEntry Node() Node - - // File is an alias for ent.Node().(File). If the file isn't a regular - // file, nil value will be returned - File() File - - // Dir is an alias for ent.Node().(directory). If the file isn't a directory, - // nil value will be returned - Dir() Directory } // DirIterator is a iterator over directory entries. diff --git a/file_test.go b/file_test.go index 81bcabe..ef93cf7 100644 --- a/file_test.go +++ b/file_test.go @@ -2,27 +2,25 @@ package files import ( "io" - "io/ioutil" "mime/multipart" "strings" "testing" ) func TestSliceFiles(t *testing.T) { - files := []DirEntry{ - FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("Some text!\n")), nil)), - FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), - FileEntry("", NewReaderFile(ioutil.NopCloser(strings.NewReader("boop")), nil)), - } + sf := DirFrom(map[string]Node{ + "1": FileFrom([]byte("Some text!\n")), + "2": FileFrom([]byte("beep")), + "3": FileFrom([]byte("boop")), + }) buf := make([]byte, 20) - sf := NewSliceFile(files) it := sf.Entries() if !it.Next() { t.Fatal("Expected a file") } - rf := it.File() + rf := ToFile(it.Node()) if rf == nil { t.Fatal("Expected a regular file") } @@ -48,7 +46,7 @@ func TestSliceFiles(t *testing.T) { func TestReaderFiles(t *testing.T) { message := "beep boop" - rf := NewReaderFile(ioutil.NopCloser(strings.NewReader(message)), nil) + rf := FileFrom([]byte(message)) buf := make([]byte, len(message)) if n, err := rf.Read(buf); n == 0 || err != nil { diff --git a/linkfile.go b/linkfile.go index 11f7c92..409309b 100644 --- a/linkfile.go +++ b/linkfile.go @@ -45,4 +45,9 @@ func (lf *Symlink) Size() (int64, error) { return 0, ErrNotSupported } +func ToSymlink(n Node) *Symlink { + l, _ := n.(*Symlink) + return l +} + var _ File = &Symlink{} diff --git a/multifilereader_test.go b/multifilereader_test.go index 46efe0e..154c5b2 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -2,24 +2,21 @@ package files import ( "io" - "io/ioutil" "mime/multipart" - "strings" "testing" ) var text = "Some text! :)" func getTestMultiFileReader(t *testing.T) *MultiFileReader { - fileset := []DirEntry{ - FileEntry("file.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader(text)), nil)), - FileEntry("boop", NewSliceFile([]DirEntry{ - FileEntry("a.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bleep")), nil)), - FileEntry("b.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("bloop")), nil)), - })), - FileEntry("beep.txt", NewReaderFile(ioutil.NopCloser(strings.NewReader("beep")), nil)), - } - sf := NewSliceFile(fileset) + sf := DirFrom(map[string]Node{ + "file.txt": FileFrom([]byte(text)), + "boop": DirFrom(map[string]Node{ + "a.txt": FileFrom([]byte("bleep")), + "b.txt": FileFrom([]byte("bloop")), + }), + "beep.txt": FileFrom([]byte("beep")), + }) // testing output by reading it with the go stdlib "mime/multipart" Reader r, err := NewMultiFileReader(sf, true) @@ -43,21 +40,21 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { } it := md.Entries() - if !it.Next() || it.Name() != "file.txt" { + if !it.Next() || it.Name() != "beep.txt" { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "boop" || it.Dir() == nil { + if !it.Next() || it.Name() != "boop" || DirFrom(it) == nil { t.Fatal("iterator didn't work as expected") } - subIt := it.Dir().Entries() + subIt := DirFrom(it).Entries() - if !subIt.Next() || subIt.Name() != "a.txt" || subIt.Dir() != nil { + if !subIt.Next() || subIt.Name() != "a.txt" || DirFrom(subIt) != nil { t.Fatal("iterator didn't work as expected") } - if !subIt.Next() || subIt.Name() != "b.txt" || subIt.Dir() != nil { + if !subIt.Next() || subIt.Name() != "b.txt" || DirFrom(subIt) != nil { t.Fatal("iterator didn't work as expected") } @@ -70,7 +67,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "beep.txt" || it.Dir() != nil || it.Err() != nil { + if !it.Next() || it.Name() != "file.txt" || DirFrom(it) != nil || it.Err() != nil { t.Fatal("iterator didn't work as expected") } @@ -96,13 +93,13 @@ func TestOutput(t *testing.T) { if !ok { t.Fatal("Expected file to be a regular file") } - if mpname != "file.txt" { + if mpname != "beep.txt" { t.Fatal("Expected filename to be \"file.txt\"") } - if n, err := mpr.Read(buf); n != len(text) || err != nil { + if n, err := mpr.Read(buf); n != 4 || err != nil { t.Fatal("Expected to read from file", n, err) } - if string(buf[:len(text)]) != text { + if string(buf[:4]) != "beep" { t.Fatal("Data read was different than expected") } @@ -165,7 +162,7 @@ func TestOutput(t *testing.T) { if mpf == nil || err != nil { t.Fatal("Expected non-nil MultipartFile, nil error") } - if mpname != "beep.txt" { + if mpname != "file.txt" { t.Fatal("Expected filename to be \"b.txt\"") } diff --git a/multipartfile.go b/multipartfile.go index 651bfdb..14b4cba 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -114,14 +114,6 @@ func (it *multipartIterator) Node() Node { return it.curFile } -func (it *multipartIterator) File() File { - return castRegular(it.Node()) -} - -func (it *multipartIterator) Dir() Directory { - return castDir(it.Node()) -} - func (it *multipartIterator) Next() bool { if it.f.Reader == nil { return false diff --git a/serialfile.go b/serialfile.go index e379574..e29752d 100644 --- a/serialfile.go +++ b/serialfile.go @@ -67,14 +67,6 @@ func (it *serialIterator) Node() Node { return it.curFile } -func (it *serialIterator) File() File { - return castRegular(it.Node()) -} - -func (it *serialIterator) Dir() Directory { - return castDir(it.Node()) -} - func (it *serialIterator) Next() bool { // if there aren't any files left in the root directory, we're done if len(it.files) == 0 { @@ -182,15 +174,5 @@ func (f *serialFile) Size() (int64, error) { return du, err } -func castRegular(f Node) File { - r, _ := f.(File) - return r -} - -func castDir(f Node) Directory { - d, _ := f.(Directory) - return d -} - var _ Directory = &serialFile{} var _ DirIterator = &serialIterator{} diff --git a/slicefile.go b/slicefile.go index 82ce223..715cee3 100644 --- a/slicefile.go +++ b/slicefile.go @@ -13,14 +13,6 @@ func (e fileEntry) Node() Node { return e.file } -func (e fileEntry) File() File { - return castRegular(e.file) -} - -func (e fileEntry) Dir() Directory { - return castDir(e.file) -} - func FileEntry(name string, file Node) DirEntry { return fileEntry{ name: name, @@ -41,14 +33,6 @@ func (it *sliceIterator) Node() Node { return it.files[it.n].Node() } -func (it *sliceIterator) File() File { - return it.files[it.n].File() -} - -func (it *sliceIterator) Dir() Directory { - return it.files[it.n].Dir() -} - func (it *sliceIterator) Next() bool { it.n++ return it.n < len(it.files) diff --git a/util.go b/util.go new file mode 100644 index 0000000..a294190 --- /dev/null +++ b/util.go @@ -0,0 +1,80 @@ +package files + +import ( + "bytes" + "io" + "io/ioutil" + "sort" +) + +// ToFile is an alias for n.(File). If the file isn't a regular file, nil value +// will be returned +func ToFile(n Node) File { + f, _ := n.(File) + return f +} + +// ToDir is an alias for n.(Directory). If the file isn't directory, a nil value +// will be returned +func ToDir(n Node) Directory { + d, _ := n.(Directory) + return d +} + +// FileFrom is a convenience function which tries to extract or create new file +// from provided value. If a passed value can't be turned into a File, nil will +// be returned. +// +// Supported types: +// * files.File (cast from Node) +// * DirEntry / DirIterator (cast from e.Node()) +// * []byte (wrapped into NewReaderFile) +// * io.Reader / io.ReadCloser (wrapped into NewReaderFile) +func FileFrom(n interface{}) File { + switch f := n.(type) { + case File: + return f + case DirEntry: + return ToFile(f.Node()) + case []byte: + return NewReaderFile(ioutil.NopCloser(bytes.NewReader(f)), nil) + case io.ReadCloser: + return NewReaderFile(f, nil) + case io.Reader: + return NewReaderFile(ioutil.NopCloser(f), nil) + default: + return nil + } +} + +// DirFrom is a convenience function which tries to extract or create new +// directory from the provided value. If a passed value can't be turned into a +// Directory, nil will be returned. +// +// Supported types: +// * files.File (cast from Node) +// * DirEntry (cast from e.Node()) +// * DirIterator (current file, cast from e.Node()) +// * []DirEntry (wrapped into NewSliceFile) +// * map[string]Node (wrapped into NewSliceFile) +func DirFrom(n interface{}) Directory { + switch f := n.(type) { + case Directory: + return f + case DirEntry: + return ToDir(f.Node()) + case []DirEntry: + return NewSliceFile(f) + case map[string]Node: + ents := make([]DirEntry, 0, len(f)) + for name, nd := range f { + ents = append(ents, FileEntry(name, nd)) + } + sort.Slice(ents, func(i, j int) bool { + return ents[i].Name() < ents[j].Name() + }) + return NewSliceFile(ents) + default: + return nil + } +} From bc7a700c01a34b4cc17bceb706381060afb30868 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 3 Dec 2018 14:20:43 +0100 Subject: [PATCH 16/25] files2.0: no error from NewMultiFileReader --- multifilereader.go | 4 ++-- multifilereader_test.go | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/multifilereader.go b/multifilereader.go index 758b501..cf3d14c 100644 --- a/multifilereader.go +++ b/multifilereader.go @@ -34,7 +34,7 @@ type MultiFileReader struct { // NewMultiFileReader constructs a MultiFileReader. `file` can be any `commands.Directory`. // If `form` is set to true, the multipart data will have a Content-Type of 'multipart/form-data', // if `form` is false, the Content-Type will be 'multipart/mixed'. -func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { +func NewMultiFileReader(file Directory, form bool) *MultiFileReader { it := file.Entries() mfr := &MultiFileReader{ @@ -45,7 +45,7 @@ func NewMultiFileReader(file Directory, form bool) (*MultiFileReader, error) { } mfr.mpWriter = multipart.NewWriter(&mfr.buf) - return mfr, nil + return mfr } func (mfr *MultiFileReader) Read(buf []byte) (written int, err error) { diff --git a/multifilereader_test.go b/multifilereader_test.go index 154c5b2..a7d6421 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -19,11 +19,7 @@ func getTestMultiFileReader(t *testing.T) *MultiFileReader { }) // testing output by reading it with the go stdlib "mime/multipart" Reader - r, err := NewMultiFileReader(sf, true) - if err != nil { - t.Fatal(err) - } - return r + return NewMultiFileReader(sf, true) } func TestMultiFileReaderToMultiFile(t *testing.T) { From 66bfc7ab36a24fff01f239acf4d381ad755d6faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 3 Dec 2018 14:27:21 +0100 Subject: [PATCH 17/25] gx publish 2.0.1 --- .gx/lastpubver | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gx/lastpubver b/.gx/lastpubver index cd21251..91a48a0 100644 --- a/.gx/lastpubver +++ b/.gx/lastpubver @@ -1 +1 @@ -2.0.0: QmXy8ePeFSRHMUJMpjJ32fKS6qgbGWc2P6oJNa6bFPiKqN +2.0.1: QmPhx9B9cuaXc4vuw62567BF5NxfpsdD1AVE9HbTn7t1Y6 diff --git a/package.json b/package.json index 457913c..a185bf0 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,6 @@ "license": "", "name": "go-ipfs-files", "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", - "version": "2.0.0" + "version": "2.0.1" } From b925960e72275f9cf944e8965ee18838a3b7fc6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 9 Dec 2018 16:40:06 +0100 Subject: [PATCH 18/25] Allow skipping entries in multipartIterator --- multifilereader_test.go | 31 +++++++++++++++++++++++++++ multipartfile.go | 47 +++++++++++++++++++++++++++++------------ 2 files changed, 65 insertions(+), 13 deletions(-) diff --git a/multifilereader_test.go b/multifilereader_test.go index a7d6421..31ef5c0 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -72,6 +72,37 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { } } +func TestMultiFileReaderToMultiFileSkip(t *testing.T) { + mfr := getTestMultiFileReader(t) + mpReader := multipart.NewReader(mfr, mfr.Boundary()) + mf, err := NewFileFromPartReader(mpReader, multipartFormdataType) + if err != nil { + t.Fatal(err) + } + + md, ok := mf.(Directory) + if !ok { + t.Fatal("Expected a directory") + } + it := md.Entries() + + if !it.Next() || it.Name() != "beep.txt" { + t.Fatal("iterator didn't work as expected") + } + + if !it.Next() || it.Name() != "boop" || DirFrom(it) == nil { + t.Fatal("iterator didn't work as expected") + } + + if !it.Next() || it.Name() != "file.txt" || DirFrom(it) != nil || it.Err() != nil { + t.Fatal("iterator didn't work as expected") + } + + if it.Next() || it.Err() != nil { + t.Fatal("iterator didn't work as expected") + } +} + func TestOutput(t *testing.T) { mfr := getTestMultiFileReader(t) mpReader := &peekReader{r: multipart.NewReader(mfr, mfr.Boundary())} diff --git a/multipartfile.go b/multipartfile.go index 14b4cba..7bb535f 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -8,6 +8,7 @@ import ( "mime/multipart" "net/url" "path" + "strings" ) const ( @@ -22,6 +23,7 @@ const ( ) var ErrPartOutsideParent = errors.New("file outside parent dir") +var ErrPartInChildTree = errors.New("file in child tree") // MultipartFile implements Node, and is created from a `multipart.Part`. // @@ -56,7 +58,19 @@ func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (st } dir, base := path.Split(f.fileName()) - if path.Clean(dir) != path.Clean(parent) { + dir = path.Clean(dir) + parent = path.Clean(parent) + if dir == "." { + dir = "" + } + if parent == "." { + parent = "" + } + + if dir != parent { + if strings.HasPrefix(dir, parent) { + return "", nil, ErrPartInChildTree + } return "", nil, ErrPartOutsideParent } @@ -118,21 +132,28 @@ func (it *multipartIterator) Next() bool { if it.f.Reader == nil { return false } - part, err := it.f.Reader.NextPart() - if err != nil { - if err == io.EOF { + var part *multipart.Part + for { + var err error + part, err = it.f.Reader.NextPart() + if err != nil { + if err == io.EOF { + return false + } + it.err = err return false } - it.err = err - return false - } - name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) - if err != ErrPartOutsideParent { - it.curFile = cf - it.curName = name - it.err = err - return err == nil + name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) + if err == ErrPartOutsideParent { + break + } + if err != ErrPartInChildTree { + it.curFile = cf + it.curName = name + it.err = err + return err == nil + } } // we read too much, try to fix this From 26bbce7b6144f0e87f50b04a2ac55d6d360796ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Sun, 9 Dec 2018 17:28:11 +0100 Subject: [PATCH 19/25] Expand docstring for Directory.Entries --- file.go | 12 ++++++++++++ multipartfile.go | 4 ---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/file.go b/file.go index 41545be..15cf108 100644 --- a/file.go +++ b/file.go @@ -80,6 +80,18 @@ type Directory interface { // Note: // - Some implementations of this functions may define some constraints in how // it can be used + // - Each implementation MUST support: + // - Pre-order sequential iteration: + // - Meaning that after calling `Next` you can call `Next` if the returned + // node is a directory or read the returned file + // - Skipping entries: + // - Meaning that if `Next` returns a directory, you can skip reading it's + // entries and skip to next entry. Files don't have to be read in full. + // Note that you can't go back to unread entries, this only allows + // skipping parts of a directory tree + // - This is to allow listing files in a directory without having to read + // the entire tree + // - Entries may not be sorted Entries() DirIterator } diff --git a/multipartfile.go b/multipartfile.go index 7bb535f..c11cc82 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -26,10 +26,6 @@ var ErrPartOutsideParent = errors.New("file outside parent dir") var ErrPartInChildTree = errors.New("file in child tree") // MultipartFile implements Node, and is created from a `multipart.Part`. -// -// Note: iterating entries can be done only once and must be done in order, -// meaning that when iterator returns a directory, you MUST read all it's -// children before calling Next again type MultipartFile struct { Node From 8d1df4d43f7be3059e4291fb55a0e7d47a5b2de5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Mon, 10 Dec 2018 22:57:55 +0100 Subject: [PATCH 20/25] More type-safe utility functions --- file_test.go | 10 ++--- multifilereader_test.go | 26 ++++++------ readerfile.go | 19 ++++++++- slicefile.go => slicedirectory.go | 16 +++++++- util.go | 67 +++---------------------------- 5 files changed, 56 insertions(+), 82 deletions(-) rename slicefile.go => slicedirectory.go (79%) diff --git a/file_test.go b/file_test.go index ef93cf7..af60761 100644 --- a/file_test.go +++ b/file_test.go @@ -8,10 +8,10 @@ import ( ) func TestSliceFiles(t *testing.T) { - sf := DirFrom(map[string]Node{ - "1": FileFrom([]byte("Some text!\n")), - "2": FileFrom([]byte("beep")), - "3": FileFrom([]byte("boop")), + sf := NewMapDirectory(map[string]Node{ + "1": NewBytesFile([]byte("Some text!\n")), + "2": NewBytesFile([]byte("beep")), + "3": NewBytesFile([]byte("boop")), }) buf := make([]byte, 20) @@ -46,7 +46,7 @@ func TestSliceFiles(t *testing.T) { func TestReaderFiles(t *testing.T) { message := "beep boop" - rf := FileFrom([]byte(message)) + rf := NewBytesFile([]byte(message)) buf := make([]byte, len(message)) if n, err := rf.Read(buf); n == 0 || err != nil { diff --git a/multifilereader_test.go b/multifilereader_test.go index 31ef5c0..3357b23 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -9,13 +9,13 @@ import ( var text = "Some text! :)" func getTestMultiFileReader(t *testing.T) *MultiFileReader { - sf := DirFrom(map[string]Node{ - "file.txt": FileFrom([]byte(text)), - "boop": DirFrom(map[string]Node{ - "a.txt": FileFrom([]byte("bleep")), - "b.txt": FileFrom([]byte("bloop")), + sf := NewMapDirectory(map[string]Node{ + "file.txt": NewBytesFile([]byte(text)), + "boop": NewMapDirectory(map[string]Node{ + "a.txt": NewBytesFile([]byte("bleep")), + "b.txt": NewBytesFile([]byte("bloop")), }), - "beep.txt": FileFrom([]byte("beep")), + "beep.txt": NewBytesFile([]byte("beep")), }) // testing output by reading it with the go stdlib "mime/multipart" Reader @@ -40,17 +40,17 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "boop" || DirFrom(it) == nil { + if !it.Next() || it.Name() != "boop" || DirFromEntry(it) == nil { t.Fatal("iterator didn't work as expected") } - subIt := DirFrom(it).Entries() + subIt := DirFromEntry(it).Entries() - if !subIt.Next() || subIt.Name() != "a.txt" || DirFrom(subIt) != nil { + if !subIt.Next() || subIt.Name() != "a.txt" || DirFromEntry(subIt) != nil { t.Fatal("iterator didn't work as expected") } - if !subIt.Next() || subIt.Name() != "b.txt" || DirFrom(subIt) != nil { + if !subIt.Next() || subIt.Name() != "b.txt" || DirFromEntry(subIt) != nil { t.Fatal("iterator didn't work as expected") } @@ -63,7 +63,7 @@ func TestMultiFileReaderToMultiFile(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "file.txt" || DirFrom(it) != nil || it.Err() != nil { + if !it.Next() || it.Name() != "file.txt" || DirFromEntry(it) != nil || it.Err() != nil { t.Fatal("iterator didn't work as expected") } @@ -90,11 +90,11 @@ func TestMultiFileReaderToMultiFileSkip(t *testing.T) { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "boop" || DirFrom(it) == nil { + if !it.Next() || it.Name() != "boop" || DirFromEntry(it) == nil { t.Fatal("iterator didn't work as expected") } - if !it.Next() || it.Name() != "file.txt" || DirFrom(it) != nil || it.Err() != nil { + if !it.Next() || it.Name() != "file.txt" || DirFromEntry(it) != nil || it.Err() != nil { t.Fatal("iterator didn't work as expected") } diff --git a/readerfile.go b/readerfile.go index 261273c..cc965c8 100644 --- a/readerfile.go +++ b/readerfile.go @@ -1,7 +1,9 @@ package files import ( + "bytes" "io" + "io/ioutil" "os" "path/filepath" ) @@ -14,8 +16,21 @@ type ReaderFile struct { stat os.FileInfo } -func NewReaderFile(reader io.ReadCloser, stat os.FileInfo) File { - return &ReaderFile{"", reader, stat} +func NewBytesFile(b []byte) File { + return NewReaderFile(bytes.NewReader(b)) +} + +func NewReaderFile(reader io.Reader) File { + return NewReaderStatFile(reader, nil) +} + +func NewReaderStatFile(reader io.Reader, stat os.FileInfo) File { + rc, ok := reader.(io.ReadCloser) + if !ok { + rc = ioutil.NopCloser(reader) + } + + return &ReaderFile{"", rc, stat} } func NewReaderPathFile(path string, reader io.ReadCloser, stat os.FileInfo) (*ReaderFile, error) { diff --git a/slicefile.go b/slicedirectory.go similarity index 79% rename from slicefile.go rename to slicedirectory.go index 715cee3..d116562 100644 --- a/slicefile.go +++ b/slicedirectory.go @@ -1,5 +1,7 @@ package files +import "sort" + type fileEntry struct { name string file Node @@ -49,7 +51,19 @@ type SliceFile struct { files []DirEntry } -func NewSliceFile(files []DirEntry) Directory { +func NewMapDirectory(f map[string]Node) Directory { + ents := make([]DirEntry, 0, len(f)) + for name, nd := range f { + ents = append(ents, FileEntry(name, nd)) + } + sort.Slice(ents, func(i, j int) bool { + return ents[i].Name() < ents[j].Name() + }) + + return NewSliceDirectory(ents) +} + +func NewSliceDirectory(files []DirEntry) Directory { return &SliceFile{files} } diff --git a/util.go b/util.go index a294190..e727e7a 100644 --- a/util.go +++ b/util.go @@ -1,12 +1,5 @@ package files -import ( - "bytes" - "io" - "io/ioutil" - "sort" -) - // ToFile is an alias for n.(File). If the file isn't a regular file, nil value // will be returned func ToFile(n Node) File { @@ -21,60 +14,12 @@ func ToDir(n Node) Directory { return d } -// FileFrom is a convenience function which tries to extract or create new file -// from provided value. If a passed value can't be turned into a File, nil will -// be returned. -// -// Supported types: -// * files.File (cast from Node) -// * DirEntry / DirIterator (cast from e.Node()) -// * []byte (wrapped into NewReaderFile) -// * io.Reader / io.ReadCloser (wrapped into NewReaderFile) -func FileFrom(n interface{}) File { - switch f := n.(type) { - case File: - return f - case DirEntry: - return ToFile(f.Node()) - case []byte: - return NewReaderFile(ioutil.NopCloser(bytes.NewReader(f)), nil) - case io.ReadCloser: - return NewReaderFile(f, nil) - case io.Reader: - return NewReaderFile(ioutil.NopCloser(f), nil) - default: - return nil - } +// FileFromEntry calls ToFile on Node in the given entry +func FileFromEntry(e DirEntry) File { + return ToFile(e.Node()) } -// DirFrom is a convenience function which tries to extract or create new -// directory from the provided value. If a passed value can't be turned into a -// Directory, nil will be returned. -// -// Supported types: -// * files.File (cast from Node) -// * DirEntry (cast from e.Node()) -// * DirIterator (current file, cast from e.Node()) -// * []DirEntry (wrapped into NewSliceFile) -// * map[string]Node (wrapped into NewSliceFile) -func DirFrom(n interface{}) Directory { - switch f := n.(type) { - case Directory: - return f - case DirEntry: - return ToDir(f.Node()) - case []DirEntry: - return NewSliceFile(f) - case map[string]Node: - ents := make([]DirEntry, 0, len(f)) - for name, nd := range f { - ents = append(ents, FileEntry(name, nd)) - } - sort.Slice(ents, func(i, j int) bool { - return ents[i].Name() < ents[j].Name() - }) - return NewSliceFile(ents) - default: - return nil - } +// DirFromEntry calls ToDir on Node in the given entry +func DirFromEntry(e DirEntry) Directory { + return ToDir(e.Node()) } From dfb493173af8f8b4fe7bd135b141ee350bfd12e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 12 Dec 2018 13:42:15 +0100 Subject: [PATCH 21/25] Improve Directory docs --- file.go | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/file.go b/file.go index 15cf108..d575345 100644 --- a/file.go +++ b/file.go @@ -78,17 +78,13 @@ type Directory interface { // } // // Note: - // - Some implementations of this functions may define some constraints in how - // it can be used // - Each implementation MUST support: // - Pre-order sequential iteration: - // - Meaning that after calling `Next` you can call `Next` if the returned + // - After calling `Next` you can call `Next` if the returned // node is a directory or read the returned file // - Skipping entries: - // - Meaning that if `Next` returns a directory, you can skip reading it's - // entries and skip to next entry. Files don't have to be read in full. - // Note that you can't go back to unread entries, this only allows - // skipping parts of a directory tree + // - You can skip a file/directory by calling Next without reading it + // - You can't use skipped files/directories // - This is to allow listing files in a directory without having to read // the entire tree // - Entries may not be sorted From a6bcde3c953235a2454a895be4519b5566cafb81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 12 Dec 2018 13:48:33 +0100 Subject: [PATCH 22/25] More type-safety for multipartFile --- file_test.go | 16 +++++----- multifilereader_test.go | 6 ++-- multipartfile.go | 65 ++++++++++++++++++----------------------- 3 files changed, 40 insertions(+), 47 deletions(-) diff --git a/file_test.go b/file_test.go index af60761..819238f 100644 --- a/file_test.go +++ b/file_test.go @@ -95,9 +95,9 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpname, mpf, err := newFileFromPart("", part, mpReader) + mpname, mpf, err := newFileFromPart("", part, &peekReader{r: mpReader}) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } mf, ok := mpf.(File) if !ok { @@ -118,9 +118,9 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpname, mpf, err = newFileFromPart("", part, mpReader) + mpname, mpf, err = newFileFromPart("", part, &peekReader{r: mpReader}) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } md, ok := mpf.(Directory) if !ok { @@ -138,9 +138,9 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpname, mpf, err = newFileFromPart("dir/", part, mpReader) + mpname, mpf, err = newFileFromPart("dir/", part, &peekReader{r: mpReader}) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } mf, ok = mpf.(File) if !ok { @@ -161,9 +161,9 @@ anotherfile if part == nil || err != nil { t.Fatal("Expected non-nil part, nil error") } - mpname, mpf, err = newFileFromPart("dir/", part, mpReader) + mpname, mpf, err = newFileFromPart("dir/", part, &peekReader{r: mpReader}) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } ms, ok := mpf.(*Symlink) if !ok { diff --git a/multifilereader_test.go b/multifilereader_test.go index 3357b23..fb4f749 100644 --- a/multifilereader_test.go +++ b/multifilereader_test.go @@ -114,7 +114,7 @@ func TestOutput(t *testing.T) { } mpname, mpf, err := newFileFromPart("", part, mpReader) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } mpr, ok := mpf.(File) if !ok { @@ -136,7 +136,7 @@ func TestOutput(t *testing.T) { } mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } mpd, ok := mpf.(Directory) if !ok { @@ -187,7 +187,7 @@ func TestOutput(t *testing.T) { } mpname, mpf, err = newFileFromPart("", part, mpReader) if mpf == nil || err != nil { - t.Fatal("Expected non-nil MultipartFile, nil error") + t.Fatal("Expected non-nil multipartFile, nil error") } if mpname != "file.txt" { t.Fatal("Expected filename to be \"b.txt\"") diff --git a/multipartfile.go b/multipartfile.go index c11cc82..b083e70 100644 --- a/multipartfile.go +++ b/multipartfile.go @@ -25,13 +25,13 @@ const ( var ErrPartOutsideParent = errors.New("file outside parent dir") var ErrPartInChildTree = errors.New("file in child tree") -// MultipartFile implements Node, and is created from a `multipart.Part`. -type MultipartFile struct { +// multipartFile implements Node, and is created from a `multipart.Part`. +type multipartFile struct { Node - Part *multipart.Part - Reader PartReader - Mediatype string + part *multipart.Part + reader *peekReader + mediatype string } func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Directory, error) { @@ -39,18 +39,18 @@ func NewFileFromPartReader(reader *multipart.Reader, mediatype string) (Director return nil, ErrNotDirectory } - f := &MultipartFile{ - Reader: &peekReader{r: reader}, - Mediatype: mediatype, + f := &multipartFile{ + reader: &peekReader{r: reader}, + mediatype: mediatype, } return f, nil } -func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (string, Node, error) { - f := &MultipartFile{ - Part: part, - Reader: reader, +func newFileFromPart(parent string, part *multipart.Part, reader *peekReader) (string, Node, error) { + f := &multipartFile{ + part: part, + reader: reader, } dir, base := path.Split(f.fileName()) @@ -89,12 +89,12 @@ func newFileFromPart(parent string, part *multipart.Part, reader PartReader) (st } var err error - f.Mediatype, _, err = mime.ParseMediaType(contentType) + f.mediatype, _, err = mime.ParseMediaType(contentType) if err != nil { return "", nil, err } - if !isDirectory(f.Mediatype) { + if !isDirectory(f.mediatype) { return base, &ReaderFile{ reader: part, abspath: part.Header.Get("abspath"), @@ -109,7 +109,7 @@ func isDirectory(mediatype string) bool { } type multipartIterator struct { - f *MultipartFile + f *multipartFile curFile Node curName string @@ -125,13 +125,13 @@ func (it *multipartIterator) Node() Node { } func (it *multipartIterator) Next() bool { - if it.f.Reader == nil { + if it.f.reader == nil { return false } var part *multipart.Part for { var err error - part, err = it.f.Reader.NextPart() + part, err = it.f.reader.NextPart() if err != nil { if err == io.EOF { return false @@ -140,7 +140,7 @@ func (it *multipartIterator) Next() bool { return false } - name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.Reader) + name, cf, err := newFileFromPart(it.f.fileName(), part, it.f.reader) if err == ErrPartOutsideParent { break } @@ -152,14 +152,7 @@ func (it *multipartIterator) Next() bool { } } - // we read too much, try to fix this - pr, ok := it.f.Reader.(*peekReader) - if !ok { - it.err = errors.New("cannot undo NextPart") - return false - } - - it.err = pr.put(part) + it.err = it.f.reader.put(part) return false } @@ -167,31 +160,31 @@ func (it *multipartIterator) Err() error { return it.err } -func (f *MultipartFile) Entries() DirIterator { +func (f *multipartFile) Entries() DirIterator { return &multipartIterator{f: f} } -func (f *MultipartFile) fileName() string { - if f == nil || f.Part == nil { +func (f *multipartFile) fileName() string { + if f == nil || f.part == nil { return "" } - filename, err := url.QueryUnescape(f.Part.FileName()) + filename, err := url.QueryUnescape(f.part.FileName()) if err != nil { // if there is a unescape error, just treat the name as unescaped - return f.Part.FileName() + return f.part.FileName() } return filename } -func (f *MultipartFile) Close() error { - if f.Part != nil { - return f.Part.Close() +func (f *multipartFile) Close() error { + if f.part != nil { + return f.part.Close() } return nil } -func (f *MultipartFile) Size() (int64, error) { +func (f *multipartFile) Size() (int64, error) { return 0, ErrNotSupported } @@ -230,4 +223,4 @@ func (pr *peekReader) put(p *multipart.Part) error { return nil } -var _ Directory = &MultipartFile{} +var _ Directory = &multipartFile{} From 7fea77e5fe64e777f6a5cd4c42bc585383b64bbb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Wed, 12 Dec 2018 13:50:25 +0100 Subject: [PATCH 23/25] gx publish 2.0.2 --- .gx/lastpubver | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gx/lastpubver b/.gx/lastpubver index 91a48a0..e476127 100644 --- a/.gx/lastpubver +++ b/.gx/lastpubver @@ -1 +1 @@ -2.0.1: QmPhx9B9cuaXc4vuw62567BF5NxfpsdD1AVE9HbTn7t1Y6 +2.0.2: QmNW1afVkASLRbEaDKCSM7WmFUhnF5VbsguM6kSrh3rvU8 diff --git a/package.json b/package.json index a185bf0..d2984f9 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,6 @@ "license": "", "name": "go-ipfs-files", "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", - "version": "2.0.1" + "version": "2.0.2" } From 58b4b6f71acc268144e439128d2d0dd281e1b641 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Dec 2018 20:35:11 +0100 Subject: [PATCH 24/25] Reword note on Directory.Entries --- file.go | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/file.go b/file.go index d575345..4d7ef11 100644 --- a/file.go +++ b/file.go @@ -77,17 +77,8 @@ type Directory interface { // return err // } // - // Note: - // - Each implementation MUST support: - // - Pre-order sequential iteration: - // - After calling `Next` you can call `Next` if the returned - // node is a directory or read the returned file - // - Skipping entries: - // - You can skip a file/directory by calling Next without reading it - // - You can't use skipped files/directories - // - This is to allow listing files in a directory without having to read - // the entire tree - // - Entries may not be sorted + // Note that you can't store the result of it.Node() and use it after + // advancing the iterator Entries() DirIterator } From f040c2545298197ef764d7f774f779583a12ed15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Thu, 13 Dec 2018 20:35:44 +0100 Subject: [PATCH 25/25] gx publish 2.0.3 --- .gx/lastpubver | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gx/lastpubver b/.gx/lastpubver index e476127..cac7987 100644 --- a/.gx/lastpubver +++ b/.gx/lastpubver @@ -1 +1 @@ -2.0.2: QmNW1afVkASLRbEaDKCSM7WmFUhnF5VbsguM6kSrh3rvU8 +2.0.3: QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX diff --git a/package.json b/package.json index d2984f9..d7155e6 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,6 @@ "license": "", "name": "go-ipfs-files", "releaseCmd": "git commit -a -m \"gx publish $VERSION\"", - "version": "2.0.2" + "version": "2.0.3" }