From d1e56a3269470b4a6ec9f14e7518ab9dec577562 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 25 Feb 2018 14:56:29 -0300 Subject: [PATCH 1/7] fix: deb: create subfolders and use GNU tar format --- deb/deb.go | 72 ++++++++++++++++++++++++++++++++++++++----------- deb/deb_test.go | 8 ++++++ 2 files changed, 64 insertions(+), 16 deletions(-) diff --git a/deb/deb.go b/deb/deb.go index 85827661..c743e139 100644 --- a/deb/deb.go +++ b/deb/deb.go @@ -5,6 +5,7 @@ import ( "archive/tar" "bytes" "compress/gzip" + "path/filepath" // #nosec "crypto/md5" "fmt" @@ -31,12 +32,11 @@ type Deb struct{} // Package writes a new deb package to the given writer using the given info func (*Deb) Package(info nfpm.Info, deb io.Writer) (err error) { - var now = time.Now() - dataTarGz, md5sums, instSize, err := createDataTarGz(now, info) + dataTarGz, md5sums, instSize, err := createDataTarGz(info) if err != nil { return err } - controlTarGz, err := createControl(now, instSize, md5sums, info) + controlTarGz, err := createControl(instSize, md5sums, info) if err != nil { return err } @@ -44,24 +44,24 @@ func (*Deb) Package(info nfpm.Info, deb io.Writer) (err error) { if err := w.WriteGlobalHeader(); err != nil { return errors.Wrap(err, "cannot write ar header to deb file") } - if err := addArFile(now, w, "debian-binary", []byte("2.0\n")); err != nil { + if err := addArFile(w, "debian-binary", []byte("2.0\n")); err != nil { return errors.Wrap(err, "cannot pack debian-binary") } - if err := addArFile(now, w, "control.tar.gz", controlTarGz); err != nil { + if err := addArFile(w, "control.tar.gz", controlTarGz); err != nil { return errors.Wrap(err, "cannot add control.tar.gz to deb") } - if err := addArFile(now, w, "data.tar.gz", dataTarGz); err != nil { + if err := addArFile(w, "data.tar.gz", dataTarGz); err != nil { return errors.Wrap(err, "cannot add data.tar.gz to deb") } return nil } -func addArFile(now time.Time, w *ar.Writer, name string, body []byte) error { +func addArFile(w *ar.Writer, name string, body []byte) error { var header = ar.Header{ Name: name, Size: int64(len(body)), Mode: 0644, - ModTime: now, + ModTime: time.Now(), } if err := w.WriteHeader(&header); err != nil { return errors.Wrap(err, "cannot write file header") @@ -70,7 +70,7 @@ func addArFile(now time.Time, w *ar.Writer, name string, body []byte) error { return err } -func createDataTarGz(now time.Time, info nfpm.Info) (dataTarGz, md5sums []byte, instSize int64, err error) { +func createDataTarGz(info nfpm.Info) (dataTarGz, md5sums []byte, instSize int64, err error) { var buf bytes.Buffer var compress = gzip.NewWriter(&buf) var out = tar.NewWriter(compress) @@ -86,7 +86,10 @@ func createDataTarGz(now time.Time, info nfpm.Info) (dataTarGz, md5sums []byte, info.ConfigFiles, } { for src, dst := range files { - size, err := copyToTarAndDigest(out, &md5buf, now, src, dst) + if err := createTree(out, dst); err != nil { + return nil, nil, 0, err + } + size, err := copyToTarAndDigest(out, &md5buf, src, dst) if err != nil { return nil, nil, 0, err } @@ -104,7 +107,42 @@ func createDataTarGz(now time.Time, info nfpm.Info) (dataTarGz, md5sums []byte, return buf.Bytes(), md5buf.Bytes(), instSize, nil } -func copyToTarAndDigest(tarw *tar.Writer, md5w io.Writer, now time.Time, src, dst string) (int64, error) { +// this is needed because the data.tar.gz file should have the empty folders +// as well, so we walk through the dst and create all subfolders. +func createTree(tarw *tar.Writer, dst string) error { + for _, path := range pathsToCreate(dst) { + if err := tarw.WriteHeader(&tar.Header{ + Name: path + "/", + Mode: 0755, + Typeflag: tar.TypeDir, + Format: tar.FormatGNU, + }); err != nil { + return errors.Wrap(err, "failed to create folder") + } + } + return nil +} + +func pathsToCreate(dst string) []string { + var paths []string + var base = dst[1:] + for { + base = filepath.Dir(base) + if base == "." { + break + } + paths = append(paths, base) + } + // we don't really need to create those things in order apparently, but, + // it looks really weird if we dont. + var result []string + for i := len(paths) - 1; i >= 0; i-- { + result = append(result, paths[i]) + } + return result +} + +func copyToTarAndDigest(tarw *tar.Writer, md5w io.Writer, src, dst string) (int64, error) { file, err := os.OpenFile(src, os.O_RDONLY, 0600) if err != nil { return 0, errors.Wrap(err, "could not add file to the archive") @@ -122,7 +160,8 @@ func copyToTarAndDigest(tarw *tar.Writer, md5w io.Writer, now time.Time, src, ds Name: dst[1:], Size: info.Size(), Mode: int64(info.Mode()), - ModTime: now, + ModTime: time.Now(), + Format: tar.FormatGNU, } if err := tarw.WriteHeader(&header); err != nil { return 0, errors.Wrapf(err, "cannot write header of %s to data.tar.gz", header) @@ -161,7 +200,7 @@ type controlData struct { InstalledSize int64 } -func createControl(now time.Time, instSize int64, md5sums []byte, info nfpm.Info) (controlTarGz []byte, err error) { +func createControl(instSize int64, md5sums []byte, info nfpm.Info) (controlTarGz []byte, err error) { var buf bytes.Buffer var compress = gzip.NewWriter(&buf) var out = tar.NewWriter(compress) @@ -183,7 +222,7 @@ func createControl(now time.Time, instSize int64, md5sums []byte, info nfpm.Info "md5sums": md5sums, "conffiles": conffiles(info), } { - if err := newFileInsideTarGz(out, name, content, now); err != nil { + if err := newFileInsideTarGz(out, name, content); err != nil { return nil, err } } @@ -207,13 +246,14 @@ func writeControl(w io.Writer, data controlData) error { return template.Must(tmpl.Parse(controlTemplate)).Execute(w, data) } -func newFileInsideTarGz(out *tar.Writer, name string, content []byte, now time.Time) error { +func newFileInsideTarGz(out *tar.Writer, name string, content []byte) error { var header = tar.Header{ Name: name, Size: int64(len(content)), Mode: 0644, - ModTime: now, + ModTime: time.Now(), Typeflag: tar.TypeReg, + Format: tar.FormatGNU, } if err := out.WriteHeader(&header); err != nil { return errors.Wrapf(err, "cannot write header of %s file to control.tar.gz", name) diff --git a/deb/deb_test.go b/deb/deb_test.go index e93280af..6f2b6f94 100644 --- a/deb/deb_test.go +++ b/deb/deb_test.go @@ -129,3 +129,11 @@ func TestConffiles(t *testing.T) { }) assert.Equal(t, "/etc/fake\n", string(out), "should have a trailing empty line") } + +func TestPathsToCreate(t *testing.T) { + assert.Equal( + t, + []string{"usr", "usr/share", "usr/share/doc", "usr/share/doc/whatever"}, + pathsToCreate("/usr/share/doc/whatever/foo.md"), + ) +} From 4ece0503a817a6b0f56a7cbef6f168b59eabc893 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 25 Feb 2018 15:10:17 -0300 Subject: [PATCH 2/7] fix: deb: folder modtime --- deb/deb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/deb/deb.go b/deb/deb.go index c743e139..2b8a4784 100644 --- a/deb/deb.go +++ b/deb/deb.go @@ -116,6 +116,7 @@ func createTree(tarw *tar.Writer, dst string) error { Mode: 0755, Typeflag: tar.TypeDir, Format: tar.FormatGNU, + ModTime: time.Now(), }); err != nil { return errors.Wrap(err, "failed to create folder") } From 1ae64d6aa4e9a891aeb23b6a5cca941548987092 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 25 Feb 2018 15:17:55 -0300 Subject: [PATCH 3/7] fix: deb: do not create multiple entries for the same folder --- deb/deb.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/deb/deb.go b/deb/deb.go index 2b8a4784..c48e84b3 100644 --- a/deb/deb.go +++ b/deb/deb.go @@ -80,13 +80,15 @@ func createDataTarGz(info nfpm.Info) (dataTarGz, md5sums []byte, instSize int64, defer out.Close() // nolint: errcheck defer compress.Close() // nolint: errcheck + var created = map[string]bool{} + var md5buf bytes.Buffer for _, files := range []map[string]string{ info.Files, info.ConfigFiles, } { for src, dst := range files { - if err := createTree(out, dst); err != nil { + if err := createTree(out, dst, created); err != nil { return nil, nil, 0, err } size, err := copyToTarAndDigest(out, &md5buf, src, dst) @@ -109,8 +111,13 @@ func createDataTarGz(info nfpm.Info) (dataTarGz, md5sums []byte, instSize int64, // this is needed because the data.tar.gz file should have the empty folders // as well, so we walk through the dst and create all subfolders. -func createTree(tarw *tar.Writer, dst string) error { +func createTree(tarw *tar.Writer, dst string, created map[string]bool) error { for _, path := range pathsToCreate(dst) { + if created[path] { + // skipping dir that was previously created inside the archive + // (eg: usr/) + continue + } if err := tarw.WriteHeader(&tar.Header{ Name: path + "/", Mode: 0755, @@ -120,6 +127,7 @@ func createTree(tarw *tar.Writer, dst string) error { }); err != nil { return errors.Wrap(err, "failed to create folder") } + created[path] = true } return nil } @@ -155,6 +163,7 @@ func copyToTarAndDigest(tarw *tar.Writer, md5w io.Writer, src, dst string) (int6 return 0, err } if info.IsDir() { + // TODO: this should probably return an error return 0, nil } var header = tar.Header{ From a7efc0190fa814f1c8617bac1fd42f034a0248ef Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 25 Feb 2018 15:19:39 -0300 Subject: [PATCH 4/7] typo: deb --- deb/deb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deb/deb.go b/deb/deb.go index c48e84b3..522ed777 100644 --- a/deb/deb.go +++ b/deb/deb.go @@ -143,7 +143,7 @@ func pathsToCreate(dst string) []string { paths = append(paths, base) } // we don't really need to create those things in order apparently, but, - // it looks really weird if we dont. + // it looks really weird if we do. var result []string for i := len(paths) - 1; i >= 0; i-- { result = append(result, paths[i]) From 20c7fa9722cd36a4696d07eb4e7c4bb382676067 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 25 Feb 2018 17:29:24 -0300 Subject: [PATCH 5/7] style: deb: organized code --- deb/deb.go | 150 ++++++++++++++++++++++++++--------------------------- 1 file changed, 75 insertions(+), 75 deletions(-) diff --git a/deb/deb.go b/deb/deb.go index 522ed777..24fdc442 100644 --- a/deb/deb.go +++ b/deb/deb.go @@ -109,48 +109,6 @@ func createDataTarGz(info nfpm.Info) (dataTarGz, md5sums []byte, instSize int64, return buf.Bytes(), md5buf.Bytes(), instSize, nil } -// this is needed because the data.tar.gz file should have the empty folders -// as well, so we walk through the dst and create all subfolders. -func createTree(tarw *tar.Writer, dst string, created map[string]bool) error { - for _, path := range pathsToCreate(dst) { - if created[path] { - // skipping dir that was previously created inside the archive - // (eg: usr/) - continue - } - if err := tarw.WriteHeader(&tar.Header{ - Name: path + "/", - Mode: 0755, - Typeflag: tar.TypeDir, - Format: tar.FormatGNU, - ModTime: time.Now(), - }); err != nil { - return errors.Wrap(err, "failed to create folder") - } - created[path] = true - } - return nil -} - -func pathsToCreate(dst string) []string { - var paths []string - var base = dst[1:] - for { - base = filepath.Dir(base) - if base == "." { - break - } - paths = append(paths, base) - } - // we don't really need to create those things in order apparently, but, - // it looks really weird if we do. - var result []string - for i := len(paths) - 1; i >= 0; i-- { - result = append(result, paths[i]) - } - return result -} - func copyToTarAndDigest(tarw *tar.Writer, md5w io.Writer, src, dst string) (int64, error) { file, err := os.OpenFile(src, os.O_RDONLY, 0600) if err != nil { @@ -187,29 +145,6 @@ func copyToTarAndDigest(tarw *tar.Writer, md5w io.Writer, src, dst string) (int6 return info.Size(), nil } -var controlTemplate = `Package: {{.Info.Name}} -Version: {{.Info.Version}} -Section: {{.Info.Section}} -Priority: {{.Info.Priority}} -Architecture: {{.Info.Arch}} -Maintainer: {{.Info.Maintainer}} -Vendor: {{.Info.Vendor}} -Installed-Size: {{.InstalledSize}} -Replaces: {{join .Info.Replaces}} -Provides: {{join .Info.Provides}} -Depends: {{join .Info.Depends}} -Recommends: {{join .Info.Recommends}} -Suggests: {{join .Info.Suggests}} -Conflicts: {{join .Info.Conflicts}} -Homepage: {{.Info.Homepage}} -Description: {{.Info.Description}} -` - -type controlData struct { - Info nfpm.Info - InstalledSize int64 -} - func createControl(instSize int64, md5sums []byte, info nfpm.Info) (controlTarGz []byte, err error) { var buf bytes.Buffer var compress = gzip.NewWriter(&buf) @@ -246,16 +181,6 @@ func createControl(instSize int64, md5sums []byte, info nfpm.Info) (controlTarGz return buf.Bytes(), nil } -func writeControl(w io.Writer, data controlData) error { - var tmpl = template.New("control") - tmpl.Funcs(template.FuncMap{ - "join": func(strs []string) string { - return strings.Trim(strings.Join(strs, ", "), " ") - }, - }) - return template.Must(tmpl.Parse(controlTemplate)).Execute(w, data) -} - func newFileInsideTarGz(out *tar.Writer, name string, content []byte) error { var header = tar.Header{ Name: name, @@ -274,6 +199,48 @@ func newFileInsideTarGz(out *tar.Writer, name string, content []byte) error { return nil } +// this is needed because the data.tar.gz file should have the empty folders +// as well, so we walk through the dst and create all subfolders. +func createTree(tarw *tar.Writer, dst string, created map[string]bool) error { + for _, path := range pathsToCreate(dst) { + if created[path] { + // skipping dir that was previously created inside the archive + // (eg: usr/) + continue + } + if err := tarw.WriteHeader(&tar.Header{ + Name: path + "/", + Mode: 0755, + Typeflag: tar.TypeDir, + Format: tar.FormatGNU, + ModTime: time.Now(), + }); err != nil { + return errors.Wrap(err, "failed to create folder") + } + created[path] = true + } + return nil +} + +func pathsToCreate(dst string) []string { + var paths = []string{} + var base = dst[1:] + for { + base = filepath.Dir(base) + if base == "." { + break + } + paths = append(paths, base) + } + // we don't really need to create those things in order apparently, but, + // it looks really weird if we do. + var result = []string{} + for i := len(paths) - 1; i >= 0; i-- { + result = append(result, paths[i]) + } + return result +} + func conffiles(info nfpm.Info) []byte { var confs []string for _, dst := range info.ConfigFiles { @@ -281,3 +248,36 @@ func conffiles(info nfpm.Info) []byte { } return []byte(strings.Join(confs, "\n") + "\n") } + +var controlTemplate = `Package: {{.Info.Name}} +Version: {{.Info.Version}} +Section: {{.Info.Section}} +Priority: {{.Info.Priority}} +Architecture: {{.Info.Arch}} +Maintainer: {{.Info.Maintainer}} +Vendor: {{.Info.Vendor}} +Installed-Size: {{.InstalledSize}} +Replaces: {{join .Info.Replaces}} +Provides: {{join .Info.Provides}} +Depends: {{join .Info.Depends}} +Recommends: {{join .Info.Recommends}} +Suggests: {{join .Info.Suggests}} +Conflicts: {{join .Info.Conflicts}} +Homepage: {{.Info.Homepage}} +Description: {{.Info.Description}} +` + +type controlData struct { + Info nfpm.Info + InstalledSize int64 +} + +func writeControl(w io.Writer, data controlData) error { + var tmpl = template.New("control") + tmpl.Funcs(template.FuncMap{ + "join": func(strs []string) string { + return strings.Trim(strings.Join(strs, ", "), " ") + }, + }) + return template.Must(tmpl.Parse(controlTemplate)).Execute(w, data) +} From 9b70ad6db6f9a17c34b5e8e4fbfd42ac95a15099 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 25 Feb 2018 17:29:31 -0300 Subject: [PATCH 6/7] test: deb: added more tests --- deb/deb_test.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/deb/deb_test.go b/deb/deb_test.go index 6f2b6f94..c0b61e3f 100644 --- a/deb/deb_test.go +++ b/deb/deb_test.go @@ -3,6 +3,7 @@ package deb import ( "bytes" "flag" + "fmt" "io/ioutil" "testing" @@ -41,7 +42,8 @@ var info = nfpm.WithDefaults(nfpm.Info{ Homepage: "http://carlosbecker.com", Vendor: "nope", Files: map[string]string{ - "../testdata/fake": "/usr/local/bin/fake", + "../testdata/fake": "/usr/local/bin/fake", + "../testdata/whatever.conf": "/usr/share/doc/fake/fake.txt", }, ConfigFiles: map[string]string{ "../testdata/whatever.conf": "/etc/fake/fake.conf", @@ -131,9 +133,13 @@ func TestConffiles(t *testing.T) { } func TestPathsToCreate(t *testing.T) { - assert.Equal( - t, - []string{"usr", "usr/share", "usr/share/doc", "usr/share/doc/whatever"}, - pathsToCreate("/usr/share/doc/whatever/foo.md"), - ) + for path, parts := range map[string][]string{ + "/usr/share/doc/whatever/foo.md": []string{"usr", "usr/share", "usr/share/doc", "usr/share/doc/whatever"}, + "/var/moises": []string{"var"}, + "/": []string{}, + } { + t.Run(fmt.Sprintf("path: '%s'", path), func(t *testing.T) { + assert.Equal(t, parts, pathsToCreate(path)) + }) + } } From 24b75ec65c6c7d6f6baa7441f16c4de8a7188234 Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Sun, 25 Feb 2018 17:32:28 -0300 Subject: [PATCH 7/7] chore: go test -race --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 68fcd8a3..9a396548 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ TEST_OPTIONS?= setup: go get -u github.com/alecthomas/gometalinter go get -u github.com/golang/dep/cmd/dep - go get -u github.com/pierrre/gotestcover go get -u golang.org/x/tools/cmd/cover go get -u github.com/caarlos0/bandep go get -u github.com/gobuffalo/packr/... @@ -22,7 +21,7 @@ check: # Run all the tests test: - gotestcover $(TEST_OPTIONS) -covermode=atomic -coverprofile=coverage.txt $(SOURCE_FILES) -run $(TEST_PATTERN) -timeout=2m + go test $(TEST_OPTIONS) -v -coverpkg=./... -race -failfast -covermode=atomic -coverprofile=coverage.txt $(SOURCE_FILES) -run $(TEST_PATTERN) -timeout=2m .PHONY: cover # Run all the tests and opens the coverage report