From 2b17a0ce5105ce910dfcc7a0b41e702f94ec1176 Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Fri, 7 Jun 2024 16:01:56 -0700 Subject: [PATCH 1/4] flush butter before writing to file --- .../buffered_file_writer/bufferedfilewriter.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/pkg/writers/buffered_file_writer/bufferedfilewriter.go b/pkg/writers/buffered_file_writer/bufferedfilewriter.go index 63c6733bad1f..57792e6446fd 100644 --- a/pkg/writers/buffered_file_writer/bufferedfilewriter.go +++ b/pkg/writers/buffered_file_writer/bufferedfilewriter.go @@ -209,6 +209,14 @@ func (w *BufferedFileWriter) Write(data []byte) (int, error) { } } + // Write any remaining data in the buffer to the file before writing new data. + if w.buf.Len() > 0 { + if _, err := w.buf.WriteTo(w.file); err != nil { + return 0, fmt.Errorf("error flushing buffer to file: %w", err) + } + w.buf.Reset() + } + n, err := w.file.Write(data) if err != nil { return n, err @@ -273,11 +281,11 @@ func (w *BufferedFileWriter) CloseForWriting() error { defer w.bufPool.Put(w.buf) if w.buf.Len() > 0 { - _, err := w.buf.WriteTo(w.file) - if err != nil { + if _, err := w.buf.WriteTo(w.file); err != nil { return err } } + return w.file.Close() } @@ -304,6 +312,7 @@ func (w *BufferedFileWriter) ReadSeekCloser() (io.ReadSeekCloser, error) { if err != nil { return nil, err } + return newAutoDeletingFileReader(file), nil } From 3110b2abc3fb78a059d07fb070a729ad1711477b Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Fri, 7 Jun 2024 20:28:30 -0700 Subject: [PATCH 2/4] revert --- pkg/writers/buffered_file_writer/bufferedfilewriter.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/writers/buffered_file_writer/bufferedfilewriter.go b/pkg/writers/buffered_file_writer/bufferedfilewriter.go index 57792e6446fd..17bdd0456731 100644 --- a/pkg/writers/buffered_file_writer/bufferedfilewriter.go +++ b/pkg/writers/buffered_file_writer/bufferedfilewriter.go @@ -285,7 +285,6 @@ func (w *BufferedFileWriter) CloseForWriting() error { return err } } - return w.file.Close() } @@ -312,7 +311,6 @@ func (w *BufferedFileWriter) ReadSeekCloser() (io.ReadSeekCloser, error) { if err != nil { return nil, err } - return newAutoDeletingFileReader(file), nil } From c044ce7ebeec3f7e1e86d1e973f8049b23cfca1b Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Fri, 7 Jun 2024 20:30:10 -0700 Subject: [PATCH 3/4] remove redundant Reset --- pkg/writers/buffered_file_writer/bufferedfilewriter.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/pkg/writers/buffered_file_writer/bufferedfilewriter.go b/pkg/writers/buffered_file_writer/bufferedfilewriter.go index 17bdd0456731..8fa93fe16ccb 100644 --- a/pkg/writers/buffered_file_writer/bufferedfilewriter.go +++ b/pkg/writers/buffered_file_writer/bufferedfilewriter.go @@ -205,7 +205,6 @@ func (w *BufferedFileWriter) Write(data []byte) (int, error) { } return 0, err } - w.buf.Reset() } } @@ -214,7 +213,6 @@ func (w *BufferedFileWriter) Write(data []byte) (int, error) { if _, err := w.buf.WriteTo(w.file); err != nil { return 0, fmt.Errorf("error flushing buffer to file: %w", err) } - w.buf.Reset() } n, err := w.file.Write(data) From 0e8b58b0ef5614ccbdfe005be354f2d8b60c9956 Mon Sep 17 00:00:00 2001 From: Ahrav Dutta Date: Sat, 8 Jun 2024 11:06:10 -0700 Subject: [PATCH 4/4] add test case --- .../bufferedfilewriter_test.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go b/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go index 310b8086c855..f250f9420e96 100644 --- a/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go +++ b/pkg/writers/buffered_file_writer/bufferedfilewriter_test.go @@ -650,11 +650,19 @@ func TestBufferedFileWriter_ReadFrom(t *testing.T) { expectedSize: 1 << 20, }, { - name: "Input greater than threshold", + name: "Input slightly greater than threshold", input: string(make([]byte, defaultThreshold+1)), expectedOutput: string(make([]byte, defaultThreshold+1)), expectedSize: defaultThreshold + 1, }, + // Test to ensure that anytime the buffer exceeds the threshold, the data is written to a file + // and the buffer is cleared. + { + name: "Input much greater than threshold", + input: string(make([]byte, (2*defaultThreshold)+largeBufferSize+1)), + expectedOutput: string(make([]byte, (2*defaultThreshold)+largeBufferSize+1)), + expectedSize: (2 * defaultThreshold) + largeBufferSize + 1, + }, } for _, tc := range tests { @@ -668,6 +676,10 @@ func TestBufferedFileWriter_ReadFrom(t *testing.T) { size, err := writer.ReadFrom(reader) assert.NoError(t, err) + if writer.buf != nil && writer.file != nil { + assert.Len(t, writer.buf.Bytes(), 0) + } + err = writer.CloseForWriting() assert.NoError(t, err)