log file unmap/close errors (ledgerwatch#1036)

common/dbg/leak_detector.go

@@ -10,6 +10,8 @@ import (
 	"github.com/ledgerwatch/log/v3"
 )
 
+const FileCloseLogLevel = log.LvlTrace
+
 // LeakDetector - use it to find which resource was created but not closed (leaked)
 // periodically does print in logs resources which living longer than 1min with their creation stack trace
 // For example db transactions can call Add/Del from Begin/Commit/Rollback methods

compress/decompress.go

@@ -341,14 +341,16 @@ func (d *Decompressor) ModTime() time.Time {
 	return d.modTime
 }
 
-func (d *Decompressor) Close() error {
-	if err := mmap.Munmap(d.mmapHandle1, d.mmapHandle2); err != nil {
-		log.Trace("unmap", "err", err, "file", d.FileName())
-	}
-	if err := d.f.Close(); err != nil {
-		return err
+func (d *Decompressor) Close() {
+	if d.f != nil {
+		if err := mmap.Munmap(d.mmapHandle1, d.mmapHandle2); err != nil {
+			log.Log(dbg.FileCloseLogLevel, "unmap", "err", err, "file", d.FileName(), "stack", dbg.Stack())
+		}
+		if err := d.f.Close(); err != nil {
+			log.Log(dbg.FileCloseLogLevel, "close", "err", err, "file", d.FileName(), "stack", dbg.Stack())
+		}
+		d.f = nil
 	}
-	return nil
 }
 
 func (d *Decompressor) FilePath() string { return d.filePath }

kv/bitmapdb/fixed_size.go

@@ -80,18 +80,19 @@ func OpenFixedSizeBitmaps(filePath string, bitsPerBitmap int) (*FixedSizeBitmaps
 
 func (bm *FixedSizeBitmaps) FileName() string { return bm.fileName }
 func (bm *FixedSizeBitmaps) FilePath() string { return bm.filePath }
-func (bm *FixedSizeBitmaps) Close() error {
+func (bm *FixedSizeBitmaps) Close() {
 	if bm.m != nil {
 		if err := bm.m.Unmap(); err != nil {
 			log.Trace("unmap", "err", err, "file", bm.FileName())
 		}
+		bm.m = nil
 	}
 	if bm.f != nil {
 		if err := bm.f.Close(); err != nil {
-			return err
+			log.Trace("close", "err", err, "file", bm.FileName())
 		}
+		bm.f = nil
 	}
-	return nil
 }
 
 func (bm *FixedSizeBitmaps) At(item uint64) (res []uint64, err error) {
@@ -219,8 +220,18 @@ func NewFixedSizeBitmapsWriter(indexFile string, bitsPerBitmap int, amount uint6
 	return idx, nil
 }
 func (w *FixedSizeBitmapsWriter) Close() {
-	_ = w.m.Unmap()
-	_ = w.f.Close()
+	if w.m != nil {
+		if err := w.m.Unmap(); err != nil {
+			log.Trace("unmap", "err", err, "file", w.f.Name())
+		}
+		w.m = nil
+	}
+	if w.f != nil {
+		if err := w.f.Close(); err != nil {
+			log.Trace("close", "err", err, "file", w.f.Name())
+		}
+		w.f = nil
+	}
 }
 func growFileToSize(f *os.File, size int) error {
 	pageSize := os.Getpagesize()

recsplit/index.go

@@ -28,6 +28,7 @@ import (
 	"time"
 	"unsafe"
 
+	"github.com/ledgerwatch/erigon-lib/common/dbg"
 	"github.com/ledgerwatch/log/v3"
 
 	"github.com/ledgerwatch/erigon-lib/common"
@@ -174,17 +175,19 @@ func (idx *Index) BaseDataID() uint64 { return idx.baseDataID }
 func (idx *Index) FilePath() string   { return idx.filePath }
 func (idx *Index) FileName() string   { return idx.fileName }
 
-func (idx *Index) Close() error {
+func (idx *Index) Close() {
 	if idx == nil {
-		return nil
-	}
-	if err := mmap.Munmap(idx.mmapHandle1, idx.mmapHandle2); err != nil {
-		log.Trace("unmap", "err", err, "file", idx.FileName())
+		return
 	}
-	if err := idx.f.Close(); err != nil {
-		return err
+	if idx.f != nil {
+		if err := mmap.Munmap(idx.mmapHandle1, idx.mmapHandle2); err != nil {
+			log.Log(dbg.FileCloseLogLevel, "unmap", "err", err, "file", idx.FileName(), "stack", dbg.Stack())
+		}
+		if err := idx.f.Close(); err != nil {
+			log.Log(dbg.FileCloseLogLevel, "close", "err", err, "file", idx.FileName(), "stack", dbg.Stack())
+		}
+		idx.f = nil
 	}
-	return nil
 }
 
 func (idx *Index) skipBits(m uint16) int {

state/btree_index.go

@@ -16,6 +16,7 @@ import (
 
 	"github.com/c2h5oh/datasize"
 	"github.com/edsrzf/mmap-go"
+	"github.com/ledgerwatch/erigon-lib/common/dbg"
 	"github.com/ledgerwatch/log/v3"
 
 	"github.com/ledgerwatch/erigon-lib/common/background"
@@ -1089,10 +1090,11 @@ func (b *BtIndex) Close() {
 	}
 	if b.file != nil {
 		if err := b.m.Unmap(); err != nil {
-			_ = err
+			log.Log(dbg.FileCloseLogLevel, "unmap", "err", err, "file", b.FileName(), "stack", dbg.Stack())
 		}
+		b.m = nil
 		if err := b.file.Close(); err != nil {
-			_ = err
+			log.Log(dbg.FileCloseLogLevel, "close", "err", err, "file", b.FileName(), "stack", dbg.Stack())
 		}
 		b.file = nil
 	}

state/domain.go

@@ -84,9 +84,7 @@ func filesItemLess(i, j *filesItem) bool {
 }
 func (i *filesItem) closeFilesAndRemove() {
 	if i.decompressor != nil {
-		if err := i.decompressor.Close(); err != nil {
-			log.Trace("close", "err", err, "file", i.decompressor.FileName())
-		}
+		i.decompressor.Close()
 		// paranoic-mode on: don't delete frozen files
 		if !i.frozen {
 			if err := os.Remove(i.decompressor.FilePath()); err != nil {
@@ -96,9 +94,7 @@ func (i *filesItem) closeFilesAndRemove() {
 		i.decompressor = nil
 	}
 	if i.index != nil {
-		if err := i.index.Close(); err != nil {
-			log.Trace("close", "err", err, "file", i.index.FileName())
-		}
+		i.index.Close()
 		// paranoic-mode on: don't delete frozen files
 		if !i.frozen {
 			if err := os.Remove(i.index.FilePath()); err != nil {
@@ -380,15 +376,11 @@ func (d *Domain) closeWhatNotInList(fNames []string) {
 	})
 	for _, item := range toDelete {
 		if item.decompressor != nil {
-			if err := item.decompressor.Close(); err != nil {
-				d.logger.Trace("close", "err", err, "file", item.decompressor.FileName())
-			}
+			item.decompressor.Close()
 			item.decompressor = nil
 		}
 		if item.index != nil {
-			if err := item.index.Close(); err != nil {
-				d.logger.Trace("close", "err", err, "file", item.index.FileName())
-			}
+			item.index.Close()
 			item.index = nil
 		}
 		if item.bindex != nil {

state/history.go

@@ -262,15 +262,11 @@ func (h *History) closeWhatNotInList(fNames []string) {
 	})
 	for _, item := range toDelete {
 		if item.decompressor != nil {
-			if err := item.decompressor.Close(); err != nil {
-				h.logger.Trace("close", "err", err, "file", item.index.FileName())
-			}
+			item.decompressor.Close()
 			item.decompressor = nil
 		}
 		if item.index != nil {
-			if err := item.index.Close(); err != nil {
-				h.logger.Trace("close", "err", err, "file", item.index.FileName())
-			}
+			item.index.Close()
 			item.index = nil
 		}
 		h.files.Delete(item)

state/inverted_index.go

@@ -354,15 +354,11 @@ func (ii *InvertedIndex) closeWhatNotInList(fNames []string) {
 	})
 	for _, item := range toDelete {
 		if item.decompressor != nil {
-			if err := item.decompressor.Close(); err != nil {
-				ii.logger.Trace("close", "err", err, "file", item.index.FileName())
-			}
+			item.decompressor.Close()
 			item.decompressor = nil
 		}
 		if item.index != nil {
-			if err := item.index.Close(); err != nil {
-				ii.logger.Trace("close", "err", err, "file", item.index.FileName())
-			}
+			item.index.Close()
 			item.index = nil
 		}
 		ii.files.Delete(item)

state/locality_index.go

@@ -228,9 +228,7 @@ func closeLocalityIndexFilesAndRemove(i *ctxLocalityIdx, logger log.Logger) {
 		i.file.src = nil
 	}
 	if i.bm != nil {
-		if err := i.bm.Close(); err != nil {
-			logger.Trace("close", "err", err, "file", i.bm.FileName())
-		}
+		i.bm.Close()
 		if err := os.Remove(i.bm.FilePath()); err != nil {
 			logger.Trace("os.Remove", "err", err, "file", i.bm.FileName())
 		}