Revert "backward compatibility of .lock" and Backward compatibility b…

…y Giulio (#10077)

Reverts #10006 and add a proper migration routine

erigon-lib/common/dir/rw_dir.go

@@ -72,16 +72,15 @@ func WriteFileWithFsync(name string, data []byte, perm os.FileMode) error {
 		return err
 	}
 	defer f.Close()
-	if _, err = f.Write(data); err != nil {
-		return err
-	}
-	if err = f.Sync(); err != nil {
+	_, err = f.Write(data)
+	if err != nil {
 		return err
 	}
-	if err = f.Close(); err != nil {
+	err = f.Sync()
+	if err != nil {
 		return err
 	}
-	return nil
+	return err
 }
 
 func Recreate(dir string) {

erigon-lib/direct/downloader_client.go

@@ -36,8 +36,8 @@ func (c *DownloaderClient) Add(ctx context.Context, in *proto_downloader.AddRequ
 	return c.server.Add(ctx, in)
 }
 
-func (c *DownloaderClient) Prohibit(ctx context.Context, in *proto_downloader.ProhibitRequest, opts ...grpc.CallOption) (*proto_downloader.ProhibitReply, error) {
-	return c.server.Prohibit(ctx, in)
+func (c *DownloaderClient) ProhibitNewDownloads(ctx context.Context, in *proto_downloader.ProhibitNewDownloadsRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
+	return c.server.ProhibitNewDownloads(ctx, in)
 }
 func (c *DownloaderClient) Delete(ctx context.Context, in *proto_downloader.DeleteRequest, opts ...grpc.CallOption) (*emptypb.Empty, error) {
 	return c.server.Delete(ctx, in)

erigon-lib/downloader/downloader_grpc_server.go

@@ -45,9 +45,8 @@ type GrpcServer struct {
 	d *Downloader
 }
 
-func (s *GrpcServer) Prohibit(ctx context.Context, req *proto_downloader.ProhibitRequest) (*proto_downloader.ProhibitReply, error) {
-	whitelist, err := s.d.torrentFS.ProhibitNewDownloads(req.WhitelistAdd, req.WhitelistRemove)
-	return &proto_downloader.ProhibitReply{Whitelist: whitelist}, err
+func (s *GrpcServer) ProhibitNewDownloads(ctx context.Context, req *proto_downloader.ProhibitNewDownloadsRequest) (*emptypb.Empty, error) {
+	return &emptypb.Empty{}, s.d.torrentFS.ProhibitNewDownloads(req.Type)
 }
 
 // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast)

erigon-lib/downloader/torrent_files.go

@@ -3,6 +3,7 @@ package downloader
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"slices"
@@ -168,90 +169,79 @@ const ProhibitNewDownloadsFileName = "prohibit_new_downloads.lock"
 
 // Erigon "download once" - means restart/upgrade/downgrade will not download files (and will be fast)
 // After "download once" - Erigon will produce and seed new files
-// After `Prohibit` call - downloader stil will able:
-//   - seed new (generated by Erigon) files
-//   - seed existing on Disk files
-//   - download uncomplete parts of existing on Disk files (if Verify found some bad parts)
-//
-// `Prohibit` has `whitelist` feature - based on file-type
-func (tf *AtomicTorrentFS) ProhibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) {
+// Downloader will able: seed new files (already existing on FS), download uncomplete parts of existing files (if Verify found some bad parts)
+func (tf *AtomicTorrentFS) ProhibitNewDownloads(t string) error {
 	tf.lock.Lock()
 	defer tf.lock.Unlock()
-	return tf.prohibitNewDownloads(whitelistAdd, whitelistRemove)
+	return tf.prohibitNewDownloads(t)
 }
 
-func (tf *AtomicTorrentFS) prohibitNewDownloads(whitelistAdd, whitelistRemove []string) (whitelist []string, err error) {
-	fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName)
-	exist := dir.FileExist(fPath)
-
-	var _currentWhiteList []string
-	if exist {
-		torrentListJsonBytes, err := os.ReadFile(fPath)
-		if err != nil {
-			return nil, fmt.Errorf("read file: %w", err)
-		}
-		if len(torrentListJsonBytes) > 0 {
-			if err := json.Unmarshal(torrentListJsonBytes, &_currentWhiteList); err != nil {
-				return nil, fmt.Errorf("unmarshal: %w", err)
-			}
-		}
+func (tf *AtomicTorrentFS) prohibitNewDownloads(t string) error {
+	// open or create file ProhibitNewDownloadsFileName
+	f, err := os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_CREATE|os.O_RDONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("open file: %w", err)
 	}
-
-	whiteList := make([]string, 0, len(_currentWhiteList))
-	// copy all item except delted
-	for _, it := range _currentWhiteList {
-		if slices.Contains(whitelistRemove, it) {
-			continue
-		}
-		whiteList = append(whiteList, it)
+	defer f.Close()
+	var prohibitedList []string
+	torrentListJsonBytes, err := io.ReadAll(f)
+	if err != nil {
+		return fmt.Errorf("read file: %w", err)
 	}
-
-	// add all new whitelisted items
-	for _, it := range whitelistAdd {
-		if !slices.Contains(whiteList, it) {
-			whiteList = append(whiteList, it)
+	if len(torrentListJsonBytes) > 0 {
+		if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil {
+			return fmt.Errorf("unmarshal: %w", err)
 		}
 	}
-	slices.Sort(whiteList)
+	if slices.Contains(prohibitedList, t) {
+		return nil
+	}
+	prohibitedList = append(prohibitedList, t)
+	f.Close()
 
-	whiteListBytes, err := json.Marshal(whiteList)
+	// write new prohibited list by opening the file in truncate mode
+	f, err = os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_TRUNC|os.O_WRONLY, 0644)
 	if err != nil {
-		return _currentWhiteList, fmt.Errorf("marshal: %w", err)
+		return fmt.Errorf("open file for writing: %w", err)
 	}
-	if err := dir.WriteFileWithFsync(fPath, whiteListBytes, 0644); err != nil {
-		return _currentWhiteList, fmt.Errorf("write: %w", err)
+	defer f.Close()
+	prohibitedListJsonBytes, err := json.Marshal(prohibitedList)
+	if err != nil {
+		return fmt.Errorf("marshal: %w", err)
+	}
+	if _, err := f.Write(prohibitedListJsonBytes); err != nil {
+		return fmt.Errorf("write: %w", err)
 	}
-	return whiteList, nil
+
+	return f.Sync()
 }
 
-func (tf *AtomicTorrentFS) NewDownloadsAreProhibited(name string) (prohibited bool, err error) {
+func (tf *AtomicTorrentFS) NewDownloadsAreProhibited(name string) (bool, error) {
 	tf.lock.Lock()
 	defer tf.lock.Unlock()
 	return tf.newDownloadsAreProhibited(name)
 }
 
-func (tf *AtomicTorrentFS) newDownloadsAreProhibited(name string) (prohibited bool, err error) {
-	fPath := filepath.Join(tf.dir, ProhibitNewDownloadsFileName)
-	exists := dir.FileExist(fPath)
-	if !exists { // no .lock - means all allowed
-		return false, nil
+func (tf *AtomicTorrentFS) newDownloadsAreProhibited(name string) (bool, error) {
+	f, err := os.OpenFile(filepath.Join(tf.dir, ProhibitNewDownloadsFileName), os.O_CREATE|os.O_RDONLY, 0644)
+	if err != nil {
+		return false, err
 	}
-
-	var whiteList []string
-	whiteListBytes, err := os.ReadFile(fPath)
+	defer f.Close()
+	var prohibitedList []string
+	torrentListJsonBytes, err := io.ReadAll(f)
 	if err != nil {
 		return false, fmt.Errorf("NewDownloadsAreProhibited: read file: %w", err)
 	}
-	if len(whiteListBytes) > 0 {
-		if err := json.Unmarshal(whiteListBytes, &whiteList); err != nil {
+	if len(torrentListJsonBytes) > 0 {
+		if err := json.Unmarshal(torrentListJsonBytes, &prohibitedList); err != nil {
 			return false, fmt.Errorf("NewDownloadsAreProhibited: unmarshal: %w", err)
 		}
 	}
-
-	for _, whiteListedItem := range whiteList {
-		if strings.Contains(name, whiteListedItem) {
-			return false, nil
+	for _, p := range prohibitedList {
+		if strings.Contains(name, p) {
+			return true, nil
 		}
 	}
-	return true, nil
+	return false, nil
 }

erigon-lib/go.mod

@@ -5,7 +5,7 @@ go 1.21
 require (
 	github.com/erigontech/mdbx-go v0.27.24
 	github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901
-	github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08
+	github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087
 	github.com/ledgerwatch/log/v3 v3.9.0
 	github.com/ledgerwatch/secp256k1 v1.0.0
 )

erigon-lib/go.sum

@@ -269,8 +269,8 @@ github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7
 github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8=
 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901 h1:gAcI47OHnt/1e/APIV0093NVdviIfAnBUzFyybmKL1Q=
 github.com/ledgerwatch/erigon-snapshot v1.3.1-0.20240417163500-185a51876901/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo=
-github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08 h1:NQRyMIGIapAFnr7hAY0xXQZPMBjtYCUAQ0UF1/saBaE=
-github.com/ledgerwatch/interfaces v0.0.0-20240425034152-dda221776f08/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
+github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087 h1:Y59HUAT/+02Qbm6g7MuY7i8E0kUihPe7+ftDnR8oQzQ=
+github.com/ledgerwatch/interfaces v0.0.0-20240320062914-b57f05746087/go.mod h1:ugQv1QllJzBny3cKZKxUrSnykkjkBgm27eQM6dnGAcc=
 github.com/ledgerwatch/log/v3 v3.9.0 h1:iDwrXe0PVwBC68Dd94YSsHbMgQ3ufsgjzXtFNFVZFRk=
 github.com/ledgerwatch/log/v3 v3.9.0/go.mod h1:EiAY6upmI/6LkNhOVxb4eVsmsP11HZCnZ3PlJMjYiqE=
 github.com/ledgerwatch/secp256k1 v1.0.0 h1:Usvz87YoTG0uePIV8woOof5cQnLXGYa162rFf3YnwaQ=