From de8f783eeb2d123bd399de303de55803a6d17b41 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 12 Sep 2023 12:18:39 +0700 Subject: [PATCH 01/14] torrent: add --webseeds cli arg (#1122) --- common/cli.go | 12 ++ downloader/downloader.go | 153 +++++++++++++++++++--- downloader/downloader_grpc_server.go | 24 ++-- downloader/downloadercfg/downloadercfg.go | 41 +++++- downloader/snaptype/webseeds.go | 19 +++ downloader/util.go | 133 ++++++++++++------- go.mod | 1 + go.sum | 2 + 8 files changed, 304 insertions(+), 81 deletions(-) create mode 100644 downloader/snaptype/webseeds.go diff --git a/common/cli.go b/common/cli.go index 0f0008e35..c195b9151 100644 --- a/common/cli.go +++ b/common/cli.go @@ -20,6 +20,7 @@ import ( "context" "os" "os/signal" + "strings" "syscall" "github.com/ledgerwatch/log/v3" @@ -44,3 +45,14 @@ func RootContext() (context.Context, context.CancelFunc) { }() return ctx, cancel } + +func CliString2Array(input string) []string { + l := strings.Split(input, ",") + res := make([]string, 0, len(l)) + for _, r := range l { + if r = strings.TrimSpace(r); r != "" { + res = append(res, r) + } + } + return res +} diff --git a/downloader/downloader.go b/downloader/downloader.go index ee0a4c1be..0afafabb9 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -21,6 +21,8 @@ import ( "errors" "fmt" "io/fs" + "net/http" + "net/url" "os" "path/filepath" "runtime" @@ -34,10 +36,12 @@ import ( common2 "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" + "github.com/ledgerwatch/erigon-lib/downloader/snaptype" prototypes "github.com/ledgerwatch/erigon-lib/gointerfaces/types" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/log/v3" + "github.com/pelletier/go-toml/v2" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) @@ -57,6 +61,8 @@ type Downloader struct { folder storage.ClientImplCloser stopMainLoop context.CancelFunc wg sync.WaitGroup + + webseeds *WebSeeds } type AggStats struct { @@ -75,7 +81,7 @@ type AggStats struct { } func New(ctx context.Context, cfg *downloadercfg.Cfg) (*Downloader, error) { - if err := portMustBeTCPAndUDPOpen(cfg.ListenPort); err != nil { + if err := portMustBeTCPAndUDPOpen(cfg.ClientConfig.ListenPort); err != nil { return nil, err } @@ -83,10 +89,10 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg) (*Downloader, error) { // To provide such consistent view - downloader does: // add /snapshots/tmp - then method .onComplete will remove this suffix // and App only work with /snapshot s folder - if dir.FileExist(cfg.DataDir + "_tmp") { // migration from prev versions - _ = os.Rename(cfg.DataDir+"_tmp", filepath.Join(cfg.DataDir, "tmp")) // ignore error, because maybe they are on different drive, or target folder already created manually, all is fine + if dir.FileExist(cfg.SnapDir + "_tmp") { // migration from prev versions + _ = os.Rename(cfg.SnapDir+"_tmp", filepath.Join(cfg.SnapDir, "tmp")) // ignore error, because maybe they are on different drive, or target folder already created manually, all is fine } - if err := moveFromTmp(cfg.DataDir); err != nil { + if err := moveFromTmp(cfg.SnapDir); err != nil { return nil, err } @@ -99,7 +105,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg) (*Downloader, error) { if err != nil { return nil, fmt.Errorf("get peer id: %w", err) } - cfg.PeerID = string(peerID) + cfg.ClientConfig.PeerID = string(peerID) if len(peerID) == 0 { if err = savePeerID(db, torrentClient.PeerID()); err != nil { return nil, fmt.Errorf("save peer id: %w", err) @@ -115,10 +121,20 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg) (*Downloader, error) { clientLock: &sync.RWMutex{}, statsLock: &sync.RWMutex{}, + + webseeds: &WebSeeds{}, } if err := d.addSegments(ctx); err != nil { return nil, err } + // CornerCase: no peers -> no anoncments to trackers -> no magnetlink resolution (but magnetlink has filename) + // means we can start adding weebseeds without waiting for `<-t.GotInfo()` + d.wg.Add(1) + go func() { + defer d.wg.Done() + d.webseeds.Discover(ctx, d.cfg.WebSeedUrls, d.cfg.WebSeedFiles) + d.applyWebseeds() + }() return d, nil } @@ -142,6 +158,8 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { go func() { defer d.wg.Done() + // 2 loops: 1-st waiting for "torrents resolution" (receiving metadata from trackers) + // Torrents that are already taken care of torrentMap := map[metainfo.Hash]struct{}{} // First loop drops torrents that were downloaded or are already complete @@ -190,9 +208,6 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { atomic.StoreUint64(&d.stats.DroppedCompleted, 0) atomic.StoreUint64(&d.stats.DroppedTotal, 0) - if err := d.addSegments(ctx); err != nil { - return - } DownloadLoop2: torrents = d.Torrent().Torrents() for _, t := range torrents { @@ -289,7 +304,7 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { func (d *Downloader) SnapDir() string { d.clientLock.RLock() defer d.clientLock.RUnlock() - return d.cfg.DataDir + return d.cfg.SnapDir } func (d *Downloader) ReCalcStats(interval time.Duration) { @@ -451,7 +466,7 @@ func (d *Downloader) VerifyData(ctx context.Context) error { return d.db.Update(context.Background(), func(tx kv.RwTx) error { return nil }) } -func (d *Downloader) createMagnetLinkWithInfoHash(ctx context.Context, hash *prototypes.H160, snapDir string) (bool, error) { +func (d *Downloader) createMagnetLinkWithInfoHash(ctx context.Context, hash *prototypes.H160, name string, snapDir string) (bool, error) { mi := &metainfo.MetaInfo{AnnounceList: Trackers} if hash == nil { return false, nil @@ -464,7 +479,7 @@ func (d *Downloader) createMagnetLinkWithInfoHash(ctx context.Context, hash *pro return true, nil } - magnet := mi.Magnet(&infoHash, nil) + magnet := mi.Magnet(&infoHash, &metainfo.Info{Name: name}) t, err := d.torrentClient.AddMagnet(magnet.String()) if err != nil { //log.Warn("[downloader] add magnet link", "err", err) @@ -491,6 +506,23 @@ func (d *Downloader) createMagnetLinkWithInfoHash(ctx context.Context, hash *pro return false, nil } +func seedableFiles(snapDir string) ([]string, error) { + files, err := seedableSegmentFiles(snapDir) + if err != nil { + return nil, fmt.Errorf("seedableSegmentFiles: %w", err) + } + files2, err := seedableHistorySnapshots(snapDir, "history") + if err != nil { + return nil, fmt.Errorf("seedableHistorySnapshots: %w", err) + } + files = append(files, files2...) + files2, err = seedableHistorySnapshots(snapDir, "warm") + if err != nil { + return nil, fmt.Errorf("seedableHistorySnapshots: %w", err) + } + files = append(files, files2...) + return files, nil +} func (d *Downloader) addSegments(ctx context.Context) error { logEvery := time.NewTicker(20 * time.Second) defer logEvery.Stop() @@ -498,18 +530,16 @@ func (d *Downloader) addSegments(ctx context.Context) error { if err != nil { return err } - files, err := seedableSegmentFiles(d.SnapDir()) - if err != nil { - return fmt.Errorf("seedableSegmentFiles: %w", err) - } - files2, err := seedableHistorySnapshots(d.SnapDir()) + err = AddTorrentFiles(d.SnapDir(), d.torrentClient) if err != nil { - return fmt.Errorf("seedableHistorySnapshots: %w", err) + return fmt.Errorf("AddTorrentFiles: %w", err) } - files = append(files, files2...) - g, ctx := errgroup.WithContext(ctx) i := atomic.Int64{} + files, err := seedableFiles(d.SnapDir()) + if err != nil { + return err + } for _, f := range files { f := f g.Go(func() error { @@ -518,7 +548,7 @@ func (d *Downloader) addSegments(ctx context.Context) error { return ctx.Err() default: } - _, err := AddSegment(f, d.cfg.DataDir, d.torrentClient) + _, err := AddSegment(f, d.cfg.SnapDir, d.torrentClient) if err != nil { return err } @@ -612,3 +642,86 @@ func openClient(cfg *torrent.ClientConfig) (db kv.RwDB, c storage.PieceCompletio return db, c, m, torrentClient, nil } + +func (d *Downloader) applyWebseeds() { + for _, t := range d.Torrent().Torrents() { + urls, ok := d.webseeds.GetByFileNames()[t.Name()] + if !ok { + continue + } + log.Debug("[downloader] addd webseeds", "file", t.Name()) + t.AddWebSeeds(urls) + } +} + +type WebSeeds struct { + lock sync.Mutex + webSeedsByFilName snaptype.WebSeeds +} + +func (d *WebSeeds) GetByFileNames() snaptype.WebSeeds { + d.lock.Lock() + defer d.lock.Unlock() + return d.webSeedsByFilName +} +func (d *WebSeeds) SetByFileNames(l snaptype.WebSeeds) { + d.lock.Lock() + defer d.lock.Unlock() + d.webSeedsByFilName = l +} + +func (d *WebSeeds) callWebSeedsProvider(ctx context.Context, webSeedProviderUrl *url.URL) (snaptype.WebSeedsFromProvider, error) { + request, err := http.NewRequest(http.MethodGet, webSeedProviderUrl.String(), nil) + if err != nil { + return nil, err + } + request = request.WithContext(ctx) + resp, err := http.DefaultClient.Do(request) + if err != nil { + return nil, err + } + defer resp.Body.Close() + response := snaptype.WebSeedsFromProvider{} + if err := toml.NewDecoder(resp.Body).Decode(&response); err != nil { + return nil, err + } + return response, nil +} +func (d *WebSeeds) readWebSeedsFile(webSeedProviderPath string) (snaptype.WebSeedsFromProvider, error) { + data, err := os.ReadFile(webSeedProviderPath) + if err != nil { + return nil, err + } + response := snaptype.WebSeedsFromProvider{} + if err := toml.Unmarshal(data, &response); err != nil { + return nil, err + } + return response, nil +} + +func (d *WebSeeds) Discover(ctx context.Context, urls []*url.URL, files []string) { + list := make([]snaptype.WebSeedsFromProvider, len(urls)+len(files)) + for _, webSeedProviderURL := range urls { + select { + case <-ctx.Done(): + break + default: + } + response, err := d.callWebSeedsProvider(ctx, webSeedProviderURL) + if err != nil { // don't fail on error + log.Warn("[downloader] callWebSeedsProvider", "err", err, "url", webSeedProviderURL.EscapedPath()) + continue + } + list = append(list, response) + } + for _, webSeedFile := range files { + response, err := d.readWebSeedsFile(webSeedFile) + if err != nil { // don't fail on error + _, fileName := filepath.Split(webSeedFile) + log.Warn("[downloader] readWebSeedsFile", "err", err, "file", fileName) + continue + } + list = append(list, response) + } + d.SetByFileNames(snaptype.NewWebSeeds(list)) +} diff --git a/downloader/downloader_grpc_server.go b/downloader/downloader_grpc_server.go index e6e7a5c2e..812fe9648 100644 --- a/downloader/downloader_grpc_server.go +++ b/downloader/downloader_grpc_server.go @@ -47,13 +47,16 @@ type GrpcServer struct { func (s *GrpcServer) Download(ctx context.Context, request *proto_downloader.DownloadRequest) (*emptypb.Empty, error) { logEvery := time.NewTicker(20 * time.Second) defer logEvery.Stop() + defer s.d.applyWebseeds() torrentClient := s.d.Torrent() snapDir := s.d.SnapDir() for i, it := range request.Items { + if it.Path == "" { + return nil, fmt.Errorf("field 'path' is required") + } + select { - case <-ctx.Done(): - return nil, ctx.Err() case <-logEvery.C: log.Info("[snapshots] initializing", "files", fmt.Sprintf("%d/%d", i, len(request.Items))) default: @@ -62,7 +65,7 @@ func (s *GrpcServer) Download(ctx context.Context, request *proto_downloader.Dow if it.TorrentHash == nil { // if we dont have the torrent hash then we seed a new snapshot log.Info("[snapshots] seeding a new snapshot") - ok, err := seedNewSnapshot(it, torrentClient, snapDir) + ok, err := seedNewSnapshot(ctx, it.Path, torrentClient, snapDir) if err != nil { return nil, err } @@ -74,7 +77,7 @@ func (s *GrpcServer) Download(ctx context.Context, request *proto_downloader.Dow continue } - _, err := s.d.createMagnetLinkWithInfoHash(ctx, it.TorrentHash, snapDir) + _, err := s.d.createMagnetLinkWithInfoHash(ctx, it.TorrentHash, it.Path, snapDir) if err != nil { return nil, err } @@ -117,14 +120,19 @@ func Proto2InfoHash(in *prototypes.H160) metainfo.Hash { // decides what we do depending on wether we have the .seg file or the .torrent file // have .torrent no .seg => get .seg file from .torrent // have .seg no .torrent => get .torrent from .seg -func seedNewSnapshot(it *proto_downloader.DownloadItem, torrentClient *torrent.Client, snapDir string) (bool, error) { +func seedNewSnapshot(ctx context.Context, name string, torrentClient *torrent.Client, snapDir string) (bool, error) { + select { + case <-ctx.Done(): + return false, ctx.Err() + default: + } // if we dont have the torrent file we build it if we have the .seg file - if err := buildTorrentIfNeed(it.Path, snapDir); err != nil { + if err := buildTorrentIfNeed(ctx, name, snapDir); err != nil { return false, err } // we add the .seg file we have and create the .torrent file if we dont have it - ok, err := AddSegment(it.Path, snapDir, torrentClient) + ok, err := AddSegment(name, snapDir, torrentClient) if err != nil { return false, fmt.Errorf("AddSegment: %w", err) } @@ -137,5 +145,3 @@ func seedNewSnapshot(it *proto_downloader.DownloadItem, torrentClient *torrent.C // we skip the item in for loop since we build the seg and torrent file here return true, nil } - -// we dont have .seg or .torrent so we get them through the torrent hash diff --git a/downloader/downloadercfg/downloadercfg.go b/downloader/downloadercfg/downloadercfg.go index aa5dbf241..1cdacd12c 100644 --- a/downloader/downloadercfg/downloadercfg.go +++ b/downloader/downloadercfg/downloadercfg.go @@ -19,6 +19,8 @@ package downloadercfg import ( "io/ioutil" "net" + "net/url" + "path/filepath" "runtime" "strings" @@ -26,6 +28,9 @@ import ( lg "github.com/anacrolix/log" "github.com/anacrolix/torrent" "github.com/c2h5oh/datasize" + "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/datadir" + "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/log/v3" "golang.org/x/time/rate" ) @@ -40,8 +45,11 @@ const DefaultPieceSize = 2 * 1024 * 1024 const DefaultNetworkChunkSize = 512 * 1024 type Cfg struct { - *torrent.ClientConfig + ClientConfig *torrent.ClientConfig + SnapDir string DownloadSlots int + WebSeedUrls []*url.URL + WebSeedFiles []string } func Default() *torrent.ClientConfig { @@ -51,7 +59,6 @@ func Default() *torrent.ClientConfig { torrentConfig.NoDHT = true //torrentConfig.DisableTrackers = true //torrentConfig.DisableWebtorrent = true - //torrentConfig.DisableWebseeds = true // Reduce defaults - to avoid peers with very bad geography //torrentConfig.MinDialTimeout = 1 * time.Second // default: 3sec @@ -70,13 +77,14 @@ func Default() *torrent.ClientConfig { return torrentConfig } -func New(snapDir string, version string, verbosity lg.Level, downloadRate, uploadRate datasize.ByteSize, port, connsPerFile, downloadSlots int, staticPeers []string) (*Cfg, error) { +func New(dataDir datadir.Dirs, version string, verbosity lg.Level, downloadRate, uploadRate datasize.ByteSize, port, connsPerFile, downloadSlots int, staticPeers []string, webseeds string) (*Cfg, error) { torrentConfig := Default() + torrentConfig.DataDir = dataDir.Snap // `DataDir` of torrent-client-lib is different from Erigon's `DataDir`. Just same naming. + torrentConfig.ExtendedHandshakeClientVersion = version // We would-like to reduce amount of goroutines in Erigon, so reducing next params torrentConfig.EstablishedConnsPerTorrent = connsPerFile // default: 50 - torrentConfig.DataDir = snapDir torrentConfig.ListenPort = port // check if ipv6 is enabled @@ -94,7 +102,7 @@ func New(snapDir string, version string, verbosity lg.Level, downloadRate, uploa // debug // torrentConfig.Debug = false - torrentConfig.Logger = lg.Default.FilterLevel(verbosity) + torrentConfig.Logger.WithFilterLevel(verbosity) torrentConfig.Logger.Handlers = []lg.Handler{adapterHandler{}} if len(staticPeers) > 0 { @@ -133,7 +141,28 @@ func New(snapDir string, version string, verbosity lg.Level, downloadRate, uploa //staticPeers } - return &Cfg{ClientConfig: torrentConfig, DownloadSlots: downloadSlots}, nil + webseedUrlsOrFiles := common.CliString2Array(webseeds) + webseedUrls := make([]*url.URL, 0, len(webseedUrlsOrFiles)) + webseedFiles := make([]string, 0, len(webseedUrlsOrFiles)) + for _, webseed := range webseedUrlsOrFiles { + uri, err := url.ParseRequestURI(webseed) + if err != nil { + if strings.HasSuffix(webseed, ".toml") && dir.FileExist(webseed) { + webseedFiles = append(webseedFiles, webseed) + } + continue + } + webseedUrls = append(webseedUrls, uri) + } + localCfgFile := filepath.Join(dataDir.DataDir, "webseeds.toml") // datadir/webseeds.toml allowed + if dir.FileExist(localCfgFile) { + webseedFiles = append(webseedFiles, localCfgFile) + } + + return &Cfg{SnapDir: torrentConfig.DataDir, + ClientConfig: torrentConfig, DownloadSlots: downloadSlots, + WebSeedUrls: webseedUrls, WebSeedFiles: webseedFiles, + }, nil } func getIpv6Enabled() bool { diff --git a/downloader/snaptype/webseeds.go b/downloader/snaptype/webseeds.go new file mode 100644 index 000000000..0a7e6a7db --- /dev/null +++ b/downloader/snaptype/webseeds.go @@ -0,0 +1,19 @@ +package snaptype + +import "github.com/anacrolix/torrent/metainfo" + +// Each provider can provide only 1 WebSeed url per file +// but overall BitTorrent protocol allowing multiple +type WebSeedsFromProvider map[string]string // fileName -> Url, can be Http/Ftp + +type WebSeeds map[string]metainfo.UrlList // fileName -> []Url, can be Http/Ftp + +func NewWebSeeds(list []WebSeedsFromProvider) WebSeeds { + merged := WebSeeds{} + for _, m := range list { + for name, wUrl := range m { + merged[name] = append(merged[name], wUrl) + } + } + return merged +} diff --git a/downloader/util.go b/downloader/util.go index 88cd38293..0eb4dbc35 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -127,13 +127,6 @@ func seedableSegmentFiles(dir string) ([]string, error) { if !snaptype.IsCorrectFileName(f.Name()) { continue } - fileInfo, err := f.Info() - if err != nil { - return nil, err - } - if fileInfo.Size() == 0 { - continue - } if filepath.Ext(f.Name()) != ".seg" { // filter out only compressed files continue } @@ -149,10 +142,10 @@ func seedableSegmentFiles(dir string) ([]string, error) { return res, nil } -var historyFileRegex = regexp.MustCompile("^([[:lower:]]+).([0-9]+)-([0-9]+).(v|ef)$") +var historyFileRegex = regexp.MustCompile("^([[:lower:]]+).([0-9]+)-([0-9]+).(.*)$") -func seedableHistorySnapshots(dir string) ([]string, error) { - historyDir := filepath.Join(dir, "history") +func seedableHistorySnapshots(dir, subDir string) ([]string, error) { + historyDir := filepath.Join(dir, subDir) dir2.MustExist(historyDir) files, err := os.ReadDir(historyDir) if err != nil { @@ -166,13 +159,6 @@ func seedableHistorySnapshots(dir string) ([]string, error) { if !f.Type().IsRegular() { continue } - fileInfo, err := f.Info() - if err != nil { - return nil, err - } - if fileInfo.Size() == 0 { - continue - } ext := filepath.Ext(f.Name()) if ext != ".v" && ext != ".ef" && ext != ".kv" { // filter out only compressed files continue @@ -182,7 +168,7 @@ func seedableHistorySnapshots(dir string) ([]string, error) { if len(subs) != 5 { continue } - + // Check that it's seedable from, err := strconv.ParseUint(subs[2], 10, 64) if err != nil { return nil, fmt.Errorf("ParseFileName: %w", err) @@ -191,15 +177,15 @@ func seedableHistorySnapshots(dir string) ([]string, error) { if err != nil { return nil, fmt.Errorf("ParseFileName: %w", err) } - if to-from != snaptype.Erigon3SeedableSteps { + if (to-from)%snaptype.Erigon3SeedableSteps != 0 { continue } - res = append(res, filepath.Join("history", f.Name())) + res = append(res, filepath.Join(subDir, f.Name())) } return res, nil } -func buildTorrentIfNeed(fName, root string) (err error) { +func buildTorrentIfNeed(ctx context.Context, fName, root string) (err error) { fPath := filepath.Join(root, fName) if dir2.FileExist(fPath + ".torrent") { return @@ -213,7 +199,7 @@ func buildTorrentIfNeed(fName, root string) (err error) { } info.Name = fName - return createTorrentFileFromInfo(root, info, nil) + return CreateTorrentFileFromInfo(root, info, nil) } // AddSegment - add existing .seg file, create corresponding .torrent if need @@ -222,7 +208,11 @@ func AddSegment(originalFileName, snapDir string, client *torrent.Client) (bool, if !dir2.FileExist(fPath + ".torrent") { return false, nil } - _, err := AddTorrentFile(fPath+".torrent", client) + ts, err := loadTorrent(fPath + ".torrent") + if err != nil { + return false, err + } + _, err = AddTorrentFile(ts, client) if err != nil { return false, fmt.Errorf("AddTorrentFile: %w", err) } @@ -234,15 +224,10 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err logEvery := time.NewTicker(20 * time.Second) defer logEvery.Stop() - files, err := seedableSegmentFiles(snapDir) - if err != nil { - return nil, err - } - files2, err := seedableHistorySnapshots(snapDir) + files, err := seedableFiles(snapDir) if err != nil { return nil, err } - files = append(files, files2...) errs := make(chan error, len(files)*2) wg := &sync.WaitGroup{} @@ -258,7 +243,7 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err defer i.Add(1) defer sem.Release(1) defer wg.Done() - if err := buildTorrentIfNeed(f, snapDir); err != nil { + if err := buildTorrentIfNeed(ctx, f, snapDir); err != nil { errs <- err } @@ -288,17 +273,17 @@ func CreateTorrentFileIfNotExists(root string, info *metainfo.Info, mi *metainfo if dir2.FileExist(fPath + ".torrent") { return nil } - if err := createTorrentFileFromInfo(root, info, mi); err != nil { + if err := CreateTorrentFileFromInfo(root, info, mi); err != nil { return err } return nil } -func createTorrentFileFromInfo(root string, info *metainfo.Info, mi *metainfo.MetaInfo) error { +func CreateMetaInfo(info *metainfo.Info, mi *metainfo.MetaInfo) (*metainfo.MetaInfo, error) { if mi == nil { infoBytes, err := bencode.Marshal(info) if err != nil { - return err + return nil, err } mi = &metainfo.MetaInfo{ CreationDate: time.Now().Unix(), @@ -309,8 +294,10 @@ func createTorrentFileFromInfo(root string, info *metainfo.Info, mi *metainfo.Me } else { mi.AnnounceList = Trackers } + return mi, nil +} +func CreateTorrentFromMetaInfo(root string, info *metainfo.Info, mi *metainfo.MetaInfo) error { torrentFileName := filepath.Join(root, info.Name+".torrent") - file, err := os.Create(torrentFileName) if err != nil { return err @@ -322,28 +309,82 @@ func createTorrentFileFromInfo(root string, info *metainfo.Info, mi *metainfo.Me file.Sync() return nil } +func CreateTorrentFileFromInfo(root string, info *metainfo.Info, mi *metainfo.MetaInfo) (err error) { + mi, err = CreateMetaInfo(info, mi) + if err != nil { + return err + } + return CreateTorrentFromMetaInfo(root, info, mi) +} -// nolint -func segmentFileNameFromTorrentFileName(in string) string { - ext := filepath.Ext(in) - return in[0 : len(in)-len(ext)] +func AddTorrentFiles(snapDir string, torrentClient *torrent.Client) error { + files, err := allTorrentFiles(snapDir) + if err != nil { + return err + } + for _, ts := range files { + _, err := AddTorrentFile(ts, torrentClient) + if err != nil { + return err + } + } + + return nil } -// AddTorrentFile - adding .torrent file to torrentClient (and checking their hashes), if .torrent file -// added first time - pieces verification process will start (disk IO heavy) - Progress -// kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again. -// Don't need call torrent.VerifyData manually -func AddTorrentFile(torrentFilePath string, torrentClient *torrent.Client) (*torrent.Torrent, error) { - mi, err := metainfo.LoadFromFile(torrentFilePath) +func allTorrentFiles(snapDir string) (res []*torrent.TorrentSpec, err error) { + res, err = torrentInDir(snapDir) if err != nil { return nil, err } - mi.AnnounceList = Trackers - ts, err := torrent.TorrentSpecFromMetaInfoErr(mi) + res2, err := torrentInDir(filepath.Join(snapDir, "history")) if err != nil { return nil, err } + res = append(res, res2...) + res2, err = torrentInDir(filepath.Join(snapDir, "warm")) + if err != nil { + return nil, err + } + res = append(res, res2...) + return res, nil +} +func torrentInDir(snapDir string) (res []*torrent.TorrentSpec, err error) { + files, err := os.ReadDir(snapDir) + if err != nil { + return nil, err + } + for _, f := range files { + if f.IsDir() || !f.Type().IsRegular() { + continue + } + if filepath.Ext(f.Name()) != ".torrent" { // filter out only compressed files + continue + } + + a, err := loadTorrent(filepath.Join(snapDir, f.Name())) + if err != nil { + return nil, err + } + res = append(res, a) + } + return res, nil +} + +func loadTorrent(torrentFilePath string) (*torrent.TorrentSpec, error) { + mi, err := metainfo.LoadFromFile(torrentFilePath) + if err != nil { + return nil, fmt.Errorf("LoadFromFile: %w, file=%s", err, torrentFilePath) + } + mi.AnnounceList = Trackers + return torrent.TorrentSpecFromMetaInfoErr(mi) +} +// AddTorrentFile - adding .torrent file to torrentClient (and checking their hashes), if .torrent file +// added first time - pieces verification process will start (disk IO heavy) - Progress +// kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again. +// Don't need call torrent.VerifyData manually +func AddTorrentFile(ts *torrent.TorrentSpec, torrentClient *torrent.Client) (*torrent.Torrent, error) { if _, ok := torrentClient.Torrent(ts.InfoHash); !ok { // can set ChunkSize only for new torrents ts.ChunkSize = downloadercfg.DefaultNetworkChunkSize } else { diff --git a/go.mod b/go.mod index 0d74ff6d8..a99a43743 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/holiman/uint256 v1.2.3 github.com/matryer/moq v0.3.2 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 + github.com/pelletier/go-toml/v2 v2.1.0 github.com/quasilyte/go-ruleguard/dsl v0.3.22 github.com/spaolacci/murmur3 v1.1.0 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index cb7bdc001..f37f81e07 100644 --- a/go.sum +++ b/go.sum @@ -277,6 +277,8 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E= From 5e87ab7e20956f868bb2b3385c3be7c6d59a00a8 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 12 Sep 2023 12:51:37 +0700 Subject: [PATCH 02/14] save --- downloader/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downloader/util.go b/downloader/util.go index 454a34ef6..38d72039b 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -111,7 +111,7 @@ func AllTorrentFiles(dir string) ([]string, error) { return res, nil } -func seedableBlocksSnapshots(dir string) ([]string, error) { +func seedableSegmentFiles(dir string) ([]string, error) { files, err := os.ReadDir(dir) if err != nil { return nil, err From 84de6dfaae5e3e9d716d5fd510151df3e36ce9ea Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 13 Sep 2023 10:20:21 +0700 Subject: [PATCH 03/14] save --- downloader/util.go | 56 +++++++++++++++++++--------------------------- 1 file changed, 23 insertions(+), 33 deletions(-) diff --git a/downloader/util.go b/downloader/util.go index 0eb4dbc35..ff9473212 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -18,15 +18,16 @@ package downloader import ( "context" + "github.com/ledgerwatch/erigon-lib/common/cmp" + "runtime" + //nolint:gosec "fmt" "net" "os" "path/filepath" "regexp" - "runtime" "strconv" - "sync" "sync/atomic" "time" @@ -34,14 +35,12 @@ import ( "github.com/anacrolix/torrent/bencode" "github.com/anacrolix/torrent/metainfo" common2 "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/cmp" dir2 "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/log/v3" - - "golang.org/x/sync/semaphore" + "golang.org/x/sync/errgroup" ) // `github.com/anacrolix/torrent` library spawning several goroutines and producing many requests for each tracker. So we limit amout of trackers by 7 @@ -229,41 +228,32 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err return nil, err } - errs := make(chan error, len(files)*2) - wg := &sync.WaitGroup{} - workers := cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 2 - var sem = semaphore.NewWeighted(int64(workers)) - i := atomic.Int32{} - for _, file := range files { - wg.Add(1) - if err := sem.Acquire(ctx, 1); err != nil { - return nil, err - } - go func(f string) { - defer i.Add(1) - defer sem.Release(1) - defer wg.Done() - if err := buildTorrentIfNeed(ctx, f, snapDir); err != nil { - errs <- err - } - + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 4) + var i atomic.Int32 + g.Go(func() error { + for { select { default: case <-ctx.Done(): - errs <- ctx.Err() + return ctx.Err() case <-logEvery.C: log.Info("[snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i.Load(), len(files))) } - }(file) - } - go func() { - wg.Wait() - close(errs) - }() - for err := range errs { - if err != nil { - return nil, err } + }) + for _, file := range files { + file := file + g.Go(func() error { + defer i.Add(1) + if err := buildTorrentIfNeed(ctx, file, snapDir); err != nil { + return err + } + return nil + }) + } + if err := g.Wait(); err != nil { + return nil, err } return files, nil } From 8bb948c65ce7cfea4599727827e290582e66520c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 13 Sep 2023 10:32:17 +0700 Subject: [PATCH 04/14] save --- downloader/util.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/downloader/util.go b/downloader/util.go index bf24ff549..b78182f8f 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -19,6 +19,7 @@ package downloader import ( "context" "github.com/ledgerwatch/erigon-lib/common/cmp" + "github.com/ledgerwatch/erigon-lib/common/dbg" "runtime" //nolint:gosec @@ -244,13 +245,15 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err g.SetLimit(cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 4) var i atomic.Int32 g.Go(func() error { + var m runtime.MemStats for { select { default: case <-ctx.Done(): return ctx.Err() case <-logEvery.C: - log.Info("[snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i.Load(), len(files))) + dbg.ReadMemStats(&m) + log.Info("[snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) } } }) From a1d72054687797c9342d69088492caf3acd7bd24 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 13 Sep 2023 10:32:59 +0700 Subject: [PATCH 05/14] save --- downloader/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/downloader/util.go b/downloader/util.go index b78182f8f..918f4d2bf 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -242,7 +242,7 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err } g, ctx := errgroup.WithContext(ctx) - g.SetLimit(cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 4) + g.SetLimit(cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 8) var i atomic.Int32 g.Go(func() error { var m runtime.MemStats From 139fe8fe33e1c98c741bb118b6e6810f4f3efdd7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 13 Sep 2023 10:42:26 +0700 Subject: [PATCH 06/14] save --- downloader/util.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/downloader/util.go b/downloader/util.go index ff9473212..4d4d34f85 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -231,17 +231,16 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err g, ctx := errgroup.WithContext(ctx) g.SetLimit(cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 4) var i atomic.Int32 - g.Go(func() error { + go func() { // will exit when `errgroup` exit, but will not block it for { select { - default: case <-ctx.Done(): - return ctx.Err() + return case <-logEvery.C: log.Info("[snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i.Load(), len(files))) } } - }) + }() for _, file := range files { file := file g.Go(func() error { From 0a502dd64a022fdee4018937f319570a091a749b Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 13 Sep 2023 10:45:39 +0700 Subject: [PATCH 07/14] save --- downloader/util.go | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/downloader/util.go b/downloader/util.go index e47fce6c3..30fbc7b58 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -20,6 +20,7 @@ import ( "context" "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/dbg" + "github.com/ledgerwatch/log/v3" "runtime" //nolint:gosec @@ -40,7 +41,6 @@ import ( "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/kv" - "github.com/ledgerwatch/log/v3" "golang.org/x/sync/errgroup" ) @@ -244,18 +244,7 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err g, ctx := errgroup.WithContext(ctx) g.SetLimit(cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 8) var i atomic.Int32 - go func() { // will exit when `errgroup` exit, but will not block it - var m runtime.MemStats - for { - select { - case <-ctx.Done(): - return - case <-logEvery.C: - dbg.ReadMemStats(&m) - log.Info("[snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) - } - } - }() + for _, file := range files { file := file g.Go(func() error { @@ -266,6 +255,18 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err return nil }) } + + var m runtime.MemStats +Loop: + for { + select { + case <-ctx.Done(): + break Loop // g.Wait() will return right error + case <-logEvery.C: + dbg.ReadMemStats(&m) + log.Info("[snapshots] Creating .torrent files", "progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) + } + } if err := g.Wait(); err != nil { return nil, err } From d4cfe814383c9458fdc412d33f39fb273d1b99b0 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 13 Sep 2023 11:02:19 +0700 Subject: [PATCH 08/14] save --- downloader/util.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/downloader/util.go b/downloader/util.go index 30fbc7b58..952e633a1 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -18,9 +18,6 @@ package downloader import ( "context" - "github.com/ledgerwatch/erigon-lib/common/cmp" - "github.com/ledgerwatch/erigon-lib/common/dbg" - "github.com/ledgerwatch/log/v3" "runtime" //nolint:gosec @@ -37,10 +34,13 @@ import ( "github.com/anacrolix/torrent/bencode" "github.com/anacrolix/torrent/metainfo" common2 "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common/cmp" + "github.com/ledgerwatch/erigon-lib/common/dbg" dir2 "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/log/v3" "golang.org/x/sync/errgroup" ) @@ -252,21 +252,17 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err if err := buildTorrentIfNeed(ctx, file, snapDir); err != nil { return err } + select { + case <-ctx.Done(): + return ctx.Err() + case <-logEvery.C: + var m runtime.MemStats + dbg.ReadMemStats(&m) + log.Info("[snapshots] Creating .torrent files", "progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) + } return nil }) } - - var m runtime.MemStats -Loop: - for { - select { - case <-ctx.Done(): - break Loop // g.Wait() will return right error - case <-logEvery.C: - dbg.ReadMemStats(&m) - log.Info("[snapshots] Creating .torrent files", "progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) - } - } if err := g.Wait(); err != nil { return nil, err } From 2e7538a3f098b559f85d6a773f14e9bf2741c304 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 13 Sep 2023 11:06:23 +0700 Subject: [PATCH 09/14] save --- downloader/util.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/downloader/util.go b/downloader/util.go index 952e633a1..57123b606 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -252,17 +252,22 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err if err := buildTorrentIfNeed(ctx, file, snapDir); err != nil { return err } - select { - case <-ctx.Done(): - return ctx.Err() - case <-logEvery.C: - var m runtime.MemStats - dbg.ReadMemStats(&m) - log.Info("[snapshots] Creating .torrent files", "progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) - } + return nil }) } + + var m runtime.MemStats +Loop: + for int(i.Load()) < len(files) { + select { + case <-ctx.Done(): + break Loop // g.Wait() will return right error + case <-logEvery.C: + dbg.ReadMemStats(&m) + log.Info("[snapshots] Creating .torrent files", "progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) + } + } if err := g.Wait(); err != nil { return nil, err } From 810c4022f9762b07f9d4072568e07a8fa424fa37 Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Wed, 13 Sep 2023 11:49:42 +0700 Subject: [PATCH 10/14] Downloader: correct logging when create .torrent files (#1123) --- downloader/util.go | 79 +++++++++++++++++++++++------------------ tools/licenses_check.sh | 2 ++ 2 files changed, 46 insertions(+), 35 deletions(-) diff --git a/downloader/util.go b/downloader/util.go index 0eb4dbc35..d326b1b02 100644 --- a/downloader/util.go +++ b/downloader/util.go @@ -18,7 +18,6 @@ package downloader import ( "context" - //nolint:gosec "fmt" "net" "os" @@ -26,7 +25,6 @@ import ( "regexp" "runtime" "strconv" - "sync" "sync/atomic" "time" @@ -35,13 +33,13 @@ import ( "github.com/anacrolix/torrent/metainfo" common2 "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/cmp" + "github.com/ledgerwatch/erigon-lib/common/dbg" dir2 "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/log/v3" - - "golang.org/x/sync/semaphore" + "golang.org/x/sync/errgroup" ) // `github.com/anacrolix/torrent` library spawning several goroutines and producing many requests for each tracker. So we limit amout of trackers by 7 @@ -145,6 +143,18 @@ func seedableSegmentFiles(dir string) ([]string, error) { var historyFileRegex = regexp.MustCompile("^([[:lower:]]+).([0-9]+)-([0-9]+).(.*)$") func seedableHistorySnapshots(dir, subDir string) ([]string, error) { + l, err := seedableSnapshotsBySubDir(dir, "history") + if err != nil { + return nil, err + } + l2, err := seedableSnapshotsBySubDir(dir, "warm") + if err != nil { + return nil, err + } + return append(l, l2...), nil +} + +func seedableSnapshotsBySubDir(dir, subDir string) ([]string, error) { historyDir := filepath.Join(dir, subDir) dir2.MustExist(historyDir) files, err := os.ReadDir(historyDir) @@ -186,6 +196,12 @@ func seedableHistorySnapshots(dir, subDir string) ([]string, error) { } func buildTorrentIfNeed(ctx context.Context, fName, root string) (err error) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + fPath := filepath.Join(root, fName) if dir2.FileExist(fPath + ".torrent") { return @@ -229,42 +245,35 @@ func BuildTorrentFilesIfNeed(ctx context.Context, snapDir string) ([]string, err return nil, err } - errs := make(chan error, len(files)*2) - wg := &sync.WaitGroup{} - workers := cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 2 - var sem = semaphore.NewWeighted(int64(workers)) - i := atomic.Int32{} + g, ctx := errgroup.WithContext(ctx) + g.SetLimit(cmp.Max(1, runtime.GOMAXPROCS(-1)-1) * 4) + var i atomic.Int32 + for _, file := range files { - wg.Add(1) - if err := sem.Acquire(ctx, 1); err != nil { - return nil, err - } - go func(f string) { + file := file + g.Go(func() error { defer i.Add(1) - defer sem.Release(1) - defer wg.Done() - if err := buildTorrentIfNeed(ctx, f, snapDir); err != nil { - errs <- err - } - - select { - default: - case <-ctx.Done(): - errs <- ctx.Err() - case <-logEvery.C: - log.Info("[snapshots] Creating .torrent files", "Progress", fmt.Sprintf("%d/%d", i.Load(), len(files))) + if err := buildTorrentIfNeed(ctx, file, snapDir); err != nil { + return err } - }(file) - } - go func() { - wg.Wait() - close(errs) - }() - for err := range errs { - if err != nil { - return nil, err + return nil + }) + } + + var m runtime.MemStats +Loop: + for int(i.Load()) < len(files) { + select { + case <-ctx.Done(): + break Loop // g.Wait() will return right error + case <-logEvery.C: + dbg.ReadMemStats(&m) + log.Info("[snapshots] Creating .torrent files", "progress", fmt.Sprintf("%d/%d", i.Load(), len(files)), "alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys)) } } + if err := g.Wait(); err != nil { + return nil, err + } return files, nil } diff --git a/tools/licenses_check.sh b/tools/licenses_check.sh index b56b7e3cf..31d76e634 100755 --- a/tools/licenses_check.sh +++ b/tools/licenses_check.sh @@ -35,6 +35,8 @@ output=$(find "$projectDir" -type 'd' -maxdepth 1 \ | grep -v "github.com/anacrolix/multiless" `# MPL-2.0` \ | grep -v "github.com/anacrolix/sync" `# MPL-2.0` \ | grep -v "github.com/anacrolix/upnp" `# MPL-2.0` \ + | grep -v "github.com/go-llsqlite/adapter" `# MPL-2.0` \ + | grep -v "github.com/go-llsqlite/crawshaw" `# ISC` \ | grep -v "github.com/consensys/gnark-crypto" `# Apache-2.0` \ | grep -v "github.com/erigontech/mdbx-go" `# Apache-2.0` \ | grep -v "github.com/ledgerwatch/secp256k1" `# BSD-3-Clause` \ From b157b162f73348ef719130a3e245dd2b92a5d96f Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Wed, 13 Sep 2023 15:21:38 +0700 Subject: [PATCH 11/14] downloader: support --chain parameter (#1124) --- downloader/downloader.go | 15 +++++---------- downloader/downloader_grpc_server.go | 2 +- downloader/snaptype/files.go | 11 +++++++++++ 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/downloader/downloader.go b/downloader/downloader.go index 0afafabb9..88fe35e03 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -37,7 +37,6 @@ import ( "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" - prototypes "github.com/ledgerwatch/erigon-lib/gointerfaces/types" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/log/v3" @@ -466,24 +465,20 @@ func (d *Downloader) VerifyData(ctx context.Context) error { return d.db.Update(context.Background(), func(tx kv.RwTx) error { return nil }) } -func (d *Downloader) createMagnetLinkWithInfoHash(ctx context.Context, hash *prototypes.H160, name string, snapDir string) (bool, error) { +func (d *Downloader) AddInfoHashAsMagnetLink(ctx context.Context, infoHash metainfo.Hash, name string) error { mi := &metainfo.MetaInfo{AnnounceList: Trackers} - if hash == nil { - return false, nil - } - infoHash := Proto2InfoHash(hash) //log.Debug("[downloader] downloading torrent and seg file", "hash", infoHash) if _, ok := d.torrentClient.Torrent(infoHash); ok { //log.Debug("[downloader] torrent client related to hash found", "hash", infoHash) - return true, nil + return nil } magnet := mi.Magnet(&infoHash, &metainfo.Info{Name: name}) t, err := d.torrentClient.AddMagnet(magnet.String()) if err != nil { //log.Warn("[downloader] add magnet link", "err", err) - return false, err + return err } t.DisallowDataDownload() t.AllowDataUpload() @@ -497,13 +492,13 @@ func (d *Downloader) createMagnetLinkWithInfoHash(ctx context.Context, hash *pro } mi := t.Metainfo() - if err := CreateTorrentFileIfNotExists(snapDir, t.Info(), &mi); err != nil { + if err := CreateTorrentFileIfNotExists(d.SnapDir(), t.Info(), &mi); err != nil { log.Warn("[downloader] create torrent file", "err", err) return } }(t) //log.Debug("[downloader] downloaded both seg and torrent files", "hash", infoHash) - return false, nil + return nil } func seedableFiles(snapDir string) ([]string, error) { diff --git a/downloader/downloader_grpc_server.go b/downloader/downloader_grpc_server.go index 812fe9648..9dfc524c8 100644 --- a/downloader/downloader_grpc_server.go +++ b/downloader/downloader_grpc_server.go @@ -77,7 +77,7 @@ func (s *GrpcServer) Download(ctx context.Context, request *proto_downloader.Dow continue } - _, err := s.d.createMagnetLinkWithInfoHash(ctx, it.TorrentHash, it.Path, snapDir) + err := s.d.AddInfoHashAsMagnetLink(ctx, Proto2InfoHash(it.TorrentHash), it.Path) if err != nil { return nil, err } diff --git a/downloader/snaptype/files.go b/downloader/snaptype/files.go index 01b0cc95b..43b0a9392 100644 --- a/downloader/snaptype/files.go +++ b/downloader/snaptype/files.go @@ -17,8 +17,10 @@ package snaptype import ( + "encoding/hex" "errors" "fmt" + "github.com/anacrolix/torrent/metainfo" "os" "path/filepath" "strconv" @@ -225,3 +227,12 @@ func ParseDir(dir string) (res []FileInfo, err error) { return res, nil } + +func Hex2InfoHash(in string) (infoHash metainfo.Hash) { + inHex, err := hex.DecodeString(in) + if err != nil { + panic(err) + } + copy(infoHash[:], inHex) + return infoHash +} From 992036de162043dd8b855482e56f0ff57c56a3dc Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 14 Sep 2023 16:58:41 +0700 Subject: [PATCH 12/14] save --- state/domain_shared.go | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/state/domain_shared.go b/state/domain_shared.go index 5861f9b64..d7a4ef152 100644 --- a/state/domain_shared.go +++ b/state/domain_shared.go @@ -570,7 +570,8 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func sd.Storage.stats.FilesQueries.Add(1) var cp CursorHeap - heap.Init(&cp) + cpPtr := &cp + heap.Init(cpPtr) var k, v []byte var err error @@ -581,7 +582,7 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func k = []byte(kx) if len(kx) > 0 && bytes.HasPrefix(k, prefix) { - heap.Push(&cp, &CursorItem{t: RAM_CURSOR, key: common.Copy(k), val: common.Copy(v), iter: iter, endTxNum: sd.txNum.Load(), reverse: true}) + heap.Push(cpPtr, &CursorItem{t: RAM_CURSOR, key: common.Copy(k), val: common.Copy(v), iter: iter, endTxNum: sd.txNum.Load(), reverse: true}) } } @@ -602,7 +603,7 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func if v, err = roTx.GetOne(sd.Storage.valsTable, keySuffix); err != nil { return err } - heap.Push(&cp, &CursorItem{t: DB_CURSOR, key: k, val: v, c: keysCursor, endTxNum: txNum, reverse: true}) + heap.Push(cpPtr, &CursorItem{t: DB_CURSOR, key: k, val: v, c: keysCursor, endTxNum: txNum, reverse: true}) } sctx := sd.aggCtx.storage @@ -620,7 +621,7 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func key := cursor.Key() if key != nil && bytes.HasPrefix(key, prefix) { val := cursor.Value() - heap.Push(&cp, &CursorItem{t: FILE_CURSOR, key: key, val: val, btCursor: cursor, endTxNum: item.endTxNum, reverse: true}) + heap.Push(cpPtr, &CursorItem{t: FILE_CURSOR, key: key, val: val, btCursor: cursor, endTxNum: item.endTxNum, reverse: true}) } } @@ -629,7 +630,7 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func lastVal := common.Copy(cp[0].val) // Advance all the items that have this key (including the top) for cp.Len() > 0 && bytes.Equal(cp[0].key, lastKey) { - ci1 := heap.Pop(&cp).(*CursorItem) + ci1 := heap.Pop(cpPtr).(*CursorItem) switch ci1.t { case RAM_CURSOR: if ci1.iter.Next() { @@ -637,7 +638,7 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func if k != nil && bytes.HasPrefix(k, prefix) { ci1.key = common.Copy(k) ci1.val = common.Copy(ci1.iter.Value()) - heap.Push(&cp, ci1) + heap.Push(cpPtr, ci1) } } case FILE_CURSOR: @@ -646,7 +647,7 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func ci1.key = ci1.btCursor.Key() if ci1.key != nil && bytes.HasPrefix(ci1.key, prefix) { ci1.val = ci1.btCursor.Value() - heap.Push(&cp, ci1) + heap.Push(cpPtr, ci1) } } } else { @@ -658,7 +659,7 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func if key != nil && bytes.HasPrefix(key, prefix) { ci1.key = key ci1.val, ci1.latestOffset = ci1.dg.Next(nil) - heap.Push(&cp, ci1) + heap.Push(cpPtr, ci1) } } case DB_CURSOR: @@ -676,7 +677,7 @@ func (sd *SharedDomains) IterateStoragePrefix(roTx kv.Tx, prefix []byte, it func return err } ci1.val = common.Copy(v) - heap.Push(&cp, ci1) + heap.Push(cpPtr, ci1) } } } From 61e149f8c061fa4452f3abdf3ec4042e4c3af7a6 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 15 Sep 2023 12:43:25 +0700 Subject: [PATCH 13/14] save --- downloader/downloader.go | 135 +++++++++++++++++++-------------------- go.mod | 6 +- go.sum | 14 ++-- 3 files changed, 75 insertions(+), 80 deletions(-) diff --git a/downloader/downloader.go b/downloader/downloader.go index 88fe35e03..6d55c7be3 100644 --- a/downloader/downloader.go +++ b/downloader/downloader.go @@ -41,6 +41,7 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/log/v3" "github.com/pelletier/go-toml/v2" + "golang.org/x/exp/maps" "golang.org/x/sync/errgroup" "golang.org/x/sync/semaphore" ) @@ -157,91 +158,85 @@ func (d *Downloader) mainLoop(ctx context.Context, silent bool) error { go func() { defer d.wg.Done() - // 2 loops: 1-st waiting for "torrents resolution" (receiving metadata from trackers) - // Torrents that are already taken care of torrentMap := map[metainfo.Hash]struct{}{} // First loop drops torrents that were downloaded or are already complete // This improves efficiency of download by reducing number of active torrent (empirical observation) - DownloadLoop: - torrents := d.Torrent().Torrents() - for _, t := range torrents { - if _, already := torrentMap[t.InfoHash()]; already { - continue - } - select { - case <-ctx.Done(): - return - case <-t.GotInfo(): - } - if t.Complete.Bool() { - atomic.AddUint64(&d.stats.DroppedCompleted, uint64(t.BytesCompleted())) - atomic.AddUint64(&d.stats.DroppedTotal, uint64(t.Length())) - //t.Drop() - torrentMap[t.InfoHash()] = struct{}{} - continue - } - if err := sem.Acquire(ctx, 1); err != nil { - return - } - t.AllowDataDownload() - t.DownloadAll() - torrentMap[t.InfoHash()] = struct{}{} - d.wg.Add(1) - go func(t *torrent.Torrent) { - defer d.wg.Done() - defer sem.Release(1) + for torrents := d.Torrent().Torrents(); len(torrents) > 0; torrents = d.Torrent().Torrents() { + for _, t := range torrents { + if _, already := torrentMap[t.InfoHash()]; already { + continue + } select { case <-ctx.Done(): return - case <-t.Complete.On(): + case <-t.GotInfo(): } - atomic.AddUint64(&d.stats.DroppedCompleted, uint64(t.BytesCompleted())) - atomic.AddUint64(&d.stats.DroppedTotal, uint64(t.Length())) - //t.Drop() - }(t) - } - if len(torrents) != len(d.Torrent().Torrents()) { //if amount of torrents changed - keep downloading - goto DownloadLoop + if t.Complete.Bool() { + atomic.AddUint64(&d.stats.DroppedCompleted, uint64(t.BytesCompleted())) + atomic.AddUint64(&d.stats.DroppedTotal, uint64(t.Length())) + t.Drop() + torrentMap[t.InfoHash()] = struct{}{} + continue + } + if err := sem.Acquire(ctx, 1); err != nil { + return + } + t.AllowDataDownload() + t.DownloadAll() + torrentMap[t.InfoHash()] = struct{}{} + d.wg.Add(1) + go func(t *torrent.Torrent) { + defer d.wg.Done() + defer sem.Release(1) + select { + case <-ctx.Done(): + return + case <-t.Complete.On(): + } + atomic.AddUint64(&d.stats.DroppedCompleted, uint64(t.BytesCompleted())) + atomic.AddUint64(&d.stats.DroppedTotal, uint64(t.Length())) + t.Drop() + }(t) + } } - atomic.StoreUint64(&d.stats.DroppedCompleted, 0) atomic.StoreUint64(&d.stats.DroppedTotal, 0) - DownloadLoop2: - torrents = d.Torrent().Torrents() - for _, t := range torrents { - if _, already := torrentMap[t.InfoHash()]; already { - continue - } - select { - case <-ctx.Done(): - return - case <-t.GotInfo(): - } - if t.Complete.Bool() { - //t.Drop() - torrentMap[t.InfoHash()] = struct{}{} - continue - } - if err := sem.Acquire(ctx, 1); err != nil { - return - } - t.AllowDataDownload() - t.DownloadAll() - torrentMap[t.InfoHash()] = struct{}{} - d.wg.Add(1) - go func(t *torrent.Torrent) { - defer d.wg.Done() - defer sem.Release(1) + d.addSegments(ctx) + maps.Clear(torrentMap) + for { + torrents := d.Torrent().Torrents() + for _, t := range torrents { + if _, already := torrentMap[t.InfoHash()]; already { + continue + } select { case <-ctx.Done(): return - case <-t.Complete.On(): + case <-t.GotInfo(): } - }(t) - } - if len(torrents) != len(d.Torrent().Torrents()) { //if amount of torrents changed - keep downloading - goto DownloadLoop2 + if t.Complete.Bool() { + torrentMap[t.InfoHash()] = struct{}{} + continue + } + if err := sem.Acquire(ctx, 1); err != nil { + return + } + t.AllowDataDownload() + t.DownloadAll() + torrentMap[t.InfoHash()] = struct{}{} + d.wg.Add(1) + go func(t *torrent.Torrent) { + defer d.wg.Done() + defer sem.Release(1) + select { + case <-ctx.Done(): + return + case <-t.Complete.On(): + } + }(t) + } + time.Sleep(10 * time.Second) } }() diff --git a/go.mod b/go.mod index a99a43743..b10c4d26b 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/anacrolix/dht/v2 v2.19.2-0.20221121215055-066ad8494444 github.com/anacrolix/go-libutp v1.3.1 github.com/anacrolix/log v0.14.3-0.20230823030427-4b296d71a6b4 - github.com/anacrolix/torrent v1.52.6-0.20230911001013-87f6cdc1e96f + github.com/anacrolix/torrent v1.52.6-0.20230914125831-4fb12d06b31b github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b github.com/crate-crypto/go-kzg-4844 v0.3.0 github.com/deckarep/golang-set/v2 v2.3.1 @@ -37,7 +37,7 @@ require ( golang.org/x/sync v0.3.0 golang.org/x/sys v0.12.0 golang.org/x/time v0.3.0 - google.golang.org/grpc v1.58.0 + google.golang.org/grpc v1.58.1 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.31.0 ) @@ -65,7 +65,7 @@ require ( github.com/consensys/gnark-crypto v0.10.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/dustin/go-humanize v1.0.0 // indirect - github.com/go-llsqlite/adapter v0.0.0-20230910110622-f955011c1e41 // indirect + github.com/go-llsqlite/adapter v0.0.0-20230912124304-94ed0e573c23 // indirect github.com/go-llsqlite/crawshaw v0.0.0-20230910110433-7e901377eb6c // indirect github.com/go-logr/logr v1.2.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index f37f81e07..9a4f689d4 100644 --- a/go.sum +++ b/go.sum @@ -73,8 +73,8 @@ github.com/anacrolix/sync v0.4.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DC github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= -github.com/anacrolix/torrent v1.52.6-0.20230911001013-87f6cdc1e96f h1:KGyzNz+MiH/7gbEMz18x4I2YmWIF068qKIaZP/cfcuM= -github.com/anacrolix/torrent v1.52.6-0.20230911001013-87f6cdc1e96f/go.mod h1:U1BtbBNsjLeOGIukQaqXV5OqjOwkHaaWKFUThinxkE0= +github.com/anacrolix/torrent v1.52.6-0.20230914125831-4fb12d06b31b h1:Asaf/ETwCIEIYya0+oX2ZCIhHsV6Zt77VGHCP82fchA= +github.com/anacrolix/torrent v1.52.6-0.20230914125831-4fb12d06b31b/go.mod h1:6lKyJNzkkY68p+LeSfv62auyyceWn12Uji+kme5cpaI= github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 h1:QAVZ3pN/J4/UziniAhJR2OZ9Ox5kOY2053tBbbqUPYA= github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96/go.mod h1:Wa6n8cYIdaG35x15aH3Zy6d03f7P728QfdcDeD/IEOs= github.com/anacrolix/utp v0.1.0 h1:FOpQOmIwYsnENnz7tAGohA+r6iXpRjrq8ssKSre2Cp4= @@ -130,7 +130,7 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/erigontech/mdbx-go v0.27.14 h1:IVVeQVCAjZRpAR8bThlP2ISxrOwdV35NZdGwAgotaRw= github.com/erigontech/mdbx-go v0.27.14/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= @@ -142,8 +142,8 @@ github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1T github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-llsqlite/adapter v0.0.0-20230910110622-f955011c1e41 h1:1Us730PRZRfoXzS6fVIog/HNtQwZbMWfHgHfIPOuRuo= -github.com/go-llsqlite/adapter v0.0.0-20230910110622-f955011c1e41/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= +github.com/go-llsqlite/adapter v0.0.0-20230912124304-94ed0e573c23 h1:7krbnPREaxbmEaAkZovTNCMjmiZXEy/Gz9isFbqFK0I= +github.com/go-llsqlite/adapter v0.0.0-20230912124304-94ed0e573c23/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= github.com/go-llsqlite/crawshaw v0.0.0-20230910110433-7e901377eb6c h1:pm7z8uwA2q3s8fAsJmKuGckNohqIrw2PRtv6yJ6z0Ro= github.com/go-llsqlite/crawshaw v0.0.0-20230910110433-7e901377eb6c/go.mod h1:UdTSzmN3nr5dJNuZCsbPLfhSQB76u16rWh8pn+WFx9Q= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -568,8 +568,8 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= -google.golang.org/grpc v1.58.0/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58= +google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= From 13118be82edbfb292568a37dd28239b35f3c0254 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 15 Sep 2023 12:44:15 +0700 Subject: [PATCH 14/14] save --- tools/licenses_check.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/licenses_check.sh b/tools/licenses_check.sh index 31d76e634..cde94ebc1 100755 --- a/tools/licenses_check.sh +++ b/tools/licenses_check.sh @@ -24,6 +24,7 @@ export GOFLAGS="-tags=gorules,linux,tools" output=$(find "$projectDir" -type 'd' -maxdepth 1 \ -not -name ".*" \ -not -name tools \ + -not -name build \ | xargs go-licenses report 2>&1 \ `# exceptions` \ | grep -v "erigon-lib has empty version" `# self` \