From 6915b5e5ebddbdccdac6faa08408deaed119927d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 21 Dec 2023 10:14:26 +0700 Subject: [PATCH 1/4] save --- erigon-lib/downloader/downloader.go | 7 ++-- erigon-lib/downloader/util.go | 61 +++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 20 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 33cb15da28a..3993dc1bb2e 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -582,7 +582,7 @@ func (d *Downloader) AddNewSeedableFile(ctx context.Context, name string) error if err != nil { return fmt.Errorf("AddNewSeedableFile: %w", err) } - err = addTorrentFile(ctx, ts, d.torrentClient, d.webseeds) + _, err = addTorrentFile(ctx, ts, d.torrentClient, d.webseeds) if err != nil { return fmt.Errorf("addTorrentFile: %w", err) } @@ -619,8 +619,7 @@ func (d *Downloader) AddMagnetLink(ctx context.Context, infoHash metainfo.Hash, if err != nil { return err } - spec.DisallowDataDownload = true - t, _, err := d.torrentClient.AddTorrentSpec(spec) + t, err := addTorrentFile(ctx, spec, d.torrentClient, d.webseeds) if err != nil { return err } @@ -672,7 +671,7 @@ func (d *Downloader) addTorrentFilesFromDisk(quiet bool) error { return err } for i, ts := range files { - err := addTorrentFile(d.ctx, ts, d.torrentClient, d.webseeds) + _, err := addTorrentFile(d.ctx, ts, d.torrentClient, d.webseeds) if err != nil { return err } diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go index 053e830c851..b6df527b98f 100644 --- a/erigon-lib/downloader/util.go +++ b/erigon-lib/downloader/util.go @@ -306,29 +306,58 @@ func loadTorrent(torrentFilePath string) (*torrent.TorrentSpec, error) { // added first time - pieces verification process will start (disk IO heavy) - Progress // kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again. // Don't need call torrent.VerifyData manually -func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) error { +func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) (t *torrent.Torrent, err error) { + ts.ChunkSize = downloadercfg.DefaultNetworkChunkSize + ts.DisallowDataDownload = true + ts.DisableInitialPieceCheck = true + defer func() { + rec := recover() //re-try on panic + if rec != nil { + ts.ChunkSize = 0 + t, err = _addTorrentFile(ctx, ts, torrentClient, webseeds) + } + }() + + t, err = _addTorrentFile(ctx, ts, torrentClient, webseeds) + if err != nil { + ts.ChunkSize = 0 + return _addTorrentFile(ctx, ts, torrentClient, webseeds) + } + return t, err +} + +func _addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) (t *torrent.Torrent, err error) { select { case <-ctx.Done(): - return ctx.Err() + return nil, ctx.Err() default: } - wsUrls, ok := webseeds.ByFileName(ts.DisplayName) - if ok { - ts.Webseeds = append(ts.Webseeds, wsUrls...) - } - _, ok = torrentClient.Torrent(ts.InfoHash) - if !ok { // can set ChunkSize only for new torrents - ts.ChunkSize = downloadercfg.DefaultNetworkChunkSize - } else { - ts.ChunkSize = 0 + ts.Webseeds, _ = webseeds.ByFileName(ts.DisplayName) + var ok bool + t, ok = torrentClient.Torrent(ts.InfoHash) + if !ok { + defer func(t time.Time) { fmt.Printf("util.go:336: %s\n", time.Since(t)) }(time.Now()) + t, _, err := torrentClient.AddTorrentSpec(ts) + if err != nil { + return t, fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err) + } + return t, nil } - ts.DisallowDataDownload = true - _, _, err := torrentClient.AddTorrentSpec(ts) - if err != nil { - return fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err) + + select { + case <-t.GotInfo(): + defer func(t time.Time) { fmt.Printf("util.go:350: %s\n", time.Since(t)) }(time.Now()) + t.AddWebSeeds(ts.Webseeds) + default: + defer func(t time.Time) { fmt.Printf("util.go:353: %s\n", time.Since(t)) }(time.Now()) + t, _, err = torrentClient.AddTorrentSpec(ts) + if err != nil { + return t, fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err) + } } - return nil + + return t, nil } func savePeerID(db kv.RwDB, peerID torrent.PeerID) error { From a34275bef6af22fa6d2e92452d0f7e2cc1d4a6db Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 21 Dec 2023 10:17:31 +0700 Subject: [PATCH 2/4] save --- erigon-lib/downloader/util.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go index b6df527b98f..69a767add1d 100644 --- a/erigon-lib/downloader/util.go +++ b/erigon-lib/downloader/util.go @@ -310,8 +310,9 @@ func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient ts.ChunkSize = downloadercfg.DefaultNetworkChunkSize ts.DisallowDataDownload = true ts.DisableInitialPieceCheck = true + //re-try on panic, with 0 ChunkSize (lib doesn't allow change this field for existing torrents) defer func() { - rec := recover() //re-try on panic + rec := recover() if rec != nil { ts.ChunkSize = 0 t, err = _addTorrentFile(ctx, ts, torrentClient, webseeds) From a0bfe61883fe7132dd4b16ed85342ccba6c5d565 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 21 Dec 2023 10:27:12 +0700 Subject: [PATCH 3/4] merge devel --- erigon-lib/downloader/downloader.go | 9 ++++++--- erigon-lib/downloader/util.go | 26 +++++++++++++------------- 2 files changed, 19 insertions(+), 16 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 3993dc1bb2e..2a438cc2432 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -582,7 +582,7 @@ func (d *Downloader) AddNewSeedableFile(ctx context.Context, name string) error if err != nil { return fmt.Errorf("AddNewSeedableFile: %w", err) } - _, err = addTorrentFile(ctx, ts, d.torrentClient, d.webseeds) + _, _, err = addTorrentFile(ctx, ts, d.torrentClient, d.webseeds) if err != nil { return fmt.Errorf("addTorrentFile: %w", err) } @@ -619,10 +619,13 @@ func (d *Downloader) AddMagnetLink(ctx context.Context, infoHash metainfo.Hash, if err != nil { return err } - t, err := addTorrentFile(ctx, spec, d.torrentClient, d.webseeds) + t, ok, err := addTorrentFile(ctx, spec, d.torrentClient, d.webseeds) if err != nil { return err } + if !ok { + return nil + } d.wg.Add(1) go func(t *torrent.Torrent) { defer d.wg.Done() @@ -671,7 +674,7 @@ func (d *Downloader) addTorrentFilesFromDisk(quiet bool) error { return err } for i, ts := range files { - _, err := addTorrentFile(d.ctx, ts, d.torrentClient, d.webseeds) + _, _, err := addTorrentFile(d.ctx, ts, d.torrentClient, d.webseeds) if err != nil { return err } diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go index 69a767add1d..f3ed5473df8 100644 --- a/erigon-lib/downloader/util.go +++ b/erigon-lib/downloader/util.go @@ -306,7 +306,7 @@ func loadTorrent(torrentFilePath string) (*torrent.TorrentSpec, error) { // added first time - pieces verification process will start (disk IO heavy) - Progress // kept in `piece completion storage` (surviving reboot). Once it done - no disk IO needed again. // Don't need call torrent.VerifyData manually -func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) (t *torrent.Torrent, err error) { +func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) (t *torrent.Torrent, ok bool, err error) { ts.ChunkSize = downloadercfg.DefaultNetworkChunkSize ts.DisallowDataDownload = true ts.DisableInitialPieceCheck = true @@ -315,35 +315,35 @@ func addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient rec := recover() if rec != nil { ts.ChunkSize = 0 - t, err = _addTorrentFile(ctx, ts, torrentClient, webseeds) + t, ok, err = _addTorrentFile(ctx, ts, torrentClient, webseeds) } }() - t, err = _addTorrentFile(ctx, ts, torrentClient, webseeds) + t, ok, err = _addTorrentFile(ctx, ts, torrentClient, webseeds) if err != nil { ts.ChunkSize = 0 return _addTorrentFile(ctx, ts, torrentClient, webseeds) } - return t, err + return t, ok, err } -func _addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) (t *torrent.Torrent, err error) { +func _addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient *torrent.Client, webseeds *WebSeeds) (t *torrent.Torrent, ok bool, err error) { select { case <-ctx.Done(): - return nil, ctx.Err() + return nil, false, ctx.Err() default: } ts.Webseeds, _ = webseeds.ByFileName(ts.DisplayName) - var ok bool - t, ok = torrentClient.Torrent(ts.InfoHash) - if !ok { + var have bool + t, have = torrentClient.Torrent(ts.InfoHash) + if !have { defer func(t time.Time) { fmt.Printf("util.go:336: %s\n", time.Since(t)) }(time.Now()) t, _, err := torrentClient.AddTorrentSpec(ts) if err != nil { - return t, fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err) + return nil, false, fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err) } - return t, nil + return t, true, nil } select { @@ -354,11 +354,11 @@ func _addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient defer func(t time.Time) { fmt.Printf("util.go:353: %s\n", time.Since(t)) }(time.Now()) t, _, err = torrentClient.AddTorrentSpec(ts) if err != nil { - return t, fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err) + return nil, false, fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err) } } - return t, nil + return t, true, nil } func savePeerID(db kv.RwDB, peerID torrent.PeerID) error { From e090b7b4456d7f13c08993fe0121e5cb5adb2f08 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 21 Dec 2023 10:36:09 +0700 Subject: [PATCH 4/4] save --- erigon-lib/downloader/util.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/erigon-lib/downloader/util.go b/erigon-lib/downloader/util.go index f3ed5473df8..4b2d3d3470f 100644 --- a/erigon-lib/downloader/util.go +++ b/erigon-lib/downloader/util.go @@ -338,7 +338,6 @@ func _addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient var have bool t, have = torrentClient.Torrent(ts.InfoHash) if !have { - defer func(t time.Time) { fmt.Printf("util.go:336: %s\n", time.Since(t)) }(time.Now()) t, _, err := torrentClient.AddTorrentSpec(ts) if err != nil { return nil, false, fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err) @@ -348,10 +347,8 @@ func _addTorrentFile(ctx context.Context, ts *torrent.TorrentSpec, torrentClient select { case <-t.GotInfo(): - defer func(t time.Time) { fmt.Printf("util.go:350: %s\n", time.Since(t)) }(time.Now()) t.AddWebSeeds(ts.Webseeds) default: - defer func(t time.Time) { fmt.Printf("util.go:353: %s\n", time.Since(t)) }(time.Now()) t, _, err = torrentClient.AddTorrentSpec(ts) if err != nil { return nil, false, fmt.Errorf("addTorrentFile %s: %w", ts.DisplayName, err)