From 69bbe69db9a87e53322b5b2acf553561045496f2 Mon Sep 17 00:00:00 2001 From: dvovk Date: Fri, 1 Dec 2023 15:32:39 +0100 Subject: [PATCH 01/11] added grabbing per file snapshot downloading --- diagnostics/diagnostic.go | 26 ++++++++++++++++++++++++++ erigon-lib/diagnostics/entities.go | 12 ++++++++++++ erigon-lib/downloader/downloader.go | 14 ++++++++++++++ 3 files changed, 52 insertions(+) diff --git a/diagnostics/diagnostic.go b/diagnostics/diagnostic.go index c8013ba90cd..31d1859cf47 100644 --- a/diagnostics/diagnostic.go +++ b/diagnostics/diagnostic.go @@ -2,6 +2,7 @@ package diagnostics import ( "context" + "fmt" "net/http" "github.com/ledgerwatch/erigon-lib/common" @@ -25,6 +26,7 @@ func NewDiagnosticClient(ctx *cli.Context, metricsMux *http.ServeMux, node *node func (d *DiagnosticClient) Setup() { d.runSnapshotListener() + d.runTorrentListener() } func (d *DiagnosticClient) runSnapshotListener() { @@ -54,3 +56,27 @@ func (d *DiagnosticClient) runSnapshotListener() { func (d *DiagnosticClient) SnapshotDownload() map[string]diaglib.DownloadStatistics { return d.snapshotDownload } + +func (d *DiagnosticClient) runTorrentListener() { + go func() { + ctx, ch, cancel := diaglib.Context[diaglib.TorrentFile](context.Background(), 1) + defer cancel() + + rootCtx, _ := common.RootContext() + + diaglib.StartProviders(ctx, diaglib.TypeOf(diaglib.TorrentFile{}), log.Root()) + for { + select { + case <-rootCtx.Done(): + cancel() + return + case info := <-ch: + fmt.Println("INFO", info) + //d.snapshotDownload[info.StagePrefix] = info + //if info.DownloadFinished { + // return + //} + } + } + }() +} diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index d8b8172fa14..369f9463c86 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -44,6 +44,18 @@ type DownloadStatistics struct { StagePrefix string `json:"stagePrefix"` } +type TorrentFile struct { + Name string `json:"name"` + TotalBytes uint64 `json:"totalBytes"` + DownloadedBytes uint64 `json:"downloadedBytes"` + SeedsCount int `json:"seedsCount"` + PeersCount int `json:"peersCount"` +} + func (ti DownloadStatistics) Type() Type { return TypeOf(ti) } + +func (ti TorrentFile) Type() Type { + return TypeOf(ti) +} diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 845c3cf11d2..f32745d50f3 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -33,6 +33,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dbg" + "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon-lib/downloader/downloadercfg" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/mdbx" @@ -115,6 +116,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, dirs datadir.Dirs, logger if err := d.addTorrentFilesFromDisk(false); err != nil { return nil, err } + // CornerCase: no peers -> no anoncments to trackers -> no magnetlink resolution (but magnetlink has filename) // means we can start adding weebseeds without waiting for `<-t.GotInfo()` d.wg.Add(1) @@ -342,6 +344,15 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { for _, peer := range t.PeerConns() { peersOfThisFile[peer.PeerID] = struct{}{} } + + diagnostics.Send(diagnostics.TorrentFile{ + Name: t.Name(), + TotalBytes: uint64(t.Length()), + DownloadedBytes: uint64(t.BytesCompleted()), + SeedsCount: len(t.Metainfo().UrlList), + PeersCount: len(peersOfThisFile), + }) + d.logger.Log(d.verbosity, "[snapshots] progress", "name", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "webseeds", len(t.Metainfo().UrlList), "peers", len(peersOfThisFile)) } } @@ -351,6 +362,9 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { stats.Completed = stats.Completed && t.Complete.Bool() } + //fmt.Println("noMetadata", noMetadata) + //fmt.Println("zeroProgress", zeroProgress) + if len(noMetadata) > 0 { amount := len(noMetadata) if len(noMetadata) > 5 { From c610cd7062888f6c664ee739a7264e0af4df5d04 Mon Sep 17 00:00:00 2001 From: dvovk Date: Wed, 6 Dec 2023 15:36:23 +0100 Subject: [PATCH 02/11] added by file grabbing data --- diagnostics/diagnostic.go | 12 ++++++------ erigon-lib/downloader/downloader.go | 2 -- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/diagnostics/diagnostic.go b/diagnostics/diagnostic.go index 31d1859cf47..b40c6a5012f 100644 --- a/diagnostics/diagnostic.go +++ b/diagnostics/diagnostic.go @@ -2,7 +2,6 @@ package diagnostics import ( "context" - "fmt" "net/http" "github.com/ledgerwatch/erigon-lib/common" @@ -18,6 +17,7 @@ type DiagnosticClient struct { node *node.ErigonNode snapshotDownload map[string]diaglib.DownloadStatistics + fileDownload map[string]diaglib.TorrentFile } func NewDiagnosticClient(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) *DiagnosticClient { @@ -71,12 +71,12 @@ func (d *DiagnosticClient) runTorrentListener() { cancel() return case info := <-ch: - fmt.Println("INFO", info) - //d.snapshotDownload[info.StagePrefix] = info - //if info.DownloadFinished { - // return - //} + d.fileDownload[info.Name] = info } } }() } + +func (d *DiagnosticClient) TorrentFileDownload() map[string]diaglib.TorrentFile { + return d.fileDownload +} diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index f32745d50f3..63101a0c01d 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -362,8 +362,6 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { stats.Completed = stats.Completed && t.Complete.Bool() } - //fmt.Println("noMetadata", noMetadata) - //fmt.Println("zeroProgress", zeroProgress) if len(noMetadata) > 0 { amount := len(noMetadata) From 170ce49b2cc5c099d78e4aae3465135ef7a9bf36 Mon Sep 17 00:00:00 2001 From: dvovk Date: Thu, 7 Dec 2023 12:13:17 +0100 Subject: [PATCH 03/11] enable metrics --- erigon-lib/diagnostics/provider.go | 2 ++ erigon-lib/downloader/downloader.go | 18 ++++++++++-------- turbo/debug/flags.go | 3 +++ 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/erigon-lib/diagnostics/provider.go b/erigon-lib/diagnostics/provider.go index ef9b3f045f5..5b199515099 100644 --- a/erigon-lib/diagnostics/provider.go +++ b/erigon-lib/diagnostics/provider.go @@ -14,6 +14,8 @@ import ( type ctxKey int +var MetricsEnabled = false + const ( ckChan ctxKey = iota ) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index e17018d389a..607dc5447b9 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -349,6 +349,16 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { break //of switch } + if diagnostics.MetricsEnabled { + diagnostics.Send(diagnostics.TorrentFile{ + Name: t.Name(), + TotalBytes: uint64(t.Length()), + DownloadedBytes: uint64(t.BytesCompleted()), + SeedsCount: len(t.Metainfo().UrlList), + PeersCount: len(peersOfThisFile), + }) + } + d.logger.Log(d.verbosity, "[snapshots] progress", "file", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "peers", len(peersOfThisFile), "webseeds", len(weebseedPeersOfThisFile)) if d.verbosity < log.LvlInfo { break //of switch @@ -363,14 +373,6 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { webseedRates = append(webseedRates, shortUrl, fmt.Sprintf("%s/s", common.ByteCount(uint64(peer.DownloadRate())))) } - diagnostics.Send(diagnostics.TorrentFile{ - Name: t.Name(), - TotalBytes: uint64(t.Length()), - DownloadedBytes: uint64(t.BytesCompleted()), - SeedsCount: len(t.Metainfo().UrlList), - PeersCount: len(peersOfThisFile), - }) - d.logger.Log(d.verbosity, "[snapshots] progress", "name", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "webseeds", len(t.Metainfo().UrlList), "peers", len(peersOfThisFile)) } } diff --git a/turbo/debug/flags.go b/turbo/debug/flags.go index 1be6efa51b3..ca58f5d1e50 100644 --- a/turbo/debug/flags.go +++ b/turbo/debug/flags.go @@ -24,6 +24,7 @@ import ( "os" "path/filepath" + "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon-lib/metrics" "github.com/ledgerwatch/log/v3" @@ -204,6 +205,8 @@ func Setup(ctx *cli.Context, rootLogger bool) (log.Logger, *http.ServeMux, error var metricsMux *http.ServeMux var metricsAddress string + diagnostics.MetricsEnabled = metricsEnabled + if metricsEnabled && (!pprofEnabled || metricsAddr != "") { metricsPort := ctx.Int(metricsPortFlag.Name) metricsAddress = fmt.Sprintf("%s:%d", metricsAddr, metricsPort) From c0b450db4fd912c4e9e7c351059c94c426a835de Mon Sep 17 00:00:00 2001 From: dvovk Date: Thu, 7 Dec 2023 17:01:05 +0100 Subject: [PATCH 04/11] updated check is diagnostic connected --- diagnostics/diagnostic.go | 4 ++ erigon-lib/diagnostics/entities.go | 2 + erigon-lib/diagnostics/provider.go | 7 ++- erigon-lib/downloader/downloader.go | 76 ++++++++++++++++++----------- turbo/debug/flags.go | 3 -- 5 files changed, 59 insertions(+), 33 deletions(-) diff --git a/diagnostics/diagnostic.go b/diagnostics/diagnostic.go index 6e7ae17cded..a768aaabe0d 100644 --- a/diagnostics/diagnostic.go +++ b/diagnostics/diagnostic.go @@ -2,6 +2,7 @@ package diagnostics import ( "context" + "fmt" "net/http" "github.com/ledgerwatch/erigon-lib/common" @@ -71,6 +72,9 @@ func (d *DiagnosticClient) runTorrentListener() { cancel() return case info := <-ch: + fmt.Println("info.Name", info.Name) + fmt.Println("info.peers", info.PeersCount, "speed", info.PeersRate) + fmt.Println("info.seeds", info.SeedsCount, "speed", info.SeedsRate) d.fileDownload[info.Name] = info } } diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 369f9463c86..210fc04852f 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -50,6 +50,8 @@ type TorrentFile struct { DownloadedBytes uint64 `json:"downloadedBytes"` SeedsCount int `json:"seedsCount"` PeersCount int `json:"peersCount"` + SeedsRate uint64 `json:"seedsRate"` + PeersRate uint64 `json:"peersRate"` } func (ti DownloadStatistics) Type() Type { diff --git a/erigon-lib/diagnostics/provider.go b/erigon-lib/diagnostics/provider.go index 5b199515099..1ff14ab4b03 100644 --- a/erigon-lib/diagnostics/provider.go +++ b/erigon-lib/diagnostics/provider.go @@ -14,8 +14,6 @@ import ( type ctxKey int -var MetricsEnabled = false - const ( ckChan ctxKey = iota ) @@ -24,6 +22,7 @@ type Type interface { reflect.Type Context() context.Context Err() error + Enabled() bool } type diagType struct { @@ -50,6 +49,10 @@ func (t diagType) Err() error { return t.Context().Err() } +func (t diagType) Enabled() bool { + return t.Err() == nil +} + type Info interface { Type() Type } diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 607dc5447b9..bb052c650b3 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -349,39 +349,59 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { break //of switch } - if diagnostics.MetricsEnabled { - diagnostics.Send(diagnostics.TorrentFile{ - Name: t.Name(), - TotalBytes: uint64(t.Length()), - DownloadedBytes: uint64(t.BytesCompleted()), - SeedsCount: len(t.Metainfo().UrlList), - PeersCount: len(peersOfThisFile), - }) - } - d.logger.Log(d.verbosity, "[snapshots] progress", "file", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "peers", len(peersOfThisFile), "webseeds", len(weebseedPeersOfThisFile)) - if d.verbosity < log.LvlInfo { - break //of switch - } - - // more detailed statistic: download rate of each peer (for each file) - webseedRates := make([]interface{}, 0, len(weebseedPeersOfThisFile)*2) - for _, peer := range weebseedPeersOfThisFile { - urlS := strings.Trim(strings.TrimPrefix(peer.String(), "webseed peer for "), "\"") - if urlObj, err := url.Parse(urlS); err == nil { - if shortUrl, err := url.JoinPath(urlObj.Host, urlObj.Path); err == nil { - webseedRates = append(webseedRates, shortUrl, fmt.Sprintf("%s/s", common.ByteCount(uint64(peer.DownloadRate())))) + isDiagEnabled := diagnostics.TypeOf(diagnostics.TorrentFile{}).Enabled() + if d.verbosity < log.LvlInfo || isDiagEnabled { + + // more detailed statistic: download rate of each peer (for each file) + seedsRates := uint64(0) + webseedRates := make([]interface{}, 0, len(weebseedPeersOfThisFile)*2) + for _, peer := range weebseedPeersOfThisFile { + urlS := strings.Trim(strings.TrimPrefix(peer.String(), "webseed peer for "), "\"") + if urlObj, err := url.Parse(urlS); err == nil { + if shortUrl, err := url.JoinPath(urlObj.Host, urlObj.Path); err == nil { + dr := uint64(peer.DownloadRate()) + webseedRates = append(webseedRates, shortUrl, fmt.Sprintf("%s/s", common.ByteCount(dr))) + seedsRates += dr + } + + d.logger.Log(d.verbosity, "[snapshots] progress", "name", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "webseeds", len(t.Metainfo().UrlList), "peers", len(peersOfThisFile)) } + } - d.logger.Log(d.verbosity, "[snapshots] progress", "name", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "webseeds", len(t.Metainfo().UrlList), "peers", len(peersOfThisFile)) + lenght := uint64(len(weebseedPeersOfThisFile)) + if lenght > 0 { + seedsRates = seedsRates / lenght + } + + d.logger.Info(fmt.Sprintf("[snapshots] webseed peers file=%s", t.Name()), webseedRates...) + rates := make([]interface{}, 0, len(peersOfThisFile)*2) + peersRates := uint64(0) + for _, peer := range peersOfThisFile { + dr := uint64(peer.DownloadRate()) + rates = append(rates, peer.PeerClientName.Load(), fmt.Sprintf("%s/s", common.ByteCount(dr))) + peersRates += dr + } + d.logger.Info(fmt.Sprintf("[snapshots] bittorrent peers file=%s", t.Name()), rates...) + + lenght = uint64(len(peersOfThisFile)) + if lenght > 0 { + peersRates = peersRates / uint64(len(peersOfThisFile)) + } + + if isDiagEnabled { + diagnostics.Send(diagnostics.TorrentFile{ + Name: t.Name(), + TotalBytes: uint64(t.Length()), + DownloadedBytes: uint64(t.BytesCompleted()), + SeedsCount: len(t.Metainfo().UrlList), + PeersCount: len(peersOfThisFile), + SeedsRate: uint64(seedsRates), + PeersRate: uint64(peersRates), + }) } } - d.logger.Info(fmt.Sprintf("[snapshots] webseed peers file=%s", t.Name()), webseedRates...) - rates := make([]interface{}, 0, len(peersOfThisFile)*2) - for _, peer := range peersOfThisFile { - rates = append(rates, peer.PeerClientName.Load(), fmt.Sprintf("%s/s", common.ByteCount(uint64(peer.DownloadRate())))) - } - d.logger.Info(fmt.Sprintf("[snapshots] bittorrent peers file=%s", t.Name()), rates...) + default: noMetadata = append(noMetadata, t.Name()) } diff --git a/turbo/debug/flags.go b/turbo/debug/flags.go index ca58f5d1e50..1be6efa51b3 100644 --- a/turbo/debug/flags.go +++ b/turbo/debug/flags.go @@ -24,7 +24,6 @@ import ( "os" "path/filepath" - "github.com/ledgerwatch/erigon-lib/diagnostics" "github.com/ledgerwatch/erigon-lib/metrics" "github.com/ledgerwatch/log/v3" @@ -205,8 +204,6 @@ func Setup(ctx *cli.Context, rootLogger bool) (log.Logger, *http.ServeMux, error var metricsMux *http.ServeMux var metricsAddress string - diagnostics.MetricsEnabled = metricsEnabled - if metricsEnabled && (!pprofEnabled || metricsAddr != "") { metricsPort := ctx.Int(metricsPortFlag.Name) metricsAddress = fmt.Sprintf("%s:%d", metricsAddr, metricsPort) From 04b1f0d099ea3b36f5f1b419b33116004d3798e9 Mon Sep 17 00:00:00 2001 From: dvovk Date: Thu, 7 Dec 2023 19:01:39 +0100 Subject: [PATCH 05/11] update --- diagnostics/diagnostic.go | 4 ---- erigon-lib/diagnostics/entities.go | 25 +++++++++++++------------ 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/diagnostics/diagnostic.go b/diagnostics/diagnostic.go index a768aaabe0d..6e7ae17cded 100644 --- a/diagnostics/diagnostic.go +++ b/diagnostics/diagnostic.go @@ -2,7 +2,6 @@ package diagnostics import ( "context" - "fmt" "net/http" "github.com/ledgerwatch/erigon-lib/common" @@ -72,9 +71,6 @@ func (d *DiagnosticClient) runTorrentListener() { cancel() return case info := <-ch: - fmt.Println("info.Name", info.Name) - fmt.Println("info.peers", info.PeersCount, "speed", info.PeersRate) - fmt.Println("info.seeds", info.SeedsCount, "speed", info.SeedsRate) d.fileDownload[info.Name] = info } } diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 210fc04852f..afc5629d3d4 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -30,18 +30,19 @@ type PeerStatistics struct { } type DownloadStatistics struct { - Downloaded uint64 `json:"downloaded"` - Total uint64 `json:"total"` - TotalTime float64 `json:"totalTime"` - DownloadRate uint64 `json:"downloadRate"` - UploadRate uint64 `json:"uploadRate"` - Peers int32 `json:"peers"` - Files int32 `json:"files"` - Connections uint64 `json:"connections"` - Alloc uint64 `json:"alloc"` - Sys uint64 `json:"sys"` - DownloadFinished bool `json:"downloadFinished"` - StagePrefix string `json:"stagePrefix"` + Downloaded uint64 `json:"downloaded"` + Total uint64 `json:"total"` + TotalTime float64 `json:"totalTime"` + DownloadRate uint64 `json:"downloadRate"` + UploadRate uint64 `json:"uploadRate"` + Peers int32 `json:"peers"` + Files int32 `json:"files"` + Connections uint64 `json:"connections"` + Alloc uint64 `json:"alloc"` + Sys uint64 `json:"sys"` + DownloadFinished bool `json:"downloadFinished"` + StagePrefix string `json:"stagePrefix"` + DownloadFiles map[string]TorrentFile `json:"downloadFiles"` } type TorrentFile struct { From 9dd85ab03fdbd34dd836b1d7d121aeb9ed422ddd Mon Sep 17 00:00:00 2001 From: dvovk Date: Thu, 7 Dec 2023 19:44:02 +0100 Subject: [PATCH 06/11] fix lint error --- erigon-lib/downloader/downloader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index bb052c650b3..76b833d5573 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -396,8 +396,8 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { DownloadedBytes: uint64(t.BytesCompleted()), SeedsCount: len(t.Metainfo().UrlList), PeersCount: len(peersOfThisFile), - SeedsRate: uint64(seedsRates), - PeersRate: uint64(peersRates), + SeedsRate: seedsRates, + PeersRate: peersRates, }) } } From e1997e12894ad4b734fa336b185340c9df5601fd Mon Sep 17 00:00:00 2001 From: dvovk Date: Thu, 7 Dec 2023 19:56:44 +0100 Subject: [PATCH 07/11] fix lint error --- erigon-lib/diagnostics/entities.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index afc5629d3d4..210fc04852f 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -30,19 +30,18 @@ type PeerStatistics struct { } type DownloadStatistics struct { - Downloaded uint64 `json:"downloaded"` - Total uint64 `json:"total"` - TotalTime float64 `json:"totalTime"` - DownloadRate uint64 `json:"downloadRate"` - UploadRate uint64 `json:"uploadRate"` - Peers int32 `json:"peers"` - Files int32 `json:"files"` - Connections uint64 `json:"connections"` - Alloc uint64 `json:"alloc"` - Sys uint64 `json:"sys"` - DownloadFinished bool `json:"downloadFinished"` - StagePrefix string `json:"stagePrefix"` - DownloadFiles map[string]TorrentFile `json:"downloadFiles"` + Downloaded uint64 `json:"downloaded"` + Total uint64 `json:"total"` + TotalTime float64 `json:"totalTime"` + DownloadRate uint64 `json:"downloadRate"` + UploadRate uint64 `json:"uploadRate"` + Peers int32 `json:"peers"` + Files int32 `json:"files"` + Connections uint64 `json:"connections"` + Alloc uint64 `json:"alloc"` + Sys uint64 `json:"sys"` + DownloadFinished bool `json:"downloadFinished"` + StagePrefix string `json:"stagePrefix"` } type TorrentFile struct { From 3b9a677c2b5513663270465da7391edb025be70e Mon Sep 17 00:00:00 2001 From: dvovk Date: Fri, 8 Dec 2023 08:01:20 +0100 Subject: [PATCH 08/11] fixed review --- erigon-lib/diagnostics/entities.go | 2 +- erigon-lib/downloader/downloader.go | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 210fc04852f..1255b9aae93 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -50,8 +50,8 @@ type TorrentFile struct { DownloadedBytes uint64 `json:"downloadedBytes"` SeedsCount int `json:"seedsCount"` PeersCount int `json:"peersCount"` + WebseedsRate uint64 `json:"webseedsRate"` SeedsRate uint64 `json:"seedsRate"` - PeersRate uint64 `json:"peersRate"` } func (ti DownloadStatistics) Type() Type { diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 76b833d5573..989108c11b3 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -354,7 +354,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { if d.verbosity < log.LvlInfo || isDiagEnabled { // more detailed statistic: download rate of each peer (for each file) - seedsRates := uint64(0) + websRates := uint64(0) webseedRates := make([]interface{}, 0, len(weebseedPeersOfThisFile)*2) for _, peer := range weebseedPeersOfThisFile { urlS := strings.Trim(strings.TrimPrefix(peer.String(), "webseed peer for "), "\"") @@ -362,7 +362,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { if shortUrl, err := url.JoinPath(urlObj.Host, urlObj.Path); err == nil { dr := uint64(peer.DownloadRate()) webseedRates = append(webseedRates, shortUrl, fmt.Sprintf("%s/s", common.ByteCount(dr))) - seedsRates += dr + websRates += dr } d.logger.Log(d.verbosity, "[snapshots] progress", "name", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "webseeds", len(t.Metainfo().UrlList), "peers", len(peersOfThisFile)) @@ -371,22 +371,22 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { lenght := uint64(len(weebseedPeersOfThisFile)) if lenght > 0 { - seedsRates = seedsRates / lenght + websRates = websRates / lenght } d.logger.Info(fmt.Sprintf("[snapshots] webseed peers file=%s", t.Name()), webseedRates...) rates := make([]interface{}, 0, len(peersOfThisFile)*2) - peersRates := uint64(0) + seedsRates := uint64(0) for _, peer := range peersOfThisFile { dr := uint64(peer.DownloadRate()) rates = append(rates, peer.PeerClientName.Load(), fmt.Sprintf("%s/s", common.ByteCount(dr))) - peersRates += dr + seedsRates += dr } d.logger.Info(fmt.Sprintf("[snapshots] bittorrent peers file=%s", t.Name()), rates...) lenght = uint64(len(peersOfThisFile)) if lenght > 0 { - peersRates = peersRates / uint64(len(peersOfThisFile)) + seedsRates = seedsRates / uint64(len(peersOfThisFile)) } if isDiagEnabled { @@ -396,8 +396,8 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { DownloadedBytes: uint64(t.BytesCompleted()), SeedsCount: len(t.Metainfo().UrlList), PeersCount: len(peersOfThisFile), + WebseedsRate: websRates, SeedsRate: seedsRates, - PeersRate: peersRates, }) } } From 6e16312ef4d10a7c0f5b8fe5686bee49447fd23f Mon Sep 17 00:00:00 2001 From: dvovk Date: Fri, 8 Dec 2023 12:05:45 +0100 Subject: [PATCH 09/11] updated snapshot download statistics --- diagnostics/diagnostic.go | 41 +++++++++++++++++++---------- erigon-lib/diagnostics/entities.go | 36 ++++++++++++------------- erigon-lib/downloader/downloader.go | 14 +++++----- turbo/snapshotsync/snapshotsync.go | 6 ++--- 4 files changed, 54 insertions(+), 43 deletions(-) diff --git a/diagnostics/diagnostic.go b/diagnostics/diagnostic.go index 6e7ae17cded..ff49297e84a 100644 --- a/diagnostics/diagnostic.go +++ b/diagnostics/diagnostic.go @@ -2,6 +2,7 @@ package diagnostics import ( "context" + "fmt" "net/http" "github.com/ledgerwatch/erigon-lib/common" @@ -16,12 +17,11 @@ type DiagnosticClient struct { metricsMux *http.ServeMux node *node.ErigonNode - snapshotDownload map[string]diaglib.DownloadStatistics - fileDownload map[string]diaglib.TorrentFile + snapshotDownload diaglib.SnapshotDownloadStatistics } func NewDiagnosticClient(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) *DiagnosticClient { - return &DiagnosticClient{ctx: ctx, metricsMux: metricsMux, node: node, snapshotDownload: map[string]diaglib.DownloadStatistics{}, fileDownload: map[string]diaglib.TorrentFile{}} + return &DiagnosticClient{ctx: ctx, metricsMux: metricsMux, node: node, snapshotDownload: diaglib.SnapshotDownloadStatistics{}} } func (d *DiagnosticClient) Setup() { @@ -31,19 +31,32 @@ func (d *DiagnosticClient) Setup() { func (d *DiagnosticClient) runSnapshotListener() { go func() { - ctx, ch, cancel := diaglib.Context[diaglib.DownloadStatistics](context.Background(), 1) + ctx, ch, cancel := diaglib.Context[diaglib.SnapshotDownloadStatistics](context.Background(), 1) defer cancel() rootCtx, _ := common.RootContext() - diaglib.StartProviders(ctx, diaglib.TypeOf(diaglib.DownloadStatistics{}), log.Root()) + diaglib.StartProviders(ctx, diaglib.TypeOf(diaglib.SnapshotDownloadStatistics{}), log.Root()) for { select { case <-rootCtx.Done(): cancel() return case info := <-ch: - d.snapshotDownload[info.StagePrefix] = info + d.snapshotDownload.Downloaded = info.Downloaded + d.snapshotDownload.Total = info.Total + d.snapshotDownload.TotalTime = info.TotalTime + d.snapshotDownload.DownloadRate = info.DownloadRate + d.snapshotDownload.UploadRate = info.UploadRate + d.snapshotDownload.Peers = info.Peers + d.snapshotDownload.Files = info.Files + d.snapshotDownload.Connections = info.Connections + d.snapshotDownload.Alloc = info.Alloc + d.snapshotDownload.Sys = info.Sys + d.snapshotDownload.DownloadFinished = info.DownloadFinished + + fmt.Println("snapshotDownload", d.snapshotDownload) + if info.DownloadFinished { return } @@ -53,30 +66,30 @@ func (d *DiagnosticClient) runSnapshotListener() { }() } -func (d *DiagnosticClient) SnapshotDownload() map[string]diaglib.DownloadStatistics { +func (d *DiagnosticClient) SnapshotDownload() diaglib.SnapshotDownloadStatistics { return d.snapshotDownload } func (d *DiagnosticClient) runTorrentListener() { go func() { - ctx, ch, cancel := diaglib.Context[diaglib.TorrentFile](context.Background(), 1) + ctx, ch, cancel := diaglib.Context[diaglib.SegmentDownloadStatistics](context.Background(), 1) defer cancel() rootCtx, _ := common.RootContext() - diaglib.StartProviders(ctx, diaglib.TypeOf(diaglib.TorrentFile{}), log.Root()) + diaglib.StartProviders(ctx, diaglib.TypeOf(diaglib.SegmentDownloadStatistics{}), log.Root()) for { select { case <-rootCtx.Done(): cancel() return case info := <-ch: - d.fileDownload[info.Name] = info + if d.snapshotDownload.Segments == nil { + d.snapshotDownload.Segments = map[string]diaglib.SegmentDownloadStatistics{} + } + + d.snapshotDownload.Segments[info.Name] = info } } }() } - -func (d *DiagnosticClient) TorrentFileDownload() map[string]diaglib.TorrentFile { - return d.fileDownload -} diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 1255b9aae93..043ea67247e 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -29,35 +29,35 @@ type PeerStatistics struct { TypeBytesOut map[string]uint64 } -type DownloadStatistics struct { - Downloaded uint64 `json:"downloaded"` - Total uint64 `json:"total"` - TotalTime float64 `json:"totalTime"` - DownloadRate uint64 `json:"downloadRate"` - UploadRate uint64 `json:"uploadRate"` - Peers int32 `json:"peers"` - Files int32 `json:"files"` - Connections uint64 `json:"connections"` - Alloc uint64 `json:"alloc"` - Sys uint64 `json:"sys"` - DownloadFinished bool `json:"downloadFinished"` - StagePrefix string `json:"stagePrefix"` +type SnapshotDownloadStatistics struct { + Downloaded uint64 `json:"downloaded"` + Total uint64 `json:"total"` + TotalTime float64 `json:"totalTime"` + DownloadRate uint64 `json:"downloadRate"` + UploadRate uint64 `json:"uploadRate"` + Peers int32 `json:"peers"` + Files int32 `json:"files"` + Connections uint64 `json:"connections"` + Alloc uint64 `json:"alloc"` + Sys uint64 `json:"sys"` + DownloadFinished bool `json:"downloadFinished"` + Segments map[string]SegmentDownloadStatistics `json:"segments"` } -type TorrentFile struct { +type SegmentDownloadStatistics struct { Name string `json:"name"` TotalBytes uint64 `json:"totalBytes"` DownloadedBytes uint64 `json:"downloadedBytes"` - SeedsCount int `json:"seedsCount"` + WebseedsCount int `json:"webseedsCount"` PeersCount int `json:"peersCount"` WebseedsRate uint64 `json:"webseedsRate"` - SeedsRate uint64 `json:"seedsRate"` + PeersRate uint64 `json:"peersRate"` } -func (ti DownloadStatistics) Type() Type { +func (ti SnapshotDownloadStatistics) Type() Type { return TypeOf(ti) } -func (ti TorrentFile) Type() Type { +func (ti SegmentDownloadStatistics) Type() Type { return TypeOf(ti) } diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 989108c11b3..8c1e142e798 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -350,7 +350,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { } d.logger.Log(d.verbosity, "[snapshots] progress", "file", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "peers", len(peersOfThisFile), "webseeds", len(weebseedPeersOfThisFile)) - isDiagEnabled := diagnostics.TypeOf(diagnostics.TorrentFile{}).Enabled() + isDiagEnabled := diagnostics.TypeOf(diagnostics.SegmentDownloadStatistics{}).Enabled() if d.verbosity < log.LvlInfo || isDiagEnabled { // more detailed statistic: download rate of each peer (for each file) @@ -376,28 +376,28 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { d.logger.Info(fmt.Sprintf("[snapshots] webseed peers file=%s", t.Name()), webseedRates...) rates := make([]interface{}, 0, len(peersOfThisFile)*2) - seedsRates := uint64(0) + peersRates := uint64(0) for _, peer := range peersOfThisFile { dr := uint64(peer.DownloadRate()) rates = append(rates, peer.PeerClientName.Load(), fmt.Sprintf("%s/s", common.ByteCount(dr))) - seedsRates += dr + peersRates += dr } d.logger.Info(fmt.Sprintf("[snapshots] bittorrent peers file=%s", t.Name()), rates...) lenght = uint64(len(peersOfThisFile)) if lenght > 0 { - seedsRates = seedsRates / uint64(len(peersOfThisFile)) + peersRates = peersRates / uint64(len(peersOfThisFile)) } if isDiagEnabled { - diagnostics.Send(diagnostics.TorrentFile{ + diagnostics.Send(diagnostics.SegmentDownloadStatistics{ Name: t.Name(), TotalBytes: uint64(t.Length()), DownloadedBytes: uint64(t.BytesCompleted()), - SeedsCount: len(t.Metainfo().UrlList), + WebseedsCount: len(weebseedPeersOfThisFile), PeersCount: len(peersOfThisFile), WebseedsRate: websRates, - SeedsRate: seedsRates, + PeersRate: peersRates, }) } } diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 2912312db61..bcdbc87ed56 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -212,7 +212,7 @@ Loop: } */ - diagnostics.Send(diagnostics.DownloadStatistics{ + diagnostics.Send(diagnostics.SnapshotDownloadStatistics{ Downloaded: stats.BytesCompleted, Total: stats.BytesTotal, TotalTime: time.Since(downloadStartTime).Round(time.Second).Seconds(), @@ -224,7 +224,6 @@ Loop: Alloc: m.Alloc, Sys: m.Sys, DownloadFinished: stats.Completed, - StagePrefix: logPrefix, }) log.Info(fmt.Sprintf("[%s] download finished", logPrefix), "time", time.Since(downloadStartTime).String()) @@ -241,7 +240,7 @@ Loop: suffix += " (or verifying)" } - diagnostics.Send(diagnostics.DownloadStatistics{ + diagnostics.Send(diagnostics.SnapshotDownloadStatistics{ Downloaded: stats.BytesCompleted, Total: stats.BytesTotal, TotalTime: time.Since(downloadStartTime).Round(time.Second).Seconds(), @@ -253,7 +252,6 @@ Loop: Alloc: m.Alloc, Sys: m.Sys, DownloadFinished: stats.Completed, - StagePrefix: logPrefix, }) log.Info(fmt.Sprintf("[%s] %s", logPrefix, suffix), From a0c43482adbce8bd92f6b0abe75caf2ec0f49e40 Mon Sep 17 00:00:00 2001 From: dvovk Date: Fri, 8 Dec 2023 12:10:31 +0100 Subject: [PATCH 10/11] fix conflicts --- diagnostics/diagnostic.go | 28 ---------------------------- erigon-lib/diagnostics/entities.go | 14 -------------- erigon-lib/downloader/downloader.go | 27 --------------------------- 3 files changed, 69 deletions(-) diff --git a/diagnostics/diagnostic.go b/diagnostics/diagnostic.go index e4a90a21e9e..ff49297e84a 100644 --- a/diagnostics/diagnostic.go +++ b/diagnostics/diagnostic.go @@ -17,20 +17,11 @@ type DiagnosticClient struct { metricsMux *http.ServeMux node *node.ErigonNode -<<<<<<< HEAD snapshotDownload diaglib.SnapshotDownloadStatistics } func NewDiagnosticClient(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) *DiagnosticClient { return &DiagnosticClient{ctx: ctx, metricsMux: metricsMux, node: node, snapshotDownload: diaglib.SnapshotDownloadStatistics{}} -======= - snapshotDownload map[string]diaglib.DownloadStatistics - fileDownload map[string]diaglib.TorrentFile -} - -func NewDiagnosticClient(ctx *cli.Context, metricsMux *http.ServeMux, node *node.ErigonNode) *DiagnosticClient { - return &DiagnosticClient{ctx: ctx, metricsMux: metricsMux, node: node, snapshotDownload: map[string]diaglib.DownloadStatistics{}, fileDownload: map[string]diaglib.TorrentFile{}} ->>>>>>> devel } func (d *DiagnosticClient) Setup() { @@ -81,43 +72,24 @@ func (d *DiagnosticClient) SnapshotDownload() diaglib.SnapshotDownloadStatistics func (d *DiagnosticClient) runTorrentListener() { go func() { -<<<<<<< HEAD ctx, ch, cancel := diaglib.Context[diaglib.SegmentDownloadStatistics](context.Background(), 1) -======= - ctx, ch, cancel := diaglib.Context[diaglib.TorrentFile](context.Background(), 1) ->>>>>>> devel defer cancel() rootCtx, _ := common.RootContext() -<<<<<<< HEAD diaglib.StartProviders(ctx, diaglib.TypeOf(diaglib.SegmentDownloadStatistics{}), log.Root()) -======= - diaglib.StartProviders(ctx, diaglib.TypeOf(diaglib.TorrentFile{}), log.Root()) ->>>>>>> devel for { select { case <-rootCtx.Done(): cancel() return case info := <-ch: -<<<<<<< HEAD if d.snapshotDownload.Segments == nil { d.snapshotDownload.Segments = map[string]diaglib.SegmentDownloadStatistics{} } d.snapshotDownload.Segments[info.Name] = info -======= - d.fileDownload[info.Name] = info ->>>>>>> devel } } }() } -<<<<<<< HEAD -======= - -func (d *DiagnosticClient) TorrentFileDownload() map[string]diaglib.TorrentFile { - return d.fileDownload -} ->>>>>>> devel diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index d3799a10e4b..70b9706ebd4 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -44,7 +44,6 @@ type SnapshotDownloadStatistics struct { Segments map[string]SegmentDownloadStatistics `json:"segments"` } -<<<<<<< HEAD type SegmentDownloadStatistics struct { Name string `json:"name"` TotalBytes uint64 `json:"totalBytes"` @@ -60,19 +59,6 @@ func (ti SnapshotDownloadStatistics) Type() Type { } func (ti SegmentDownloadStatistics) Type() Type { -======= -type TorrentFile struct { - Name string `json:"name"` - TotalBytes uint64 `json:"totalBytes"` - DownloadedBytes uint64 `json:"downloadedBytes"` - SeedsCount int `json:"seedsCount"` - PeersCount int `json:"peersCount"` - WebseedsRate uint64 `json:"webseedsRate"` - SeedsRate uint64 `json:"seedsRate"` -} - -func (ti DownloadStatistics) Type() Type { ->>>>>>> devel return TypeOf(ti) } diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index d37c118ba49..8c1e142e798 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -350,11 +350,7 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { } d.logger.Log(d.verbosity, "[snapshots] progress", "file", t.Name(), "progress", fmt.Sprintf("%.2f%%", progress), "peers", len(peersOfThisFile), "webseeds", len(weebseedPeersOfThisFile)) -<<<<<<< HEAD isDiagEnabled := diagnostics.TypeOf(diagnostics.SegmentDownloadStatistics{}).Enabled() -======= - isDiagEnabled := diagnostics.TypeOf(diagnostics.TorrentFile{}).Enabled() ->>>>>>> devel if d.verbosity < log.LvlInfo || isDiagEnabled { // more detailed statistic: download rate of each peer (for each file) @@ -380,25 +376,16 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { d.logger.Info(fmt.Sprintf("[snapshots] webseed peers file=%s", t.Name()), webseedRates...) rates := make([]interface{}, 0, len(peersOfThisFile)*2) -<<<<<<< HEAD peersRates := uint64(0) for _, peer := range peersOfThisFile { dr := uint64(peer.DownloadRate()) rates = append(rates, peer.PeerClientName.Load(), fmt.Sprintf("%s/s", common.ByteCount(dr))) peersRates += dr -======= - seedsRates := uint64(0) - for _, peer := range peersOfThisFile { - dr := uint64(peer.DownloadRate()) - rates = append(rates, peer.PeerClientName.Load(), fmt.Sprintf("%s/s", common.ByteCount(dr))) - seedsRates += dr ->>>>>>> devel } d.logger.Info(fmt.Sprintf("[snapshots] bittorrent peers file=%s", t.Name()), rates...) lenght = uint64(len(peersOfThisFile)) if lenght > 0 { -<<<<<<< HEAD peersRates = peersRates / uint64(len(peersOfThisFile)) } @@ -411,20 +398,6 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { PeersCount: len(peersOfThisFile), WebseedsRate: websRates, PeersRate: peersRates, -======= - seedsRates = seedsRates / uint64(len(peersOfThisFile)) - } - - if isDiagEnabled { - diagnostics.Send(diagnostics.TorrentFile{ - Name: t.Name(), - TotalBytes: uint64(t.Length()), - DownloadedBytes: uint64(t.BytesCompleted()), - SeedsCount: len(t.Metainfo().UrlList), - PeersCount: len(peersOfThisFile), - WebseedsRate: websRates, - SeedsRate: seedsRates, ->>>>>>> devel }) } } From 50522c066cf4149331e39a077bf5fbb0590b46da Mon Sep 17 00:00:00 2001 From: dvovk Date: Fri, 8 Dec 2023 12:17:34 +0100 Subject: [PATCH 11/11] fix --- erigon-lib/diagnostics/entities.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/erigon-lib/diagnostics/entities.go b/erigon-lib/diagnostics/entities.go index 70b9706ebd4..043ea67247e 100644 --- a/erigon-lib/diagnostics/entities.go +++ b/erigon-lib/diagnostics/entities.go @@ -61,7 +61,3 @@ func (ti SnapshotDownloadStatistics) Type() Type { func (ti SegmentDownloadStatistics) Type() Type { return TypeOf(ti) } - -func (ti TorrentFile) Type() Type { - return TypeOf(ti) -}