Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Rounds chunk bytes to kb in tsdb and includes benchmarking script #5479

Merged
merged 3 commits into from
Feb 25, 2022
Merged

Rounds chunk bytes to kb in tsdb and includes benchmarking script #5479

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
8967c77
rounds chunk bytes to kb in tsdb and includes benchmarking script
owen-d Feb 25, 2022
5a3ca30
shellcheck
owen-d Feb 25, 2022
a9d92aa
fix querier test
owen-d Feb 25, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 2 additions & 6 deletions pkg/storage/tsdb/index/chunk.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,8 @@ type ChunkMeta struct {

MinTime, MaxTime int64

// Bytes use an uint64 as an uint32 can only hold [0,4GB)
// While this is well within current chunk guidelines (1.5MB being "standard"),
// I (owen-d) prefer to overallocate here
// Since TSDB accesses are seeked rather than scanned, this choice
// should have little effect as long as there is enough memory available
Bytes uint64
// Bytes stored, rounded to nearest KB
KB uint32

Entries uint32
}
Expand Down
12 changes: 6 additions & 6 deletions pkg/storage/tsdb/index/index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
c := chunks[0]
w.buf2.PutVarint64(c.MinTime)
w.buf2.PutUvarint64(uint64(c.MaxTime - c.MinTime))
w.buf2.PutUvarint64(c.Bytes)
w.buf2.PutUvarint32(c.KB)
w.buf2.PutUvarint32(c.Entries)
w.buf2.PutBE32(c.Checksum)
t0 := c.MaxTime
Expand All @@ -468,7 +468,7 @@ func (w *Writer) AddSeries(ref storage.SeriesRef, lset labels.Labels, chunks ...
// instead of uvarint because chunks may overlap
w.buf2.PutVarint64(c.MinTime - t0)
w.buf2.PutUvarint64(uint64(c.MaxTime - c.MinTime))
w.buf2.PutUvarint64(c.Bytes)
w.buf2.PutUvarint32(c.KB)
w.buf2.PutUvarint32(c.Entries)
t0 = c.MaxTime

Expand Down Expand Up @@ -1869,15 +1869,15 @@ func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]ChunkMeta) err

t0 := d.Varint64()
maxt := int64(d.Uvarint64()) + t0
nBytes := d.Uvarint64()
kb := uint32(d.Uvarint())
entries := uint32(d.Uvarint64())
checksum := d.Be32()

*chks = append(*chks, ChunkMeta{
Checksum: checksum,
MinTime: t0,
MaxTime: maxt,
Bytes: nBytes,
KB: kb,
Entries: entries,
})
t0 = maxt
Expand All @@ -1887,7 +1887,7 @@ func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]ChunkMeta) err
// instead of uvarint because chunks may overlap
mint := d.Varint64() + t0
maxt := int64(d.Uvarint64()) + mint
nBytes := d.Uvarint64()
kb := uint32(d.Uvarint())
entries := uint32(d.Uvarint64())
checksum := d.Be32()
t0 = maxt
Expand All @@ -1900,7 +1900,7 @@ func (dec *Decoder) Series(b []byte, lbls *labels.Labels, chks *[]ChunkMeta) err
Checksum: checksum,
MinTime: mint,
MaxTime: maxt,
Bytes: nBytes,
KB: kb,
Entries: entries,
})
}
Expand Down
12 changes: 6 additions & 6 deletions pkg/storage/tsdb/querier_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,14 +33,14 @@ func TestQueryIndex(t *testing.T) {
Checksum: 1,
MinTime: 1,
MaxTime: 10,
Bytes: 10,
KB: 10,
Entries: 10,
},
{
Checksum: 2,
MinTime: 5,
MaxTime: 15,
Bytes: 10,
KB: 10,
Entries: 10,
},
},
Expand All @@ -52,14 +52,14 @@ func TestQueryIndex(t *testing.T) {
Checksum: 3,
MinTime: 20,
MaxTime: 30,
Bytes: 10,
KB: 10,
Entries: 10,
},
{
Checksum: 4,
MinTime: 40,
MaxTime: 50,
Bytes: 10,
KB: 10,
Entries: 10,
},
},
Expand All @@ -71,14 +71,14 @@ func TestQueryIndex(t *testing.T) {
Checksum: 1,
MinTime: 1,
MaxTime: 10,
Bytes: 10,
KB: 10,
Entries: 10,
},
{
Checksum: 2,
MinTime: 5,
MaxTime: 15,
Bytes: 10,
KB: 10,
Entries: 10,
},
},
Expand Down
18 changes: 18 additions & 0 deletions tools/tsdb/tsdb-map/diff.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
#!/usr/bin/env bash

old=$1
new=$2

echo benchmarks:
echo

benchstat \
<(LOKI_TSDB_PATH="${old}" go test github.com/grafana/loki/tools/tsdb/tsdb-map -bench=BenchmarkQuery -run '^$' -benchmem) \
<(LOKI_TSDB_PATH="${new}" go test github.com/grafana/loki/tools/tsdb/tsdb-map -bench=BenchmarkQuery -run '^$' -benchmem)

echo
echo sizing:
echo

ls -lh "${old}"
ls -lh "${new}"
4 changes: 2 additions & 2 deletions tools/tsdb/tsdb-map/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,8 @@ func main() {
Checksum: extractChecksumFromChunkID(entry.ChunkID),
MinTime: int64(entry.From),
MaxTime: int64(entry.Through),
Bytes: (3 << 20) / 4, // guess: 0.75mb, 1/2 of the max size
Entries: 10000, // guess: 10k entries
KB: ((3 << 20) / 4) / 1024, // guess: 0.75mb, 1/2 of the max size, rounded to KB
Entries: 10000, // guess: 10k entries
}})
}

Expand Down