Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove Durango codec check #2818

Merged
merged 9 commits into from
Mar 11, 2024
Merged

Remove Durango codec check #2818

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
271565e
Remove Durango codec check
StephenButtolph Mar 6, 2024
a86552e
update coreth
StephenButtolph Mar 6, 2024
8e72e65
Prevent zero length values in codec
StephenButtolph Mar 6, 2024
36fd2cc
Allow zero length values but prevent them in dynamic environments
StephenButtolph Mar 6, 2024
57f9c5c
reduce diff
StephenButtolph Mar 6, 2024
c9f5d96
fix-reuse
StephenButtolph Mar 6, 2024
409846c
Merge branch 'ban-zero-length-values' into remove-durango-codec-check
StephenButtolph Mar 6, 2024
57b01c0
merged
StephenButtolph Mar 9, 2024
8c8c024
merged
StephenButtolph Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions api/keystore/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
package keystore

import (
"time"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/linearcodec"
"github.com/ava-labs/avalanchego/utils/units"
Expand All @@ -20,7 +18,7 @@ const (
var Codec codec.Manager

func init() {
lc := linearcodec.NewDefault(time.Time{})
lc := linearcodec.NewDefault()
Codec = codec.NewManager(maxPackerSize)
if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
Expand Down
3 changes: 1 addition & 2 deletions chains/atomic/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ package atomic

import (
"math"
"time"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/linearcodec"
Expand All @@ -17,7 +16,7 @@ const CodecVersion = 0
var Codec codec.Manager

func init() {
lc := linearcodec.NewDefault(time.Time{})
lc := linearcodec.NewDefault()
Codec = codec.NewManager(math.MaxInt)
if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
panic(err)
Expand Down
14 changes: 4 additions & 10 deletions codec/hierarchycodec/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,13 @@ import (
"fmt"
"reflect"
"sync"
"time"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/reflectcodec"
"github.com/ava-labs/avalanchego/utils/bimap"
"github.com/ava-labs/avalanchego/utils/wrappers"
)

const (
// default max length of a slice being marshalled by Marshal(). Should be <= math.MaxUint32.
defaultMaxSliceLength = 256 * 1024
)

var (
_ Codec = (*hierarchyCodec)(nil)
_ codec.Codec = (*hierarchyCodec)(nil)
Expand Down Expand Up @@ -51,19 +45,19 @@ type hierarchyCodec struct {
}

// New returns a new, concurrency-safe codec
func New(durangoTime time.Time, tagNames []string, maxSliceLen uint32) Codec {
func New(tagNames []string) Codec {
hCodec := &hierarchyCodec{
currentGroupID: 0,
nextTypeID: 0,
registeredTypes: bimap.New[typeID, reflect.Type](),
}
hCodec.Codec = reflectcodec.New(hCodec, tagNames, durangoTime, maxSliceLen)
hCodec.Codec = reflectcodec.New(hCodec, tagNames)
return hCodec
}

// NewDefault returns a new codec with reasonable default values
func NewDefault(durangoTime time.Time) Codec {
return New(durangoTime, []string{reflectcodec.DefaultTagName}, defaultMaxSliceLength)
func NewDefault() Codec {
return New([]string{reflectcodec.DefaultTagName})
}

// SkipRegistrations some number of type IDs
Expand Down
22 changes: 3 additions & 19 deletions codec/hierarchycodec/codec_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,41 +5,25 @@ package hierarchycodec

import (
"testing"
"time"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/utils/timer/mockable"
)

func TestVectors(t *testing.T) {
for _, test := range codec.Tests {
c := NewDefault(mockable.MaxTime)
c := NewDefault()
test(c, t)
}
}

func TestMultipleTags(t *testing.T) {
for _, test := range codec.MultipleTagsTests {
c := New(mockable.MaxTime, []string{"tag1", "tag2"}, defaultMaxSliceLength)
test(c, t)
}
}

func TestEnforceSliceLen(t *testing.T) {
for _, test := range codec.EnforceSliceLenTests {
c := NewDefault(mockable.MaxTime)
test(c, t)
}
}

func TestIgnoreSliceLen(t *testing.T) {
for _, test := range codec.IgnoreSliceLenTests {
c := NewDefault(time.Time{})
c := New([]string{"tag1", "tag2"})
test(c, t)
}
}

func FuzzStructUnmarshalHierarchyCodec(f *testing.F) {
c := NewDefault(mockable.MaxTime)
c := NewDefault()
codec.FuzzStructUnmarshal(c, f)
}
25 changes: 7 additions & 18 deletions codec/linearcodec/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,19 +7,13 @@ import (
"fmt"
"reflect"
"sync"
"time"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/reflectcodec"
"github.com/ava-labs/avalanchego/utils/bimap"
"github.com/ava-labs/avalanchego/utils/wrappers"
)

const (
// default max length of a slice being marshalled by Marshal(). Should be <= math.MaxUint32.
DefaultMaxSliceLength = 256 * 1024
)

var (
_ Codec = (*linearCodec)(nil)
_ codec.Codec = (*linearCodec)(nil)
Expand All @@ -43,25 +37,20 @@ type linearCodec struct {
registeredTypes *bimap.BiMap[uint32, reflect.Type]
}

// New returns a new, concurrency-safe codec; it allow to specify
// both tagNames and maxSlicelenght
func New(durangoTime time.Time, tagNames []string, maxSliceLen uint32) Codec {
// New returns a new, concurrency-safe codec; it allow to specify tagNames.
func New(tagNames []string) Codec {
hCodec := &linearCodec{
nextTypeID: 0,
registeredTypes: bimap.New[uint32, reflect.Type](),
}
hCodec.Codec = reflectcodec.New(hCodec, tagNames, durangoTime, maxSliceLen)
hCodec.Codec = reflectcodec.New(hCodec, tagNames)
return hCodec
}

// NewDefault is a convenience constructor; it returns a new codec with reasonable default values
func NewDefault(durangoTime time.Time) Codec {
return New(durangoTime, []string{reflectcodec.DefaultTagName}, DefaultMaxSliceLength)
}

// NewCustomMaxLength is a convenience constructor; it returns a new codec with custom max length and default tags
func NewCustomMaxLength(durangoTime time.Time, maxSliceLen uint32) Codec {
return New(durangoTime, []string{reflectcodec.DefaultTagName}, maxSliceLen)
// NewDefault is a convenience constructor; it returns a new codec with default
// tagNames.
func NewDefault() Codec {
return New([]string{reflectcodec.DefaultTagName})
}

// Skip some number of type IDs
Expand Down
22 changes: 3 additions & 19 deletions codec/linearcodec/codec_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,41 +5,25 @@ package linearcodec

import (
"testing"
"time"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/utils/timer/mockable"
)

func TestVectors(t *testing.T) {
for _, test := range codec.Tests {
c := NewDefault(mockable.MaxTime)
c := NewDefault()
test(c, t)
}
}

func TestMultipleTags(t *testing.T) {
for _, test := range codec.MultipleTagsTests {
c := New(mockable.MaxTime, []string{"tag1", "tag2"}, DefaultMaxSliceLength)
test(c, t)
}
}

func TestEnforceSliceLen(t *testing.T) {
for _, test := range codec.EnforceSliceLenTests {
c := NewDefault(mockable.MaxTime)
test(c, t)
}
}

func TestIgnoreSliceLen(t *testing.T) {
for _, test := range codec.IgnoreSliceLenTests {
c := NewDefault(time.Time{})
c := New([]string{"tag1", "tag2"})
test(c, t)
}
}

func FuzzStructUnmarshalLinearCodec(f *testing.F) {
c := NewDefault(mockable.MaxTime)
c := NewDefault()
codec.FuzzStructUnmarshal(c, f)
}
43 changes: 5 additions & 38 deletions codec/reflectcodec/type_codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"math"
"reflect"
"slices"
"time"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/utils/set"
Expand Down Expand Up @@ -71,19 +70,15 @@ type TypeCodec interface {
// 6. Serialized fields must be exported
// 7. nil slices are marshaled as empty slices
type genericCodec struct {
typer TypeCodec
durangoTime time.Time // Time after which [maxSliceLen] will be ignored
maxSliceLen uint32
fielder StructFielder
typer TypeCodec
fielder StructFielder
}

// New returns a new, concurrency-safe codec
func New(typer TypeCodec, tagNames []string, durangoTime time.Time, maxSliceLen uint32) codec.Codec {
func New(typer TypeCodec, tagNames []string) codec.Codec {
return &genericCodec{
typer: typer,
durangoTime: durangoTime,
maxSliceLen: maxSliceLen,
fielder: NewStructFielder(tagNames),
typer: typer,
fielder: NewStructFielder(tagNames),
}
}

Expand Down Expand Up @@ -378,13 +373,6 @@ func (c *genericCodec) marshal(
math.MaxInt32,
)
}
if time.Now().Before(c.durangoTime) && uint32(numElts) > c.maxSliceLen {
return fmt.Errorf("%w; slice length, %d, exceeds maximum length, %d",
codec.ErrMaxSliceLenExceeded,
numElts,
c.maxSliceLen,
)
}
p.PackInt(uint32(numElts)) // pack # elements
if p.Err != nil {
return p.Err
Expand Down Expand Up @@ -445,13 +433,6 @@ func (c *genericCodec) marshal(
math.MaxInt32,
)
}
if time.Now().Before(c.durangoTime) && uint32(numElts) > c.maxSliceLen {
return fmt.Errorf("%w; map length, %d, exceeds maximum length, %d",
codec.ErrMaxSliceLenExceeded,
numElts,
c.maxSliceLen,
)
}
p.PackInt(uint32(numElts)) // pack # elements
if p.Err != nil {
return p.Err
Expand Down Expand Up @@ -619,13 +600,6 @@ func (c *genericCodec) unmarshal(
math.MaxInt32,
)
}
if time.Now().Before(c.durangoTime) && numElts32 > c.maxSliceLen {
return fmt.Errorf("%w; array length, %d, exceeds maximum length, %d",
codec.ErrMaxSliceLenExceeded,
numElts32,
c.maxSliceLen,
)
}
numElts := int(numElts32)

sliceType := value.Type()
Expand Down Expand Up @@ -732,13 +706,6 @@ func (c *genericCodec) unmarshal(
math.MaxInt32,
)
}
if time.Now().Before(c.durangoTime) && numElts32 > c.maxSliceLen {
return fmt.Errorf("%w; map length, %d, exceeds maximum length, %d",
codec.ErrMaxSliceLenExceeded,
numElts32,
c.maxSliceLen,
)
}

var (
numElts = int(numElts32)
Expand Down
41 changes: 1 addition & 40 deletions codec/test_codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,6 @@ import (
"testing"

"github.com/stretchr/testify/require"

"github.com/ava-labs/avalanchego/utils/wrappers"
)

var (
Expand Down Expand Up @@ -46,20 +44,12 @@ var (
TestExtraSpace,
TestSliceLengthOverflow,
TestMap,
TestCanMarshalLargeSlices,
}

MultipleTagsTests = []func(c GeneralCodec, t testing.TB){
TestMultipleTags,
}

EnforceSliceLenTests = []func(c GeneralCodec, t testing.TB){
TestCanNotMarshalLargeSlices,
TestCanNotUnmarshalLargeSlices,
}

IgnoreSliceLenTests = []func(c GeneralCodec, t testing.TB){
TestCanMarshalLargeSlices,
}
)

// The below structs and interfaces exist
Expand Down Expand Up @@ -1088,35 +1078,6 @@ func TestMap(codec GeneralCodec, t testing.TB) {
require.Len(outerArrayBytes, outerArraySize)
}

func TestCanNotMarshalLargeSlices(codec GeneralCodec, t testing.TB) {
require := require.New(t)

data := make([]uint16, 1_000_000)

manager := NewManager(math.MaxInt)
require.NoError(manager.RegisterCodec(0, codec))

_, err := manager.Marshal(0, data)
require.ErrorIs(err, ErrMaxSliceLenExceeded)
}

func TestCanNotUnmarshalLargeSlices(codec GeneralCodec, t testing.TB) {
require := require.New(t)

writer := wrappers.Packer{
Bytes: make([]byte, 2+4+2_000_000),
}
writer.PackShort(0)
writer.PackInt(1_000_000)

manager := NewManager(math.MaxInt)
require.NoError(manager.RegisterCodec(0, codec))

var data []uint16
_, err := manager.Unmarshal(writer.Bytes, &data)
require.ErrorIs(err, ErrMaxSliceLenExceeded)
}

func TestCanMarshalLargeSlices(codec GeneralCodec, t testing.TB) {
require := require.New(t)

Expand Down
4 changes: 1 addition & 3 deletions database/encdb/codec.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@
package encdb

import (
"time"

"github.com/ava-labs/avalanchego/codec"
"github.com/ava-labs/avalanchego/codec/linearcodec"
)
Expand All @@ -15,7 +13,7 @@ const CodecVersion = 0
var Codec codec.Manager

func init() {
lc := linearcodec.NewDefault(time.Time{})
lc := linearcodec.NewDefault()
Codec = codec.NewDefaultManager()

if err := Codec.RegisterCodec(CodecVersion, lc); err != nil {
Expand Down
Loading
Loading