compat

bytedance · Aug 26, 2024 · a8a2acd · a8a2acd
1 parent 2b59c86
commit a8a2acd
Show file tree

Hide file tree

Showing 6 changed files with 90 additions and 76 deletions.
diff --git a/.gitignore b/.gitignore
@@ -52,3 +52,4 @@ fuzz/testdata
 *__debug_bin*
 *pprof
 *coverage.txt
+tools/venv/*
diff --git a/api.go b/api.go
@@ -84,6 +84,10 @@ type Config struct {
     // NoValidateJSONMarshaler indicates that the encoder should not validate the output string
     // after encoding the JSONMarshaler to JSON.
     NoValidateJSONMarshaler       bool
+
+    // NoValidateJSONSkip indicates the decoder should not validate the JSON value when skipping it,
+    // such as unknown-fields, mismatched-type, redundant elements..
+    NoValidateJSONSkip bool
 
     // NoEncoderNewline indicates that the encoder should not add a newline after every message
     NoEncoderNewline bool
@@ -109,6 +113,7 @@ var (
     ConfigFastest = Config{
         NoQuoteTextMarshaler: true,
         NoValidateJSONMarshaler: true,
+        NoValidateJSONSkip: true,
     }.Froze()
 )
 

diff --git a/decoder/decoder_compat.go b/decoder/decoder_compat.go
@@ -42,6 +42,7 @@ const (
      _F_use_number      = types.B_USE_NUMBER
      _F_validate_string = types.B_VALIDATE_STRING
      _F_allow_control   = types.B_ALLOW_CONTROL
+     _F_no_validate_json   = types.B_NO_VALIDATE_JSON
 )
 
 type Options uint64
@@ -53,6 +54,7 @@ const (
      OptionDisableUnknown   Options = 1 << _F_disable_unknown
      OptionCopyString       Options = 1 << _F_copy_string
      OptionValidateString   Options = 1 << _F_validate_string
+     OptionNoValidateJSON   Options = 1 << _F_no_validate_json
 )
 
 func (self *Decoder) SetOptions(opts Options) {

diff --git a/decoder/decoder_native_test.go b/decoder/decoder_native_test.go
@@ -1,7 +1,6 @@
 //go:build (amd64 && go1.17 && !go1.24) || (arm64 && go1.20 && !go1.24)
 // +build amd64,go1.17,!go1.24 arm64,go1.20,!go1.24
 
-
 /*
  * Copyright 2021 ByteDance Inc.
  *
@@ -21,15 +20,88 @@
 package decoder
 
 import (
-    `encoding/json`
-    _`strings`
-    `testing`
-    _`reflect`
+	"encoding/json"
+	"fmt"
+	_ "reflect"
+	"strings"
+	_ "strings"
+	"testing"
+	"time"
 
-    `github.com/bytedance/sonic/internal/rt`
-    `github.com/stretchr/testify/assert`
+	"github.com/bytedance/sonic/internal/rt"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
+
+func BenchmarkSkipValidate(b *testing.B) {
+    type skiptype struct {
+        A int `json:"a"` // mismatched
+        B string `json:"-"` // ommited
+        C [1]int `json:"c"` // fast int
+        D struct {} `json:"d"` // empty struct
+        E map[string]int `json:"e"` // mismatched elem
+        // Unknonwn
+    }
+    type C struct {
+        name string
+        json string
+        expTime float64
+    }
+    var sam = map[int]interface{}{}
+    for i := 0; i < 1; i++ {
+        sam[i] = _BindingValue
+    }
+    comptd, err := json.Marshal(sam)
+    if err != nil {
+        b.Fatal("invalid json")
+    }
+    compt := string(comptd)
+    var cases = []C{
+        {"mismatched", `{"a":`+compt+`}`, 5},
+        {"ommited", `{"b":`+compt+`}`, 5},
+        {"number", `{"c":[`+strings.Repeat("-1.23456e-19,", 1000)+`1]}`, 1.5},
+        {"unknown", `{"unknown":`+compt+`}`, 5},
+        {"empty", `{"d":`+compt+`}`, 5},
+        {"mismatched elem", `{"e":`+compt+`}`, 5},
+    }
+    _ = NewDecoder(`{}`).Decode(&skiptype{})
+
+    var avg1, avg2 time.Duration
+    for _, c := range cases {
+        b.Run(c.name, func(b *testing.B) {
+            b.Run("validate", func(b *testing.B) {
+                b.ResetTimer()
+                t1 := time.Now()
+                for i := 0; i < b.N; i++ {
+                    var obj1 = &skiptype{}
+                    // validate skip
+                    d := NewDecoder(c.json)
+                    _ = d.Decode(obj1)
+                }
+                d1 := time.Since(t1)
+                avg1 = d1/time.Duration(b.N)
+            })
+            b.Run("fast", func(b *testing.B) {
+                b.ResetTimer()
+                t2 := time.Now()
+                for i := 0; i < b.N; i++ {
+                    var obj2 = &skiptype{}
+                    // fask skip
+                    d := NewDecoder(c.json)
+                    d.SetOptions(OptionNoValidateJSON)
+                    _ = d.Decode(obj2)
+                }
+                d2 := time.Since(t2)
+                avg2 = d2/time.Duration(b.N)
+            })
+            // fast skip must be expTime x faster
+            require.True(b, float64(avg1)/float64(avg2) >  c.expTime, fmt.Sprintf("%v/%v=%v", avg1, avg2, float64(avg1)/float64(avg2)))
+        })
+    }
+}
+
+
 func TestSkipMismatchTypeAmd64Error(t *testing.T) {
     // t.Run("struct", func(t *testing.T) {
     //     println("TestSkipError")

diff --git a/decoder/decoder_test.go b/decoder/decoder_test.go
@@ -18,7 +18,6 @@ package decoder
 
 import (
 	"encoding/json"
-	"fmt"
 	"runtime"
 	"runtime/debug"
 	"strings"
@@ -86,74 +85,6 @@ func init() {
     _ = json.Unmarshal([]byte(TwitterJson), &_BindingValue)
 }
 
-func BenchmarkSkipValidate(b *testing.B) {
-    type skiptype struct {
-        A int `json:"a"` // mismatched
-        B string `json:"-"` // ommited
-        C [1]int `json:"c"` // fast int
-        D struct {} `json:"d"` // empty struct
-        E map[string]int `json:"e"` // mismatched elem
-        // Unknonwn
-    }
-    type C struct {
-        name string
-        json string
-        expTime float64
-    }
-    var sam = map[int]interface{}{}
-    for i := 0; i < 1; i++ {
-        sam[i] = _BindingValue
-    }
-    comptd, err := json.Marshal(sam)
-    if err != nil {
-        b.Fatal("invalid json")
-    }
-    compt := string(comptd)
-    var cases = []C{
-        {"mismatched", `{"a":`+compt+`}`, 5},
-        {"ommited", `{"b":`+compt+`}`, 5},
-        {"number", `{"c":[`+strings.Repeat("-1.23456e-19,", 1000)+`1]}`, 1.5},
-        {"unknown", `{"unknown":`+compt+`}`, 5},
-        {"empty", `{"d":`+compt+`}`, 5},
-        {"mismatched elem", `{"e":`+compt+`}`, 5},
-    }
-    _ = NewDecoder(`{}`).Decode(&skiptype{})
-
-    var avg1, avg2 time.Duration
-    for _, c := range cases {
-        b.Run(c.name, func(b *testing.B) {
-            b.Run("validate", func(b *testing.B) {
-                b.ResetTimer()
-                t1 := time.Now()
-                for i := 0; i < b.N; i++ {
-                    var obj1 = &skiptype{}
-                    // validate skip
-                    d := NewDecoder(c.json)
-                    _ = d.Decode(obj1)
-                }
-                d1 := time.Since(t1)
-                avg1 = d1/time.Duration(b.N)
-            })
-            b.Run("fast", func(b *testing.B) {
-                b.ResetTimer()
-                t2 := time.Now()
-                for i := 0; i < b.N; i++ {
-                    var obj2 = &skiptype{}
-                    // fask skip
-                    d := NewDecoder(c.json)
-                    d.SetOptions(OptionNoValidateJSON)
-                    _ = d.Decode(obj2)
-                }
-                d2 := time.Since(t2)
-                avg2 = d2/time.Duration(b.N)
-            })
-            // fast skip must be expTime x faster
-            require.True(b, float64(avg1)/float64(avg2) >  c.expTime, fmt.Sprintf("%v/%v=%v", avg1, avg2, float64(avg1)/float64(avg2)))
-        })
-    }
-}
-
-
 func TestSkipMismatchTypeError(t *testing.T) {
     t.Run("struct", func(t *testing.T) {
         println("TestSkipError")

diff --git a/sonic.go b/sonic.go
@@ -72,6 +72,9 @@ func (cfg Config) Froze() API {
     }
 
     // configure decoder options:
+    if cfg.NoValidateJSONSkip {
+        api.decoderOpts |= decoder.OptionNoValidateJSON
+    }
     if cfg.UseInt64 {
         api.decoderOpts |= decoder.OptionUseInt64
     }