From 7466c397a0c6a2168df085b2072491a831645fe6 Mon Sep 17 00:00:00 2001 From: David Wright Date: Mon, 22 Oct 2018 19:44:01 -0700 Subject: [PATCH 1/7] make nested work --- xss_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/xss_test.go b/xss_test.go index 558303f..b8750e9 100644 --- a/xss_test.go +++ b/xss_test.go @@ -103,18 +103,18 @@ func newServer(xssMdlwr XssMw) *gin.Engine { c.JSON(200, usr) }) - //// nested JSON - //r.POST("/user_post_nested_json", func(c *gin.Context) { - // var users Users - // // fmt.Printf("%#v", users) - // err := c.Bind(&users) - // if err != nil { - // //fmt.Println(err) - // c.JSON(404, gin.H{"msg": "Bind Failed."}) - // return - // } - // c.JSON(201, users) - //}) + // nested JSON + r.POST("/user_post_nested_json", func(c *gin.Context) { + var users Users + // fmt.Printf("%#v", users) + err := c.Bind(&users) + if err != nil { + //fmt.Println(err) + c.JSON(404, gin.H{"msg": "Bind Failed."}) + return + } + c.JSON(201, users) + }) return r } From c8e4909805eddfdd9a4111cc9d992d7aff377842 Mon Sep 17 00:00:00 2001 From: David Wright Date: Mon, 22 Oct 2018 21:05:44 -0700 Subject: [PATCH 2/7] existing tests pass! --- xss.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/xss.go b/xss.go index d981310..4d86636 100644 --- a/xss.go +++ b/xss.go @@ -556,28 +556,67 @@ func (mw *XssMw) buildJsonApplyPolicy(v interface{}, buff bytes.Buffer, p *bluem buff.WriteByte(',') default: var b bytes.Buffer - apndBuff := mw.reiterateInterface(v, b, p) + apndBuff := mw.reiterateInterface(vv, b, p) buff.WriteString(apndBuff.String()) } return buff } func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { + fmt.Println("IN reiterateInterfaceSlice") + //switch vvvv := vvv.(type) { + //case map[string]interface{}: var lst bytes.Buffer - lst.WriteByte('[') + lst.WriteByte('{') for i, n := range vvv { fmt.Printf("Iter: %v= %v\n", i, n) - fmt.Println("I don't know how to handle") + fmt.Println("IN REITERATE") var r = reflect.TypeOf(n) - fmt.Printf("Unknown Type!:%v\n", r) + fmt.Printf("REiterate Type!:%v\n", r) + switch nn := n.(type) { + case map[string]interface{}: + fmt.Println("MAPY map[string]interface{}") + var scnd bytes.Buffer + scnd.WriteByte('{') + for i, nnn := range nn { + scnd.WriteByte('"') + scnd.WriteString(i) + scnd.WriteByte('"') + scnd.WriteByte(':') + scnd.WriteByte('"') + scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnn))) + scnd.WriteByte('"') + scnd.WriteByte(',') + + } + scnd.Truncate(scnd.Len() - 1) // remove last ',' + scnd.WriteByte('}') + lst.WriteString(scnd.String()) + lst.WriteByte(',') // add cause expected + + case []interface{}: + fmt.Println("MAPY []interface{}") + case json.Number: + fmt.Println("json.Number") + case string: + fmt.Println("float 64") + case float64: + fmt.Println("float 64") + default: + fmt.Println("DEFAULT") + fmt.Printf("nn %v", nn) + } //lst.WriteString(p.Sanitize(fmt.Sprintf("\"%v\"", n))) // NOTE changes from ["1", "4", "8"] to [1,4,8] - lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) - lst.WriteByte(',') + //lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) + //lst.WriteByte(',') } lst.Truncate(lst.Len() - 1) // remove last ',' - lst.WriteByte(']') + lst.WriteByte('}') + //default: + // fmt.Printf("REiterate Default") + //} return lst } @@ -585,22 +624,22 @@ func (mw *XssMw) reiterateInterface(vv interface{}, buff bytes.Buffer, p *bluemo switch vvv := vv.(type) { // map[string]interface {}{"id":"1", "assoc_ids":[]interface {}{"1", "4", "8"}} case map[string]interface{}: + //fmt.Println("IN DEFAULT I don't know how to handle") case []interface{}: - - var b bytes.Buffer - b = mw.reiterateInterfaceSlice(vvv, b, p) + //var b bytes.Buffer + b := mw.reiterateInterfaceSlice(vvv, buff, p) buff.WriteString(b.String()) buff.WriteByte(',') // add cause expected case json.Number: - //fmt.Println(k, "is number", vv) + //fmt.Println("is number", vvv) //buff.WriteString(`"` + p.Sanitize(vv) + `",`) buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) buff.WriteByte(',') default: - fmt.Println("I don't know how to handle") - var r = reflect.TypeOf(vvv) - fmt.Printf("Unknown Type!:%v\n", r) + //fmt.Println("IN DEFAULT I don't know how to handle") + //var r = reflect.TypeOf(vvv) + //fmt.Printf("IN DEFAULT Unknown Type!:%v\n", r) if vvv == nil { buff.WriteString(fmt.Sprintf("%s", "null")) @@ -609,6 +648,7 @@ func (mw *XssMw) reiterateInterface(vv interface{}, buff bytes.Buffer, p *bluemo } buff.WriteByte(',') } + //fmt.Printf("Reiterate Interface %v", buff.String()) return buff } From 996db7f7514b8cfec3995665d58a402a414533ed Mon Sep 17 00:00:00 2001 From: David Wright Date: Mon, 22 Oct 2018 21:34:28 -0700 Subject: [PATCH 3/7] close but lose order in nested --- xss.go | 28 +++++++++++++---- xss_test.go | 86 ++++++++++++++++++++++++++--------------------------- 2 files changed, 65 insertions(+), 49 deletions(-) diff --git a/xss.go b/xss.go index 4d86636..bd0195b 100644 --- a/xss.go +++ b/xss.go @@ -538,6 +538,8 @@ func (mw *XssMw) ConstructJson(xmj XssMwJson) bytes.Buffer { buff.Truncate(buff.Len() - 1) // remove last ',' buff.WriteByte('}') + fmt.Println("TOP CONStruct JSON") + fmt.Println(buff.String()) return buff } @@ -567,7 +569,7 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p //switch vvvv := vvv.(type) { //case map[string]interface{}: var lst bytes.Buffer - lst.WriteByte('{') + lst.WriteByte('[') for i, n := range vvv { fmt.Printf("Iter: %v= %v\n", i, n) fmt.Println("IN REITERATE") @@ -576,6 +578,8 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p switch nn := n.(type) { case map[string]interface{}: fmt.Println("MAPY map[string]interface{}") + ////moo := mw.reiterateInterface(nn, buff, p) + ////lst.WriteString(moo.String()) var scnd bytes.Buffer scnd.WriteByte('{') for i, nnn := range nn { @@ -583,11 +587,18 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p scnd.WriteString(i) scnd.WriteByte('"') scnd.WriteByte(':') - scnd.WriteByte('"') - scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnn))) - scnd.WriteByte('"') + var r = reflect.TypeOf(nnn) + fmt.Printf("REiterate Type!:%v\n", r) + switch nnnn := nnn.(type) { + case json.Number: + scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) + + default: + scnd.WriteByte('"') + scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) + scnd.WriteByte('"') + } scnd.WriteByte(',') - } scnd.Truncate(scnd.Len() - 1) // remove last ',' scnd.WriteByte('}') @@ -596,12 +607,16 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p case []interface{}: fmt.Println("MAPY []interface{}") + lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) case json.Number: fmt.Println("json.Number") + lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) case string: fmt.Println("float 64") + lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) case float64: fmt.Println("float 64") + lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) default: fmt.Println("DEFAULT") fmt.Printf("nn %v", nn) @@ -613,7 +628,7 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p //lst.WriteByte(',') } lst.Truncate(lst.Len() - 1) // remove last ',' - lst.WriteByte('}') + lst.WriteByte(']') //default: // fmt.Printf("REiterate Default") //} @@ -648,6 +663,7 @@ func (mw *XssMw) reiterateInterface(vv interface{}, buff bytes.Buffer, p *bluemo } buff.WriteByte(',') } + //fmt.Printf("Reiterate Interface %v", buff.String()) return buff } diff --git a/xss_test.go b/xss_test.go index b8750e9..b6897fa 100644 --- a/xss_test.go +++ b/xss_test.go @@ -157,49 +157,49 @@ func TestKeepsValuesStripsHtmlOnPost(t *testing.T) { assert.JSONEq(t, expect, resp.Body.String()) } -//func TestSupportNestedJSONPost(t *testing.T) { -// log.SetOutput(ioutil.Discard) -// defer log.SetOutput(os.Stderr) -// -// var xssMdlwr XssMw -// s := newServer(xssMdlwr) -// -// user1 := "TestUser1" -// email1 := "testUser1@example.com" -// password1 := "!@$%^ASDF1" -// cmnt := `` -// cre_at := "1481017167" -// userA := `{"id":1, "flt":1.345, "user":"` + user1 + `", "email": "` + email1 + `", "password":"` + password1 + `", "comment":"` + cmnt + `", "cre_at":` + cre_at + `}` -// -// user2 := "TestUser2" -// email2 := "testUser2@example.com" -// password2 := "!@$%^ASDF2" -// userB := `{"id":2, "flt":2.345, "user":"` + user2 + `", "email": "` + email2 + `", "password":"` + password2 + `", "comment":"` + cmnt + `", "cre_at":` + cre_at + `}` -// -// oParams := `{"id":1, "users": [ ` + userA + `,` + userB + `]}` -// -// req, _ := http.NewRequest("POST", "/user_post_nested_json", bytes.NewBufferString(oParams)) -// req.Header.Add("Content-Type", "application/json") -// req.Header.Add("Content-Length", strconv.Itoa(len(oParams))) -// -// resp := httptest.NewRecorder() -// s.ServeHTTP(resp, req) -// -// assert.Equal(t, 201, resp.Code) -// expStr := `{ -// "id":1, -// "users":[ -// {"id":1, "flt":1.345, "user":"%v", "email":"%v", "password":"%v", "comment":"%v", "cre_at":%v}, -// {"id":2, "flt":2.345, "user":"%v", "email":"%v", "password":"%v", "comment":"%v", "cre_at":%v} -// ] -// }` -// -// expect := fmt.Sprintf(expStr, user1, email1, password1, cmnt, cre_at, user2, email2, password2, cmnt, cre_at) -// fmt.Println(expect) -// -// fmt.Println(resp.Body.String()) -// assert.JSONEq(t, expect, resp.Body.String()) -//} +func TestSupportNestedJSONPost(t *testing.T) { + log.SetOutput(ioutil.Discard) + defer log.SetOutput(os.Stderr) + + var xssMdlwr XssMw + s := newServer(xssMdlwr) + + user1 := "TestUser1" + email1 := "testUser1@example.com" + password1 := "!@$%^ASDF1" + cmnt := `` + cre_at := "1481017167" + userA := `{"id":1, "flt":1.345, "user":"` + user1 + `", "email": "` + email1 + `", "password":"` + password1 + `", "comment":"` + cmnt + `", "cre_at":` + cre_at + `}` + + user2 := "TestUser2" + email2 := "testUser2@example.com" + password2 := "!@$%^ASDF2" + userB := `{"id":2, "flt":2.345, "user":"` + user2 + `", "email": "` + email2 + `", "password":"` + password2 + `", "comment":"` + cmnt + `", "cre_at":` + cre_at + `}` + + oParams := `{"id":1, "users": [ ` + userA + `,` + userB + `]}` + + req, _ := http.NewRequest("POST", "/user_post_nested_json", bytes.NewBufferString(oParams)) + req.Header.Add("Content-Type", "application/json") + req.Header.Add("Content-Length", strconv.Itoa(len(oParams))) + + resp := httptest.NewRecorder() + s.ServeHTTP(resp, req) + + assert.Equal(t, 201, resp.Code) + expStr := `{ + "id":1, + "users":[ + {"id":1, "flt":1.345, "user":"%v", "email":"%v", "password":"%v", "comment":"%v", "cre_at":%v}, + {"id":2, "flt":2.345, "user":"%v", "email":"%v", "password":"%v", "comment":"%v", "cre_at":%v} + ] + }` + + expect := fmt.Sprintf(expStr, user1, email1, password1, cmnt, cre_at, user2, email2, password2, cmnt, cre_at) + fmt.Println(expect) + + fmt.Println(resp.Body.String()) + assert.JSONEq(t, expect, resp.Body.String()) +} func TestKeepsValuesStripsHtmlOnPut(t *testing.T) { log.SetOutput(ioutil.Discard) From 7ad9f05924739a0b54830fcc5fcf89fabba4e5eb Mon Sep 17 00:00:00 2001 From: David Wright Date: Mon, 22 Oct 2018 22:00:43 -0700 Subject: [PATCH 4/7] supports some nesting test case --- xss.go | 112 ++++++++++++++++++++++++++++------------------------ xss_test.go | 7 ++-- 2 files changed, 65 insertions(+), 54 deletions(-) diff --git a/xss.go b/xss.go index bd0195b..70b64ab 100644 --- a/xss.go +++ b/xss.go @@ -46,7 +46,7 @@ import ( "io/ioutil" "mime/multipart" "net/url" - "reflect" // debugging type + //"reflect" // debugging type "strconv" "strings" ) @@ -538,8 +538,8 @@ func (mw *XssMw) ConstructJson(xmj XssMwJson) bytes.Buffer { buff.Truncate(buff.Len() - 1) // remove last ',' buff.WriteByte('}') - fmt.Println("TOP CONStruct JSON") - fmt.Println(buff.String()) + //fmt.Println("TOP CONStruct JSON") + //fmt.Println(buff.String()) return buff } @@ -564,31 +564,74 @@ func (mw *XssMw) buildJsonApplyPolicy(v interface{}, buff bytes.Buffer, p *bluem return buff } +func (mw *XssMw) reiterateInterface(vv interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { + switch vvv := vv.(type) { + // map[string]interface {}{"id":"1", "assoc_ids":[]interface {}{"1", "4", "8"}} + case map[string]interface{}: + //fmt.Println("IN DEFAULT I don't know how to handle") + + case []interface{}: + //var b bytes.Buffer + b := mw.reiterateInterfaceSlice(vvv, buff, p) + buff.WriteString(b.String()) + buff.WriteByte(',') // add cause expected + case json.Number: + //fmt.Println("is number", vvv) + //buff.WriteString(`"` + p.Sanitize(vv) + `",`) + buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) + buff.WriteByte(',') + default: + //fmt.Println("IN DEFAULT I don't know how to handle") + //var r = reflect.TypeOf(vvv) + //fmt.Printf("IN DEFAULT Unknown Type!:%v\n", r) + + if vvv == nil { + buff.WriteString(fmt.Sprintf("%s", "null")) + } else { + buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) + } + buff.WriteByte(',') + } + + //fmt.Printf("Reiterate Interface %v", buff.String()) + return buff +} + func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { - fmt.Println("IN reiterateInterfaceSlice") + //fmt.Println("IN reiterateInterfaceSlice") //switch vvvv := vvv.(type) { //case map[string]interface{}: var lst bytes.Buffer lst.WriteByte('[') - for i, n := range vvv { - fmt.Printf("Iter: %v= %v\n", i, n) - fmt.Println("IN REITERATE") - var r = reflect.TypeOf(n) - fmt.Printf("REiterate Type!:%v\n", r) + for _, n := range vvv { + //fmt.Printf("Iter: %v= %v\n", i, n) + //fmt.Println("IN REITERATE") + //var r = reflect.TypeOf(n) + //fmt.Printf("REiterate Type!:%v\n", r) switch nn := n.(type) { case map[string]interface{}: - fmt.Println("MAPY map[string]interface{}") + //fmt.Println("MAPY map[string]interface{}") ////moo := mw.reiterateInterface(nn, buff, p) ////lst.WriteString(moo.String()) var scnd bytes.Buffer scnd.WriteByte('{') + Top: for i, nnn := range nn { scnd.WriteByte('"') scnd.WriteString(i) scnd.WriteByte('"') scnd.WriteByte(':') - var r = reflect.TypeOf(nnn) - fmt.Printf("REiterate Type!:%v\n", r) + + for _, fts := range mw.FieldsToSkip { + if string(i) == fts { + scnd.WriteString(fmt.Sprintf("%q", nnn)) + scnd.WriteByte(',') + continue Top + } + } + + //var r = reflect.TypeOf(nnn) + //fmt.Printf("REiterate Type!:%v\n", r) switch nnnn := nnn.(type) { case json.Number: scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) @@ -606,20 +649,20 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p lst.WriteByte(',') // add cause expected case []interface{}: - fmt.Println("MAPY []interface{}") + //fmt.Println("MAPY []interface{}") lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) case json.Number: - fmt.Println("json.Number") + //fmt.Println("json.Number") lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) case string: - fmt.Println("float 64") + //fmt.Println("float 64") lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) case float64: - fmt.Println("float 64") + //fmt.Println("float 64") lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) default: - fmt.Println("DEFAULT") - fmt.Printf("nn %v", nn) + //fmt.Println("DEFAULT") + //fmt.Printf("nn %v", nn) } //lst.WriteString(p.Sanitize(fmt.Sprintf("\"%v\"", n))) @@ -635,39 +678,6 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p return lst } -func (mw *XssMw) reiterateInterface(vv interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { - switch vvv := vv.(type) { - // map[string]interface {}{"id":"1", "assoc_ids":[]interface {}{"1", "4", "8"}} - case map[string]interface{}: - //fmt.Println("IN DEFAULT I don't know how to handle") - - case []interface{}: - //var b bytes.Buffer - b := mw.reiterateInterfaceSlice(vvv, buff, p) - buff.WriteString(b.String()) - buff.WriteByte(',') // add cause expected - case json.Number: - //fmt.Println("is number", vvv) - //buff.WriteString(`"` + p.Sanitize(vv) + `",`) - buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) - buff.WriteByte(',') - default: - //fmt.Println("IN DEFAULT I don't know how to handle") - //var r = reflect.TypeOf(vvv) - //fmt.Printf("IN DEFAULT Unknown Type!:%v\n", r) - - if vvv == nil { - buff.WriteString(fmt.Sprintf("%s", "null")) - } else { - buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) - } - buff.WriteByte(',') - } - - //fmt.Printf("Reiterate Interface %v", buff.String()) - return buff -} - // TODO // add feature to accept all content in Request and filter out on Response // NOTE: I don't know how to achieve this yet... will something like this help? diff --git a/xss_test.go b/xss_test.go index b6897fa..8ddd649 100644 --- a/xss_test.go +++ b/xss_test.go @@ -194,10 +194,11 @@ func TestSupportNestedJSONPost(t *testing.T) { ] }` - expect := fmt.Sprintf(expStr, user1, email1, password1, cmnt, cre_at, user2, email2, password2, cmnt, cre_at) - fmt.Println(expect) + cmnt_clnd := `` // malicious markup content stripped + expect := fmt.Sprintf(expStr, user1, email1, password1, cmnt_clnd, cre_at, user2, email2, password2, cmnt_clnd, cre_at) + //fmt.Println(expect) - fmt.Println(resp.Body.String()) + //fmt.Println(resp.Body.String()) assert.JSONEq(t, expect, resp.Body.String()) } From 005ef05d6d17e414e239a529c406f88856e83f72 Mon Sep 17 00:00:00 2001 From: David Wright Date: Tue, 23 Oct 2018 22:33:03 -0700 Subject: [PATCH 5/7] move coverage for nested json --- xss.go | 95 ++++++++++++++++++++++++++++++--------------------- xss_test.go | 98 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 39 deletions(-) diff --git a/xss.go b/xss.go index 70b64ab..ca29a37 100644 --- a/xss.go +++ b/xss.go @@ -538,7 +538,7 @@ func (mw *XssMw) ConstructJson(xmj XssMwJson) bytes.Buffer { buff.Truncate(buff.Len() - 1) // remove last ',' buff.WriteByte('}') - //fmt.Println("TOP CONStruct JSON") + //fmt.Println("TOP CONSTRUCT JSON") //fmt.Println(buff.String()) return buff } @@ -568,9 +568,12 @@ func (mw *XssMw) reiterateInterface(vv interface{}, buff bytes.Buffer, p *bluemo switch vvv := vv.(type) { // map[string]interface {}{"id":"1", "assoc_ids":[]interface {}{"1", "4", "8"}} case map[string]interface{}: - //fmt.Println("IN DEFAULT I don't know how to handle") - + //fmt.Println("map[string]interface{}") + scnd := mw.reiterateMapStringInterfaceSlice(vvv, buff, p) + buff.WriteString(scnd.String()) + buff.WriteByte(',') // add cause expected case []interface{}: + //fmt.Println("[]interface{}") //var b bytes.Buffer b := mw.reiterateInterfaceSlice(vvv, buff, p) buff.WriteString(b.String()) @@ -580,6 +583,12 @@ func (mw *XssMw) reiterateInterface(vv interface{}, buff bytes.Buffer, p *bluemo //buff.WriteString(`"` + p.Sanitize(vv) + `",`) buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) buff.WriteByte(',') + case string: + //fmt.Println("is string", vvv) + buff.WriteByte('"') + buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) + buff.WriteByte('"') + buff.WriteByte(',') default: //fmt.Println("IN DEFAULT I don't know how to handle") //var r = reflect.TypeOf(vvv) @@ -610,44 +619,9 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p //fmt.Printf("REiterate Type!:%v\n", r) switch nn := n.(type) { case map[string]interface{}: - //fmt.Println("MAPY map[string]interface{}") - ////moo := mw.reiterateInterface(nn, buff, p) - ////lst.WriteString(moo.String()) - var scnd bytes.Buffer - scnd.WriteByte('{') - Top: - for i, nnn := range nn { - scnd.WriteByte('"') - scnd.WriteString(i) - scnd.WriteByte('"') - scnd.WriteByte(':') - - for _, fts := range mw.FieldsToSkip { - if string(i) == fts { - scnd.WriteString(fmt.Sprintf("%q", nnn)) - scnd.WriteByte(',') - continue Top - } - } - - //var r = reflect.TypeOf(nnn) - //fmt.Printf("REiterate Type!:%v\n", r) - switch nnnn := nnn.(type) { - case json.Number: - scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) - - default: - scnd.WriteByte('"') - scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) - scnd.WriteByte('"') - } - scnd.WriteByte(',') - } - scnd.Truncate(scnd.Len() - 1) // remove last ',' - scnd.WriteByte('}') + scnd := mw.reiterateMapStringInterfaceSlice(nn, lst, p) lst.WriteString(scnd.String()) lst.WriteByte(',') // add cause expected - case []interface{}: //fmt.Println("MAPY []interface{}") lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) @@ -678,6 +652,49 @@ func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p return lst } +func (mw *XssMw) reiterateMapStringInterfaceSlice(nn map[string]interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { + //fmt.Println("MAPY map[string]interface{}") + ////moo := mw.reiterateInterface(nn, buff, p) + ////lst.WriteString(moo.String()) + var scnd bytes.Buffer + scnd.WriteByte('{') +Top: + for i, nnn := range nn { + scnd.WriteByte('"') + scnd.WriteString(i) + scnd.WriteByte('"') + scnd.WriteByte(':') + + for _, fts := range mw.FieldsToSkip { + if string(i) == fts { + scnd.WriteString(fmt.Sprintf("%q", nnn)) + scnd.WriteByte(',') + continue Top + } + } + + //var r = reflect.TypeOf(nnn) + //fmt.Printf("REiterate Type!:%v\n", r) + var apndBuff bytes.Buffer + apndBuff = mw.reiterateInterface(nnn, apndBuff, p) + scnd.WriteString(apndBuff.String()) + + //switch nnnn := nnn.(type) { + //case json.Number: + // scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) + + //default: + // scnd.WriteByte('"') + // scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) + // scnd.WriteByte('"') + //} + //scnd.WriteByte(',') + } + scnd.Truncate(scnd.Len() - 1) // remove last ',' + scnd.WriteByte('}') + return scnd +} + // TODO // add feature to accept all content in Request and filter out on Response // NOTE: I don't know how to achieve this yet... will something like this help? diff --git a/xss_test.go b/xss_test.go index 8ddd649..4123a0c 100644 --- a/xss_test.go +++ b/xss_test.go @@ -177,6 +177,104 @@ func TestSupportNestedJSONPost(t *testing.T) { userB := `{"id":2, "flt":2.345, "user":"` + user2 + `", "email": "` + email2 + `", "password":"` + password2 + `", "comment":"` + cmnt + `", "cre_at":` + cre_at + `}` oParams := `{"id":1, "users": [ ` + userA + `,` + userB + `]}` + // YES Can Parse This + //oParams = `{ + //"id": "0001", + //"type": "donut", + //"name": "Cake", + //"ppu": 0.55, + //"batters": + // { + // "batter": + // [ + // { "id": "1001", "type": "Regular" }, + // { "id": "1002", "type": "Chocolate" }, + // { "id": "1003", "type": "Blueberry" }, + // { "id": "1004", "type": "Devil's Food" } + // ] + // }, + //"topping": + // [ + // { "id": "5001", "type": "None" }, + // { "id": "5002", "type": "Glazed" }, + // { "id": "5005", "type": "Sugar" }, + // { "id": "5007", "type": "Powdered Sugar" }, + // { "id": "5006", "type": "Chocolate with Sprinkles" }, + // { "id": "5003", "type": "Chocolate" }, + // { "id": "5004", "type": "Maple" } + // ] + //}` + // YES - Parses all correctly BUT does not collect all together, so only last record is returned + //oParams = `[ + //{ + // "id": "0001", + // "type": "donut", + // "name": "Cake", + // "ppu": 0.55, + // "batters": + // { + // "batter": + // [ + // { "id": "1001", "type": "Regular" }, + // { "id": "1002", "type": "Chocolate" }, + // { "id": "1003", "type": "Blueberry" }, + // { "id": "1004", "type": "Devil's Food" } + // ] + // }, + // "topping": + // [ + // { "id": "5001", "type": "None" }, + // { "id": "5002", "type": "Glazed" }, + // { "id": "5005", "type": "Sugar" }, + // { "id": "5007", "type": "Powdered Sugar" }, + // { "id": "5006", "type": "Chocolate with Sprinkles" }, + // { "id": "5003", "type": "Chocolate" }, + // { "id": "5004", "type": "Maple" } + // ] + //}, + //{ + // "id": "0002", + // "type": "donut", + // "name": "Raised", + // "ppu": 0.55, + // "batters": + // { + // "batter": + // [ + // { "id": "1001", "type": "Regular" } + // ] + // }, + // "topping": + // [ + // { "id": "5001", "type": "None" }, + // { "id": "5002", "type": "Glazed" }, + // { "id": "5005", "type": "Sugar" }, + // { "id": "5003", "type": "Chocolate" }, + // { "id": "5004", "type": "Maple" } + // ] + //}, + //{ + // "id": "0003", + // "type": "donut", + // "name": "Old Fashioned", + // "ppu": 0.55, + // "batters": + // { + // "batter": + // [ + // { "id": "1001", "type": "Regular" }, + // { "id": "1002", "type": "Chocolate" } + // ] + // }, + // "topping": + // [ + // { "id": "5001", "type": "None" }, + // { "id": "5002", "type": "Glazed" }, + // { "id": "5003", "type": "Chocolate" }, + // { "id": "5004", "type": "Maple" } + // ] + //} + //]` req, _ := http.NewRequest("POST", "/user_post_nested_json", bytes.NewBufferString(oParams)) req.Header.Add("Content-Type", "application/json") From 0d54f0447b8834d1a56b883442e18adeb6336520 Mon Sep 17 00:00:00 2001 From: David Wright Date: Sat, 27 Oct 2018 13:25:31 -0700 Subject: [PATCH 6/7] cleanup +supports nested examples created --- xss.go | 194 +++++++++------------------------------------------- xss_test.go | 2 +- 2 files changed, 33 insertions(+), 163 deletions(-) diff --git a/xss.go b/xss.go index ca29a37..20cb93a 100644 --- a/xss.go +++ b/xss.go @@ -421,6 +421,9 @@ func (mw *XssMw) HandleJson(c *gin.Context) error { if err != nil { return err } + //fmt.Println("DONE JSON") + //fmt.Println(buff.String()) + err = mw.SetRequestBodyJson(c, buff) if err != nil { //fmt.Println("Set request body failed") @@ -446,7 +449,8 @@ func (mw *XssMw) jsonToStringMap(buff bytes.Buffer, jsonBod interface{}) (bytes. case map[string]interface{}: //fmt.Printf("\n\n\n1st type\n\n\n") xmj := jsonBod.(map[string]interface{}) - buff := mw.ConstructJson(xmj) + var sbuff bytes.Buffer + buff := mw.ConstructJson(xmj, sbuff) return buff, nil // TODO: need a test to prove this case []interface{}: @@ -455,7 +459,8 @@ func (mw *XssMw) jsonToStringMap(buff bytes.Buffer, jsonBod interface{}) (bytes. for _, n := range jbt { //fmt.Printf("Item: %v= %v\n", i, n) xmj := n.(map[string]interface{}) - buff = mw.ConstructJson(xmj) + var sbuff bytes.Buffer + buff = mw.ConstructJson(xmj, sbuff) multiRec.WriteString(buff.String()) multiRec.WriteByte(',') } @@ -499,18 +504,14 @@ func (mw *XssMw) SetRequestBodyJson(c *gin.Context, buff bytes.Buffer) error { // keeps the good content to construct // returns the cleaned http request // Map to Bytes (struct to json string...) -func (mw *XssMw) ConstructJson(xmj XssMwJson) bytes.Buffer { - //fmt.Printf("JSON BOD: %#v\n", xmj) - - var buff bytes.Buffer +func (mw *XssMw) ConstructJson(xmj XssMwJson, buff bytes.Buffer) bytes.Buffer { + //var buff bytes.Buffer buff.WriteByte('{') p := mw.GetBlueMondayPolicy() - m := xmj //m := jsonBod.(map[string]interface{}) + m := xmj for k, v := range m { - //fmt.Println(k, v) - buff.WriteByte('"') buff.WriteString(k) buff.WriteByte('"') @@ -538,177 +539,46 @@ func (mw *XssMw) ConstructJson(xmj XssMwJson) bytes.Buffer { buff.Truncate(buff.Len() - 1) // remove last ',' buff.WriteByte('}') - //fmt.Println("TOP CONSTRUCT JSON") - //fmt.Println(buff.String()) - return buff -} - -func (mw *XssMw) buildJsonApplyPolicy(v interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { - switch vv := v.(type) { // FYI, JSON data is string or float - case string: - //fmt.Println(k, "is string", vv) - //buff.WriteString(`"` + p.Sanitize(vv) + `",`) - buff.WriteString(fmt.Sprintf("%q", p.Sanitize(vv))) - buff.WriteByte(',') - case float64: - //fmt.Println(k, "is float", vv) - //buff.WriteString(strconv.FormatFloat(vv, 'g', 0, 64)) - //buff.WriteString(html.EscapeString(strconv.FormatFloat(vv, 'g', 0, 64))) - buff.WriteString(p.Sanitize(strconv.FormatFloat(vv, 'g', 0, 64))) - buff.WriteByte(',') - default: - var b bytes.Buffer - apndBuff := mw.reiterateInterface(vv, b, p) - buff.WriteString(apndBuff.String()) - } return buff } -func (mw *XssMw) reiterateInterface(vv interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { - switch vvv := vv.(type) { - // map[string]interface {}{"id":"1", "assoc_ids":[]interface {}{"1", "4", "8"}} +func (mw *XssMw) buildJsonApplyPolicy(interf interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { + switch v := interf.(type) { case map[string]interface{}: - //fmt.Println("map[string]interface{}") - scnd := mw.reiterateMapStringInterfaceSlice(vvv, buff, p) + var sbuff bytes.Buffer + scnd := mw.ConstructJson(v, sbuff) buff.WriteString(scnd.String()) - buff.WriteByte(',') // add cause expected + buff.WriteByte(',') case []interface{}: - //fmt.Println("[]interface{}") - //var b bytes.Buffer - b := mw.reiterateInterfaceSlice(vvv, buff, p) + b := mw.unravelSlice(v) buff.WriteString(b.String()) - buff.WriteByte(',') // add cause expected + buff.WriteByte(',') case json.Number: - //fmt.Println("is number", vvv) - //buff.WriteString(`"` + p.Sanitize(vv) + `",`) - buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) + buff.WriteString(p.Sanitize(fmt.Sprintf("%v", v))) buff.WriteByte(',') case string: - //fmt.Println("is string", vvv) - buff.WriteByte('"') - buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) - buff.WriteByte('"') + buff.WriteString(fmt.Sprintf("%q", p.Sanitize(v))) buff.WriteByte(',') - default: - //fmt.Println("IN DEFAULT I don't know how to handle") - //var r = reflect.TypeOf(vvv) - //fmt.Printf("IN DEFAULT Unknown Type!:%v\n", r) - - if vvv == nil { - buff.WriteString(fmt.Sprintf("%s", "null")) - } else { - buff.WriteString(p.Sanitize(fmt.Sprintf("%v", vvv))) - } + case float64: + buff.WriteString(p.Sanitize(strconv.FormatFloat(v, 'g', 0, 64))) buff.WriteByte(',') } - - //fmt.Printf("Reiterate Interface %v", buff.String()) return buff } -func (mw *XssMw) reiterateInterfaceSlice(vvv []interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { - //fmt.Println("IN reiterateInterfaceSlice") - //switch vvvv := vvv.(type) { - //case map[string]interface{}: - var lst bytes.Buffer - lst.WriteByte('[') - for _, n := range vvv { - //fmt.Printf("Iter: %v= %v\n", i, n) - //fmt.Println("IN REITERATE") - //var r = reflect.TypeOf(n) - //fmt.Printf("REiterate Type!:%v\n", r) +func (mw *XssMw) unravelSlice(slce []interface{}) bytes.Buffer { + var buff bytes.Buffer + buff.WriteByte('[') + for _, n := range slce { switch nn := n.(type) { case map[string]interface{}: - scnd := mw.reiterateMapStringInterfaceSlice(nn, lst, p) - lst.WriteString(scnd.String()) - lst.WriteByte(',') // add cause expected - case []interface{}: - //fmt.Println("MAPY []interface{}") - lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) - case json.Number: - //fmt.Println("json.Number") - lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) - case string: - //fmt.Println("float 64") - lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) - case float64: - //fmt.Println("float 64") - lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) - default: - //fmt.Println("DEFAULT") - //fmt.Printf("nn %v", nn) - } - - //lst.WriteString(p.Sanitize(fmt.Sprintf("\"%v\"", n))) - // NOTE changes from ["1", "4", "8"] to [1,4,8] - //lst.WriteString(p.Sanitize(fmt.Sprintf("%v", n))) - //lst.WriteByte(',') - } - lst.Truncate(lst.Len() - 1) // remove last ',' - lst.WriteByte(']') - //default: - // fmt.Printf("REiterate Default") - //} - return lst -} - -func (mw *XssMw) reiterateMapStringInterfaceSlice(nn map[string]interface{}, buff bytes.Buffer, p *bluemonday.Policy) bytes.Buffer { - //fmt.Println("MAPY map[string]interface{}") - ////moo := mw.reiterateInterface(nn, buff, p) - ////lst.WriteString(moo.String()) - var scnd bytes.Buffer - scnd.WriteByte('{') -Top: - for i, nnn := range nn { - scnd.WriteByte('"') - scnd.WriteString(i) - scnd.WriteByte('"') - scnd.WriteByte(':') - - for _, fts := range mw.FieldsToSkip { - if string(i) == fts { - scnd.WriteString(fmt.Sprintf("%q", nnn)) - scnd.WriteByte(',') - continue Top - } + var sbuff bytes.Buffer + scnd := mw.ConstructJson(nn, sbuff) + buff.WriteString(scnd.String()) + buff.WriteByte(',') } - - //var r = reflect.TypeOf(nnn) - //fmt.Printf("REiterate Type!:%v\n", r) - var apndBuff bytes.Buffer - apndBuff = mw.reiterateInterface(nnn, apndBuff, p) - scnd.WriteString(apndBuff.String()) - - //switch nnnn := nnn.(type) { - //case json.Number: - // scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) - - //default: - // scnd.WriteByte('"') - // scnd.WriteString(p.Sanitize(fmt.Sprintf("%v", nnnn))) - // scnd.WriteByte('"') - //} - //scnd.WriteByte(',') } - scnd.Truncate(scnd.Len() - 1) // remove last ',' - scnd.WriteByte('}') - return scnd + buff.Truncate(buff.Len() - 1) // remove last ',' + buff.WriteByte(']') + return buff } - -// TODO -// add feature to accept all content in Request and filter out on Response -// NOTE: I don't know how to achieve this yet... will something like this help? -// gin help said call Next first to capture Response -//func ConstructRequest(next http.Handler) http.Handler { -// return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { -// fmt.Println(r.Method, "-", r.RequestURI) -// cookie, _ := r.Cookie("username") -// if cookie != nil { -// //Add data to context -// ctx := context.WithValue(r.Context(), "Username", cookie.Value) -// next.ServeHTTP(w, r.WithContext(ctx)) -// } else { -// next.ServeHTTP(w, r) -// } -// }) -//} diff --git a/xss_test.go b/xss_test.go index 4123a0c..1c7c11d 100644 --- a/xss_test.go +++ b/xss_test.go @@ -204,7 +204,7 @@ func TestSupportNestedJSONPost(t *testing.T) { // { "id": "5004", "type": "Maple" } // ] //}` - // YES - Parses all correctly BUT does not collect all together, so only last record is returned + // YES - Parses correctly //oParams = `[ //{ // "id": "0001", From 6c153c0433c50ce492dc46d652c1341ec3cde8fa Mon Sep 17 00:00:00 2001 From: David Wright Date: Sat, 27 Oct 2018 17:30:00 -0700 Subject: [PATCH 7/7] put back support for null values --- xss.go | 8 ++++++++ xss_test.go | 1 + 2 files changed, 9 insertions(+) diff --git a/xss.go b/xss.go index 20cb93a..25fe13c 100644 --- a/xss.go +++ b/xss.go @@ -562,6 +562,14 @@ func (mw *XssMw) buildJsonApplyPolicy(interf interface{}, buff bytes.Buffer, p * case float64: buff.WriteString(p.Sanitize(strconv.FormatFloat(v, 'g', 0, 64))) buff.WriteByte(',') + default: + if v == nil { + buff.WriteString(fmt.Sprintf("%s", "null")) + buff.WriteByte(',') + } else { + buff.WriteString(p.Sanitize(fmt.Sprintf("%v", v))) + buff.WriteByte(',') + } } return buff } diff --git a/xss_test.go b/xss_test.go index 1c7c11d..9edff82 100644 --- a/xss_test.go +++ b/xss_test.go @@ -275,6 +275,7 @@ func TestSupportNestedJSONPost(t *testing.T) { // ] //} //]` + //oParams = `{"comment": "test", "id": 0, "media_id": 381, "parent_id": 0, "status": null, "updated_at": 1540673110}` req, _ := http.NewRequest("POST", "/user_post_nested_json", bytes.NewBufferString(oParams)) req.Header.Add("Content-Type", "application/json")