From e295a4f7871eaaa17072232f98ebf5b9775bbb0d Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Tue, 27 Jun 2023 12:41:48 +0300 Subject: [PATCH 1/5] put null helpers back --- plugin/nulls.go | 72 +++++++++++++++++++++++++++++++++++++++++ plugin/testing_write.go | 13 ++++---- 2 files changed, 79 insertions(+), 6 deletions(-) create mode 100644 plugin/nulls.go diff --git a/plugin/nulls.go b/plugin/nulls.go new file mode 100644 index 0000000000..e60a88a24f --- /dev/null +++ b/plugin/nulls.go @@ -0,0 +1,72 @@ +package plugin + +import ( + "github.com/apache/arrow/go/v13/arrow" + "github.com/apache/arrow/go/v13/arrow/array" + "github.com/apache/arrow/go/v13/arrow/memory" +) + +func stripNullsFromLists(records []arrow.Record) { + for i := range records { + cols := records[i].Columns() + for c, col := range cols { + if col.DataType().ID() != arrow.LIST { + continue + } + + list := col.(*array.List) + bldr := array.NewListBuilder(memory.DefaultAllocator, list.DataType().(*arrow.ListType).Elem()) + for j := 0; j < list.Len(); j++ { + if list.IsNull(j) { + bldr.AppendNull() + continue + } + bldr.Append(true) + vBldr := bldr.ValueBuilder() + from, to := list.ValueOffsets(j) + slc := array.NewSlice(list.ListValues(), from, to) + for k := 0; k < int(to-from); k++ { + if slc.IsNull(k) { + continue + } + err := vBldr.AppendValueFromString(slc.ValueStr(k)) + if err != nil { + panic(err) + } + } + } + cols[c] = bldr.NewArray() + } + records[i] = array.NewRecord(records[i].Schema(), cols, records[i].NumRows()) + } +} + +type AllowNullFunc func(arrow.DataType) bool + +func (f AllowNullFunc) replaceNullsByEmpty(records []arrow.Record) { + if f == nil { + return + } + for i := range records { + cols := records[i].Columns() + for c, col := range records[i].Columns() { + if col.NullN() == 0 || f(col.DataType()) { + continue + } + + builder := array.NewBuilder(memory.DefaultAllocator, records[i].Column(c).DataType()) + for j := 0; j < col.Len(); j++ { + if col.IsNull(j) { + builder.AppendEmptyValue() + continue + } + + if err := builder.AppendValueFromString(col.ValueStr(j)); err != nil { + panic(err) + } + } + cols[c] = builder.NewArray() + } + records[i] = array.NewRecord(records[i].Schema(), cols, records[i].NumRows()) + } +} diff --git a/plugin/testing_write.go b/plugin/testing_write.go index 955de9991e..fa30575d24 100644 --- a/plugin/testing_write.go +++ b/plugin/testing_write.go @@ -4,6 +4,7 @@ import ( "context" "testing" + "github.com/apache/arrow/go/v13/arrow" "github.com/cloudquery/plugin-sdk/v4/schema" ) @@ -16,7 +17,7 @@ type WriterTestSuite struct { // Destinations that have problems representing some data types should provide a custom implementation here. // If this param is empty, the default is to allow all data types to be nullable. // When the value returned by this func is `true` the comparison is made with the empty value instead of null. - // allowNull AllowNullFunc + allowNull AllowNullFunc // IgnoreNullsInLists allows stripping null values from lists before comparison. // Destination setups that don't support nulls in lists should set this to true. @@ -56,11 +57,11 @@ type WriterTestSuiteTests struct { type NewPluginFunc func() *Plugin -// func WithTestSourceAllowNull(allowNull func(arrow.DataType) bool) func(o *WriterTestSuite) { -// return func(o *WriterTestSuite) { -// o.allowNull = allowNull -// } -// } +func WithTestSourceAllowNull(allowNull func(arrow.DataType) bool) func(o *WriterTestSuite) { + return func(o *WriterTestSuite) { + o.allowNull = allowNull + } +} func WithTestIgnoreNullsInLists() func(o *WriterTestSuite) { return func(o *WriterTestSuite) { From 4fa4143195c19d1d01dfdcae723d676cf35c9072 Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Tue, 27 Jun 2023 12:45:30 +0300 Subject: [PATCH 2/5] //nolint:unused --- plugin/nulls.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/plugin/nulls.go b/plugin/nulls.go index e60a88a24f..05922260a0 100644 --- a/plugin/nulls.go +++ b/plugin/nulls.go @@ -6,6 +6,9 @@ import ( "github.com/apache/arrow/go/v13/arrow/memory" ) +// TODO: use in v4 +// +//nolint:unused func stripNullsFromLists(records []arrow.Record) { for i := range records { cols := records[i].Columns() @@ -43,6 +46,9 @@ func stripNullsFromLists(records []arrow.Record) { type AllowNullFunc func(arrow.DataType) bool +// TODO: use in v4 +// +//nolint:unused func (f AllowNullFunc) replaceNullsByEmpty(records []arrow.Record) { if f == nil { return From 798e1efad28fc2af9d9b39c0ae2d331bcce80f08 Mon Sep 17 00:00:00 2001 From: Alex Shcherbakov Date: Thu, 29 Jun 2023 21:40:23 +0300 Subject: [PATCH 3/5] Update plugin/nulls.go Co-authored-by: Herman Schaaf --- plugin/nulls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/nulls.go b/plugin/nulls.go index 05922260a0..11a8089f5d 100644 --- a/plugin/nulls.go +++ b/plugin/nulls.go @@ -6,7 +6,7 @@ import ( "github.com/apache/arrow/go/v13/arrow/memory" ) -// TODO: use in v4 +// TODO(v4): use in v4 // //nolint:unused func stripNullsFromLists(records []arrow.Record) { From 3108343812c8f3f696862d3f705b14e45b8f372f Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 29 Jun 2023 21:40:55 +0300 Subject: [PATCH 4/5] upd todo --- plugin/nulls.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugin/nulls.go b/plugin/nulls.go index 11a8089f5d..4f52dac81e 100644 --- a/plugin/nulls.go +++ b/plugin/nulls.go @@ -46,7 +46,7 @@ func stripNullsFromLists(records []arrow.Record) { type AllowNullFunc func(arrow.DataType) bool -// TODO: use in v4 +// TODO(v4): use in v4 // //nolint:unused func (f AllowNullFunc) replaceNullsByEmpty(records []arrow.Record) { From f5f62091db2cb756bc9d3a9847b613f4c32d7e7a Mon Sep 17 00:00:00 2001 From: candiduslynx Date: Thu, 29 Jun 2023 21:42:32 +0300 Subject: [PATCH 5/5] lint --- plugin/nulls.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugin/nulls.go b/plugin/nulls.go index 4f52dac81e..8a33e5ca1e 100644 --- a/plugin/nulls.go +++ b/plugin/nulls.go @@ -8,7 +8,7 @@ import ( // TODO(v4): use in v4 // -//nolint:unused +// nolint:unused func stripNullsFromLists(records []arrow.Record) { for i := range records { cols := records[i].Columns() @@ -48,7 +48,7 @@ type AllowNullFunc func(arrow.DataType) bool // TODO(v4): use in v4 // -//nolint:unused +// nolint:unused func (f AllowNullFunc) replaceNullsByEmpty(records []arrow.Record) { if f == nil { return