Skip to content

Commit

Permalink
serialization and deserialization for tabular data resource
Browse files Browse the repository at this point in the history
  • Loading branch information
@colombod
colombod committed Jan 9, 2023
1 parent 7a62c37 commit 60269e9
Show file tree
Hide file tree
Showing 28 changed files with 661 additions and 128 deletions.
11 changes: 2 additions & 9 deletions ...e.ApiCompatibility.Tests/ApiCompatibilityTests.Formatting_api_is_not_changed.approved.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,7 @@ Microsoft.DotNet.Interactive.Formatting
public abstract class ITypeFormatterSource
public System.Collections.Generic.IEnumerable<ITypeFormatter> CreateTypeFormatters()
public abstract class JsonConverter<T> : JsonConverter<T>
protected System.Void EnsureStartArray(System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert)
protected System.Void EnsureStartObject(System.Text.Json.Utf8JsonReader reader, System.Type typeToConvert)
protected System.Void OnWrite(System.Text.Json.Utf8JsonWriter writer, T value, System.Text.Json.JsonSerializerOptions options)
public System.Void Write(System.Text.Json.Utf8JsonWriter writer, T value, System.Text.Json.JsonSerializerOptions options)
Expand Down Expand Up @@ -314,20 +315,12 @@ Microsoft.DotNet.Interactive.Formatting.TabularData
DateTime=8
GeoPoint=9
GeoJson=10
public class TableSchemaFieldTypeConverter : Microsoft.DotNet.Interactive.Formatting.JsonConverter<TableSchemaFieldType>
.ctor()
public TableSchemaFieldType Read(ref System.Text.Json.Utf8JsonReader& reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options)
public System.Void Write(System.Text.Json.Utf8JsonWriter writer, TableSchemaFieldType value, System.Text.Json.JsonSerializerOptions options)
public class TabularDataResource
.ctor(TableSchema schema, System.Collections.Generic.IEnumerable<System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<System.String,System.Object>>> data)
.ctor(TableSchema schema, System.Collections.Generic.IEnumerable<System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<System.String,System.Object>>> data, System.String profile = null)
public System.Collections.Generic.IEnumerable<System.Collections.Generic.IEnumerable<System.Collections.Generic.KeyValuePair<System.String,System.Object>>> Data { get;}
public System.String Profile { get;}
public TableSchema Schema { get;}
public TabularDataResourceJsonString ToJsonString()
public class TabularDataResourceConverter : Microsoft.DotNet.Interactive.Formatting.JsonConverter<TabularDataResource>
.ctor()
public TabularDataResource Read(ref System.Text.Json.Utf8JsonReader& reader, System.Type typeToConvert, System.Text.Json.JsonSerializerOptions options)
public System.Void Write(System.Text.Json.Utf8JsonWriter writer, TabularDataResource value, System.Text.Json.JsonSerializerOptions options)
public static class TabularDataResourceFormatter
public static System.Text.Json.JsonSerializerOptions JsonSerializerOptions { get;}
public static TableSchemaFieldType ToTableSchemaFieldType()
Expand Down
16 changes: 8 additions & 8 deletions ...ve.ExtensionLab.Tests/DataFrameKernelExtensionTests.it_registers_formatters.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@
"primaryKey": [],
"fields": [
{
"name": "id",
"type": "number"
"type": "number",
"name": "id"
},
{
"name": "name",
"type": "string"
"type": "string",
"name": "name"
},
{
"name": "color",
"type": "string"
"type": "string",
"name": "color"
},
{
"name": "deliciousness",
"type": "number"
"type": "number",
"name": "deliciousness"
}
]
},
Expand Down
8 changes: 4 additions & 4 deletions ...ourceFormatterTests.can_generate_tabular_json_from_data_with_nullable_types.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
"primaryKey": [],
"fields": [
{
"name": "Key",
"type": "string"
"type": "string",
"name": "Key"
},
{
"name": "Value",
"type": "integer"
"type": "integer",
"name": "Value"
}
]
},
Expand Down
8 changes: 4 additions & 4 deletions ...TabularDataResourceFormatterTests.can_generate_tabular_json_from_dictionary.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,12 @@
"primaryKey": [],
"fields": [
{
"name": "Key",
"type": "string"
"type": "string",
"name": "Key"
},
{
"name": "Value",
"type": "integer"
"type": "integer",
"name": "Value"
}
]
},
Expand Down
12 changes: 6 additions & 6 deletions ...bularDataResourceFormatterTests.can_generate_tabular_json_from_object_array.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@
"primaryKey": [],
"fields": [
{
"name": "Name",
"type": "string"
"type": "string",
"name": "Name"
},
{
"name": "IsValid",
"type": "boolean"
"type": "boolean",
"name": "IsValid"
},
{
"name": "Cost",
"type": "number"
"type": "number",
"name": "Cost"
}
]
},
Expand Down
16 changes: 8 additions & 8 deletions ...Tests.can_generate_tabular_json_from_sequence_of_sequences_of_KeyValuePairs.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@
"primaryKey": [],
"fields": [
{
"name": "id",
"type": "integer"
"type": "integer",
"name": "id"
},
{
"name": "name",
"type": "string"
"type": "string",
"name": "name"
},
{
"name": "color",
"type": "string"
"type": "string",
"name": "color"
},
{
"name": "deliciousness",
"type": "integer"
"type": "integer",
"name": "deliciousness"
}
]
},
Expand Down
16 changes: 8 additions & 8 deletions ...erTests.can_generate_tabular_json_from_sequence_of_sequences_of_ValueTuples.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,20 @@
"primaryKey": [],
"fields": [
{
"name": "id",
"type": "integer"
"type": "integer",
"name": "id"
},
{
"name": "name",
"type": "string"
"type": "string",
"name": "name"
},
{
"name": "color",
"type": "string"
"type": "string",
"name": "color"
},
{
"name": "deliciousness",
"type": "integer"
"type": "integer",
"name": "deliciousness"
}
]
},
Expand Down
12 changes: 6 additions & 6 deletions ...FormatterTests.can_generate_tabular_json_when_non_numeric_literals_are_used.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,16 @@
"primaryKey": [],
"fields": [
{
"name": "Name",
"type": "string"
"type": "string",
"name": "Name"
},
{
"name": "IsValid",
"type": "boolean"
"type": "boolean",
"name": "IsValid"
},
{
"name": "Cost",
"type": "number"
"type": "number",
"name": "Cost"
}
]
},
Expand Down
98 changes: 88 additions & 10 deletions src/Microsoft.DotNet.Interactive.Formatting.Tests/TabularDataResourceTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Text.Json;

using FluentAssertions;
Expand All @@ -26,7 +25,7 @@ public void can_create_from_JsonDocument()
Fields = new TableDataFieldDescriptors
{
new("name", TableSchemaFieldType.String),
new("age", TableSchemaFieldType.Number),
new("age", TableSchemaFieldType.Integer),
new("salary", TableSchemaFieldType.Number),
new("active", TableSchemaFieldType.Boolean),
}
Expand All @@ -53,14 +52,14 @@ public void schema_is_inferred_correctly_when_leading_data_item_has_null_field()
var tabularDataResource = JsonDocument.Parse(@"
[
{
""name"": ""Granny Smith apple"",
""deliciousness"": null,
""color"":null
""name"": ""Granny Smith apple"",
""deliciousness"": null,
""color"":null
},
{
""name"": ""Rainier cherry"",
""deliciousness"": 9000,
""color"":""yellow""
{
""name"": ""Rainier cherry"",
""deliciousness"": 9000,
""color"":""yellow""
}
]").ToTabularDataResource();

Expand All @@ -72,11 +71,90 @@ public void schema_is_inferred_correctly_when_leading_data_item_has_null_field()
tabularDataResource
.Schema
.Fields["deliciousness"]
.Type.Should().Be(TableSchemaFieldType.Number);
.Type.Should().Be(TableSchemaFieldType.Integer);

tabularDataResource
.Schema
.Fields["color"]
.Type.Should().Be(TableSchemaFieldType.String);
}

[Fact]
public void can_be_deserialized_from_json()
{
var json = @"
{ ""profile"": ""tabular-data-resource"",
""schema"": {
""primaryKey"": [],
""fields"": [
{
""name"": ""name"",
""type"": ""string""
},
{
""name"": ""age"",
""type"": ""integer""
},
{
""name"": ""salary"",
""type"": ""number""
},
{
""name"": ""active"",
""type"": ""boolean""
},
{
""name"": ""date"",
""type"": ""datetime""
},
{
""name"": ""summary"",
""type"": ""object""
},
{
""name"": ""list"",
""type"": ""array""
}
]
},
""data"" : [
{ ""name"": ""mitch"", ""age"": 42, ""salary"":10.0, ""active"":true, ""date"": ""2020-01-01"", ""summary"": { ""a"": 1, ""b"": 2 }, ""list"":[1,2,3,4] }
]
}";
var expected = new TabularDataResource(
new TableSchema
{
Fields = new TableDataFieldDescriptors
{
new("name", TableSchemaFieldType.String),
new("age", TableSchemaFieldType.Integer),
new("salary", TableSchemaFieldType.Number),
new("active", TableSchemaFieldType.Boolean),
new("date", TableSchemaFieldType.DateTime),
new("summary", TableSchemaFieldType.Object),
new("list", TableSchemaFieldType.Array)
}
},
new[]
{
new Dictionary<string, object>
{
["name"] = "mitch",
["age"] = 42,
["salary"] = 10.0,
["active"] = true,
["date"] = new DateTime(2020, 1, 1),
["summary"] = new Dictionary<string, object>
{
["a"] = 1,
["b"] = 2
},
["list"] = new object[]{1,2,3,4}
}
});

var actual = JsonSerializer.Deserialize<TabularDataResource>(json, JsonFormatter.SerializerOptions);

actual.Should().BeEquivalentTo(expected);
}
}
22 changes: 21 additions & 1 deletion src/Microsoft.DotNet.Interactive.Formatting/DataDictionaryConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ private static object GetValue(ref Utf8JsonReader reader, JsonSerializerOptions
itemValue = reader.GetString();
break;
case JsonTokenType.Number:
itemValue = reader.GetDouble();
itemValue = GetNumber(ref reader);
break;
case JsonTokenType.True:
itemValue = true;
Expand All @@ -63,6 +63,26 @@ private static object GetValue(ref Utf8JsonReader reader, JsonSerializerOptions
}

return itemValue;

static object GetNumber(ref Utf8JsonReader reader)
{
if (reader.TryGetInt32(out var integer))
{
return integer;
}

if (reader.TryGetInt64(out var longInt))
{
return longInt;
}

if (reader.TryGetDouble(out var doublePrecision))
{
return doublePrecision;
}

return reader.GetDecimal();
}
}

private static object ParseArray(ref Utf8JsonReader reader, JsonSerializerOptions options)
Expand Down
9 changes: 9 additions & 0 deletions src/Microsoft.DotNet.Interactive.Formatting/JsonConverter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,15 @@ protected void EnsureStartObject(Utf8JsonReader reader, Type typeToConvert)
}
}

protected void EnsureStartArray(Utf8JsonReader reader, Type typeToConvert)
{
if (reader.TokenType != JsonTokenType.StartArray)
{
throw new JsonException(
$"Cannot deserialize {typeToConvert.Name}, expecting {JsonTokenType.StartObject} but found {reader.TokenType}");
}
}

public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options)
{
OnWrite(writer, value, options);
Expand Down
Loading

0 comments on commit 60269e9

Please sign in to comment.