From 76a06e68b39044c27f00041ae19601bbd68a5b56 Mon Sep 17 00:00:00 2001 From: Felipe Dutra Tine e Silva Date: Wed, 28 Nov 2018 09:34:46 -0500 Subject: [PATCH 1/3] test: csv parser unix timestamp format --- plugins/parsers/csv/parser_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/plugins/parsers/csv/parser_test.go b/plugins/parsers/csv/parser_test.go index eff6f953f5651..97da69cd2f5f8 100644 --- a/plugins/parsers/csv/parser_test.go +++ b/plugins/parsers/csv/parser_test.go @@ -88,6 +88,24 @@ func TestTimestampError(t *testing.T) { require.Equal(t, fmt.Errorf("timestamp format must be specified"), err) } +func TestTimestampUnixFormat(t *testing.T) { + p := Parser{ + HeaderRowCount: 1, + ColumnNames: []string{"first", "second", "third"}, + MeasurementColumn: "third", + TimestampColumn: "first", + TimestampFormat: "unix", + TimeFunc: DefaultTime, + } + testCSV := `line1,line2,line3 +1243094706,70,test_name +1257609906,80,test_name2` + metrics, err := p.Parse([]byte(testCSV)) + require.NoError(t, err) + require.Equal(t, metrics[0].Time().UnixNano(), int64(1243094706000000000)) + require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906000000000)) +} + func TestQuotedCharacter(t *testing.T) { p := Parser{ HeaderRowCount: 1, From e3755fad11a508595b8b812edbfdd97d6d465385 Mon Sep 17 00:00:00 2001 From: Felipe Dutra Tine e Silva Date: Wed, 28 Nov 2018 09:58:51 -0500 Subject: [PATCH 2/3] feat: csv parsers can handle unix timestamp If we try to use unix timestamp in the csv file, we will get an error. This commit fixes that. * if the csv_timestamp_format = "unix" we try to parse the timestamp assuming that we will get an epoch in ms. * Put all the parsing timestamp part in a function (as it deserves it :D) --- plugins/parsers/csv/parser.go | 53 +++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 15 deletions(-) diff --git a/plugins/parsers/csv/parser.go b/plugins/parsers/csv/parser.go index f18068eb70075..e1bbdbbbb0cbb 100644 --- a/plugins/parsers/csv/parser.go +++ b/plugins/parsers/csv/parser.go @@ -207,21 +207,9 @@ outer: measurementName = fmt.Sprintf("%v", recordFields[p.MeasurementColumn]) } - metricTime := p.TimeFunc() - if p.TimestampColumn != "" { - if recordFields[p.TimestampColumn] == nil { - return nil, fmt.Errorf("timestamp column: %v could not be found", p.TimestampColumn) - } - tStr := fmt.Sprintf("%v", recordFields[p.TimestampColumn]) - if p.TimestampFormat == "" { - return nil, fmt.Errorf("timestamp format must be specified") - } - - var err error - metricTime, err = time.Parse(p.TimestampFormat, tStr) - if err != nil { - return nil, err - } + metricTime, err := parseTimestamp(p.TimeFunc, recordFields, p.TimestampColumn, p.TimestampFormat) + if err != nil { + return nil, err } m, err := metric.New(measurementName, tags, recordFields, metricTime) @@ -231,6 +219,41 @@ outer: return m, nil } +// ParseTimestamp return a timestamp, if there is no timestamp on the csv it will be the current timestamp, else it will try to parse the time according to the format +// if the format is "unix" it tries to parse assuming that on the csv it will find an epoch in ms. +func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{}, timestampColumn, timestampFormat string) (metricTime time.Time, err error) { + metricTime = timeFunc() + + if timestampColumn != "" { + if recordFields[timestampColumn] == nil { + err = fmt.Errorf("timestamp column: %v could not be found", timestampColumn) + return + } + + tStr := fmt.Sprintf("%v", recordFields[timestampColumn]) + + switch timestampFormat { + case "": + err = fmt.Errorf("timestamp format must be specified") + return + case "unix": + var unixTime int64 + unixTime, err = strconv.ParseInt(tStr, 10, 64) + if err != nil { + return + } + metricTime = time.Unix(unixTime, 0) + default: + metricTime, err = time.Parse(timestampFormat, tStr) + if err != nil { + return + } + } + } + return +} + +// SetDefaultTags set the DefaultTags func (p *Parser) SetDefaultTags(tags map[string]string) { p.DefaultTags = tags } From f2fdaae08c524cff3fe38327e3cb3eea9da7c366 Mon Sep 17 00:00:00 2001 From: Felipe Dutra Tine e Silva Date: Wed, 28 Nov 2018 10:03:08 -0500 Subject: [PATCH 3/3] doc: updating README for csv using unix timestamp --- plugins/parsers/csv/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/plugins/parsers/csv/README.md b/plugins/parsers/csv/README.md index e4cfbfc372842..f2cf34c691d7f 100644 --- a/plugins/parsers/csv/README.md +++ b/plugins/parsers/csv/README.md @@ -75,7 +75,8 @@ document. The `csv_timestamp_column` option specifies the column name containing the time value and `csv_timestamp_format` must be set to a Go "reference time" -which is defined to be the specific time: `Mon Jan 2 15:04:05 MST 2006`. +which is defined to be the specific time: `Mon Jan 2 15:04:05 MST 2006`, +it can also be `unix` (for epoch in ms format like 1257894000 ) Consult the Go [time][time parse] package for details and additional examples on how to set the time format.