Skip to content

Commit

Permalink
Add csv parser unix timestamp support (influxdata#5047)
Browse files Browse the repository at this point in the history
  • Loading branch information
tkanos authored and Jean-Louis Dupond committed Apr 22, 2019
1 parent b74d531 commit 2afdaf0
Show file tree
Hide file tree
Showing 3 changed files with 58 additions and 16 deletions.
3 changes: 2 additions & 1 deletion plugins/parsers/csv/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -75,7 +75,8 @@ document.

The `csv_timestamp_column` option specifies the column name containing the
time value and `csv_timestamp_format` must be set to a Go "reference time"
which is defined to be the specific time: `Mon Jan 2 15:04:05 MST 2006`.
which is defined to be the specific time: `Mon Jan 2 15:04:05 MST 2006`,
it can also be `unix` (for epoch in ms format like 1257894000 )

Consult the Go [time][time parse] package for details and additional examples
on how to set the time format.
Expand Down
53 changes: 38 additions & 15 deletions plugins/parsers/csv/parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -207,21 +207,9 @@ outer:
measurementName = fmt.Sprintf("%v", recordFields[p.MeasurementColumn])
}

metricTime := p.TimeFunc()
if p.TimestampColumn != "" {
if recordFields[p.TimestampColumn] == nil {
return nil, fmt.Errorf("timestamp column: %v could not be found", p.TimestampColumn)
}
tStr := fmt.Sprintf("%v", recordFields[p.TimestampColumn])
if p.TimestampFormat == "" {
return nil, fmt.Errorf("timestamp format must be specified")
}

var err error
metricTime, err = time.Parse(p.TimestampFormat, tStr)
if err != nil {
return nil, err
}
metricTime, err := parseTimestamp(p.TimeFunc, recordFields, p.TimestampColumn, p.TimestampFormat)
if err != nil {
return nil, err
}

m, err := metric.New(measurementName, tags, recordFields, metricTime)
Expand All @@ -231,6 +219,41 @@ outer:
return m, nil
}

// ParseTimestamp return a timestamp, if there is no timestamp on the csv it will be the current timestamp, else it will try to parse the time according to the format
// if the format is "unix" it tries to parse assuming that on the csv it will find an epoch in ms.
func parseTimestamp(timeFunc func() time.Time, recordFields map[string]interface{}, timestampColumn, timestampFormat string) (metricTime time.Time, err error) {
metricTime = timeFunc()

if timestampColumn != "" {
if recordFields[timestampColumn] == nil {
err = fmt.Errorf("timestamp column: %v could not be found", timestampColumn)
return
}

tStr := fmt.Sprintf("%v", recordFields[timestampColumn])

switch timestampFormat {
case "":
err = fmt.Errorf("timestamp format must be specified")
return
case "unix":
var unixTime int64
unixTime, err = strconv.ParseInt(tStr, 10, 64)
if err != nil {
return
}
metricTime = time.Unix(unixTime, 0)
default:
metricTime, err = time.Parse(timestampFormat, tStr)
if err != nil {
return
}
}
}
return
}

// SetDefaultTags set the DefaultTags
func (p *Parser) SetDefaultTags(tags map[string]string) {
p.DefaultTags = tags
}
18 changes: 18 additions & 0 deletions plugins/parsers/csv/parser_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,24 @@ func TestTimestampError(t *testing.T) {
require.Equal(t, fmt.Errorf("timestamp format must be specified"), err)
}

func TestTimestampUnixFormat(t *testing.T) {
p := Parser{
HeaderRowCount: 1,
ColumnNames: []string{"first", "second", "third"},
MeasurementColumn: "third",
TimestampColumn: "first",
TimestampFormat: "unix",
TimeFunc: DefaultTime,
}
testCSV := `line1,line2,line3
1243094706,70,test_name
1257609906,80,test_name2`
metrics, err := p.Parse([]byte(testCSV))
require.NoError(t, err)
require.Equal(t, metrics[0].Time().UnixNano(), int64(1243094706000000000))
require.Equal(t, metrics[1].Time().UnixNano(), int64(1257609906000000000))
}

func TestQuotedCharacter(t *testing.T) {
p := Parser{
HeaderRowCount: 1,
Expand Down

0 comments on commit 2afdaf0

Please sign in to comment.