From fc8dd28621ca918c8cb55da1e9c14b99e04a49fd Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Mon, 25 Oct 2021 09:33:13 -0400 Subject: [PATCH 01/22] Fixes #16558 CSV delimiter determiner --- modules/csv/csv.go | 29 ++++++++++++++++++++++++++--- modules/csv/csv_test.go | 5 +++-- routers/web/repo/compare.go | 9 +++++---- services/gitdiff/csv_test.go | 8 ++++---- 4 files changed, 38 insertions(+), 13 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 30698830a478..f186578ecbd3 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -9,9 +9,11 @@ import ( stdcsv "encoding/csv" "errors" "io" + "path/filepath" "regexp" "strings" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/translation" "code.gitea.io/gitea/modules/util" ) @@ -26,8 +28,8 @@ func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { return rd } -// CreateReaderAndGuessDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. -func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { +// CreateReaderAndDetermineDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. +func CreateReaderAndDetermineDelimiter(ctx *markup.RenderContext, rd io.Reader) (*stdcsv.Reader, error) { var data = make([]byte, 1e4) size, err := rd.Read(data) if err != nil { @@ -37,7 +39,7 @@ func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { return nil, err } - delimiter := guessDelimiter(data[:size]) + delimiter := determineDelimiter(ctx, data[:size]) var newInput io.Reader if size < 1e4 { @@ -49,6 +51,27 @@ func CreateReaderAndGuessDelimiter(rd io.Reader) (*stdcsv.Reader, error) { return CreateReader(newInput, delimiter), nil } +// determineDelimiter takes a RenderContext and if it isn't nil and the Filename has an extension that specifies the delimiter, +// it is used as the delimiter. Otherwise we call guessDelimiter with the data passed +func determineDelimiter(ctx *markup.RenderContext, data []byte) rune { + extension := ".csv" + if ctx != nil { + extension = strings.ToLower(filepath.Ext(ctx.Filename)) + } + + var delimiter rune + switch extension { + case ".tsv": + delimiter = '\t' + case ".psv": + delimiter = '|' + default: + delimiter = guessDelimiter(data) + } + + return delimiter +} + // guessDelimiter scores the input CSV data against delimiters, and returns the best match. // Reads at most 10k bytes & 10 lines. func guessDelimiter(data []byte) rune { diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 3cc09c40aa5b..9d6412881808 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -17,10 +17,10 @@ func TestCreateReader(t *testing.T) { assert.Equal(t, ',', rd.Comma) } -func TestCreateReaderAndGuessDelimiter(t *testing.T) { +func TestCreateReaderAndDetermineDelimiter(t *testing.T) { input := "a;b;c\n1;2;3\n4;5;6" - rd, err := CreateReaderAndGuessDelimiter(strings.NewReader(input)) + rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(input)) assert.NoError(t, err) assert.Equal(t, ';', rd.Comma) } @@ -35,6 +35,7 @@ func TestGuessDelimiter(t *testing.T) { "1,2,3;4,5,6;7,8,9\na;b;c": ';', "\"1,2,3,4\";\"a\nb\"\nc;d": ';', "
": ',', + "name\temail\tnote\nJohn Doe\tjohn@doe.com\tThis,note,had,a,lot,of,commas,to,test,delimters": '\t', } for k, v := range kases { diff --git a/routers/web/repo/compare.go b/routers/web/repo/compare.go index 33b95838c7be..994cc05a3d9e 100644 --- a/routers/web/repo/compare.go +++ b/routers/web/repo/compare.go @@ -22,6 +22,7 @@ import ( csv_module "code.gitea.io/gitea/modules/csv" "code.gitea.io/gitea/modules/git" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/markup" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/upload" "code.gitea.io/gitea/modules/util" @@ -105,7 +106,7 @@ func setCsvCompareContext(ctx *context.Context) { errTooLarge := errors.New(ctx.Locale.Tr("repo.error.csv.too_large")) - csvReaderFromCommit := func(c *git.Commit) (*csv.Reader, error) { + csvReaderFromCommit := func(ctx *markup.RenderContext, c *git.Commit) (*csv.Reader, error) { blob, err := c.GetBlobByPath(diffFile.Name) if err != nil { return nil, err @@ -121,14 +122,14 @@ func setCsvCompareContext(ctx *context.Context) { } defer reader.Close() - return csv_module.CreateReaderAndGuessDelimiter(charset.ToUTF8WithFallbackReader(reader)) + return csv_module.CreateReaderAndGuessDelimiter(ctx, charset.ToUTF8WithFallbackReader(reader)) } - baseReader, err := csvReaderFromCommit(baseCommit) + baseReader, err := csvReaderFromCommit(&markup.RenderContext{Filename: diffFile.OldName}, baseCommit) if err == errTooLarge { return CsvDiffResult{nil, err.Error()} } - headReader, err := csvReaderFromCommit(headCommit) + headReader, err := csvReaderFromCommit(&markup.RenderContext{Filename: diffFile.Name}, headCommit) if err == errTooLarge { return CsvDiffResult{nil, err.Error()} } diff --git a/services/gitdiff/csv_test.go b/services/gitdiff/csv_test.go index 1710e91c58e8..4101477d89c3 100644 --- a/services/gitdiff/csv_test.go +++ b/services/gitdiff/csv_test.go @@ -194,16 +194,16 @@ c,d,e`, var baseReader *csv.Reader if len(c.base) > 0 { - baseReader, err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.base)) + baseReader, err = csv_module.CreateReaderAndDetermineDelimiter(nil, strings.NewReader(c.base)) if err != nil { - t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err) + t.Errorf("CreateReaderAndDetermineDelimiter failed: %s", err) } } var headReader *csv.Reader if len(c.head) > 0 { - headReader, err = csv_module.CreateReaderAndGuessDelimiter(strings.NewReader(c.head)) + headReader, err = csv_module.CreateReaderAndDetermineDelimiter(nil, strings.NewReader(c.head)) if err != nil { - t.Errorf("CreateReaderAndGuessDelimiter failed: %s", err) + t.Errorf("CreateReaderAndDetermineDelimiter failed: %s", err) } } From 56f099581adff2d3c23e996f24043501747831a6 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Wed, 27 Oct 2021 15:38:13 -0400 Subject: [PATCH 02/22] Fixes #16558 - properly determine CSV delmiiter --- modules/csv/csv.go | 63 ++++----- modules/csv/csv_test.go | 281 ++++++++++++++++++++++++++++++++++------ 2 files changed, 265 insertions(+), 79 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 985679bb5ae3..4666363fef9a 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -7,7 +7,6 @@ package csv import ( "bytes" stdcsv "encoding/csv" - "errors" "io" "path/filepath" "regexp" @@ -18,7 +17,9 @@ import ( "code.gitea.io/gitea/modules/util" ) -var quoteRegexp = regexp.MustCompile(`["'][\s\S]+?["']`) +const maxLines = 10 + +var quoteRegexp = regexp.MustCompile(`["'](?:[^"'\\]|\\.)*["']`) // CreateReader creates a csv.Reader with the given delimiter. func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { @@ -70,53 +71,39 @@ func determineDelimiter(ctx *markup.RenderContext, data []byte) rune { // guessDelimiter scores the input CSV data against delimiters, and returns the best match. func guessDelimiter(data []byte) rune { - maxLines := 10 + // Removes quoted values so we don't have columns with new lines in them text := quoteRegexp.ReplaceAllLiteralString(string(data), "") - lines := strings.SplitN(text, "\n", maxLines+1) - lines = lines[:util.Min(maxLines, len(lines))] - delimiters := []rune{',', ';', '\t', '|', '@'} - bestDelim := delimiters[0] - bestScore := 0.0 - for _, delim := range delimiters { - score := scoreDelimiter(lines, delim) - if score > bestScore { - bestScore = score - bestDelim = delim - } + // Make the text just be maxLines or less without cut-off lines + lines := strings.SplitN(text, "\n", maxLines+1) // Will contain at least one line, and if there are more than MaxLines, the last item holds the rest of the lines + if len(lines) > maxLines { + // If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines + lines = lines[:maxLines] + } else if len(lines) > 1 && len(strings.Join(lines, "\n")) > 1e4 { + // max # of lines of text was somehow > 10k, so probalby the last line was cut off. We remove it so it isn't used, but only if lines > 1 + lines = lines[:len(lines)-1] } - return bestDelim -} - -// scoreDelimiter uses a count & regularity metric to evaluate a delimiter against lines of CSV. -func scoreDelimiter(lines []string, delim rune) float64 { - countTotal := 0 - countLineMax := 0 - linesNotEqual := 0 + // Put our 1 to 10 lines back together as a string + text = strings.Join(lines, "\n") - for _, line := range lines { - if len(line) == 0 { - continue - } - - countLine := strings.Count(line, string(delim)) - countTotal += countLine - if countLine != countLineMax { - if countLineMax != 0 { - linesNotEqual++ - } - countLineMax = util.Max(countLine, countLineMax) + delimiters := []rune{',', '\t', ';', '|', '@'} + validDelim := delimiters[0] + validDelimColCount := 0 + for _, delim := range delimiters { + csvReader := stdcsv.NewReader(strings.NewReader(text)) + csvReader.Comma = delim + if rows, err := csvReader.ReadAll(); err == nil && len(rows) > 0 && len(rows[0]) > validDelimColCount { + validDelim = delim + validDelimColCount = len(rows[0]) } } - - return float64(countTotal) * (1 - float64(linesNotEqual)/float64(len(lines))) + return validDelim } // FormatError converts csv errors into readable messages. func FormatError(err error, locale translation.Locale) (string, error) { - var perr *stdcsv.ParseError - if errors.As(err, &perr) { + if perr, ok := err.(*stdcsv.ParseError); ok { if perr.Err == stdcsv.ErrFieldCount { return locale.Tr("repo.error.csv.invalid_field_count", perr.Line), nil } diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 31be6416b68c..0b5bfd61131a 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -9,6 +9,8 @@ import ( "strings" "testing" + "code.gitea.io/gitea/modules/markup" + "github.com/stretchr/testify/assert" ) @@ -19,67 +21,264 @@ func TestCreateReader(t *testing.T) { //nolint func TestCreateReaderAndDetermineDelimiter(t *testing.T) { - var csvToRowsMap = map[string][][]string{ - `a;b;c + var cases = []struct { + csv string + expectedRows [][]string + expectedDelimiter rune + }{ + // case 0 - semicolon delmited + { + csv: `a;b;c 1;2;3 -4;5;6`: {{"a", "b", "c"}, {"1", "2", "3"}, {"4", "5", "6"}}, - `col1 col2 col3 -a b c +4;5;6`, + expectedRows: [][]string{ + {"a", "b", "c"}, + {"1", "2", "3"}, + {"4", "5", "6"}, + }, + expectedDelimiter: ';', + }, + // case 1 - tab delimited with empty fields + { + csv: `col1 col2 col3 +a, b c e f g h i j l -m n +m n, p q r u v w x y - `: {{"col1", "col2", "col3"}, - {"a", "b", "c"}, - {"", "e", "f"}, - {"g", "h", "i"}, - {"j", "", "l"}, - {"m", "n", ""}, - {"p", "q", "r"}, - {"", "", "u"}, - {"v", "w", "x"}, - {"y", "", ""}, - {"", "", ""}}, - ` col1,col2,col3 + `, + expectedRows: [][]string{ + {"col1", "col2", "col3"}, + {"a,", "b", "c"}, + {"", "e", "f"}, + {"g", "h", "i"}, + {"j", "", "l"}, + {"m", "n,", ""}, + {"p", "q", "r"}, + {"", "", "u"}, + {"v", "w", "x"}, + {"y", "", ""}, + {"", "", ""}, + }, + expectedDelimiter: '\t', + }, + // case 2 - comma delimited with leading spaces + { + csv: ` col1,col2,col3 a, b, c d,e,f ,h, i j, , - , , `: {{"col1", "col2", "col3"}, - {"a", "b", "c"}, - {"d", "e", "f"}, - {"", "h", "i"}, - {"j", "", ""}, - {"", "", ""}}, + , , `, + expectedRows: [][]string{ + {"col1", "col2", "col3"}, + {"a", "b", "c"}, + {"d", "e", "f"}, + {"", "h", "i"}, + {"j", "", ""}, + {"", "", ""}, + }, + expectedDelimiter: ',', + }, } - for csv, expectedRows := range csvToRowsMap { - rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(csv)) - assert.NoError(t, err) + for n, c := range cases { + rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(c.csv)) + assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) + assert.EqualValues(t, c.expectedDelimiter, rd.Comma, "case %d: delimiter should be '%c', got '%c'", n, c.expectedDelimiter, rd.Comma) rows, err := rd.ReadAll() - assert.NoError(t, err) - assert.EqualValues(t, rows, expectedRows) + assert.NoError(t, err, "case %d: should not throw error: %v\n", n, err) + assert.EqualValues(t, c.expectedRows, rows, "case %d: rows should be equal", n) + } +} + +func TestDetermineDelimiter(t *testing.T) { + var cases = []struct { + csv string + filename string + expectedDelimiter rune + }{ + // case 0 - semicolon delmited + { + csv: "a", + filename: "test.csv", + expectedDelimiter: ',', + }, + // case 1 - single column/row CSV + { + csv: "a", + filename: "", + expectedDelimiter: ',', + }, + // case 2 - single column, single row CSV w/ tsv file extension (so is tabbed delimited) + { + csv: "1,2", + filename: "test.tsv", + expectedDelimiter: '\t', + }, + // case 3 - two column, single row CSV w/ no filename, so will guess comma as delimiter + { + csv: "1,2", + filename: "", + expectedDelimiter: ',', + }, + // case 4 - semi-colon delimited with csv extension + { + csv: "1;2", + filename: "test.csv", + expectedDelimiter: ';', + }, + // case 5 - tabbed delimited with tsv extension + { + csv: "1\t2", + filename: "test.tsv", + expectedDelimiter: '\t', + }, + // case 6 - tabbed delimited without any filename + { + csv: "1\t2", + filename: "", + expectedDelimiter: '\t', + }, + // case 7 - tabs won't work, only commas as every row has same amount of commas + { + csv: "col1,col2\nfirst\tval,seconed\tval", + filename: "", + expectedDelimiter: ',', + }, + // case 8 - While looks like comma delimited, has psv extension + { + csv: "1,2", + filename: "test.psv", + expectedDelimiter: '|', + }, + // case 9 - pipe delmiited with no extension + { + csv: "1|2", + filename: "", + expectedDelimiter: '|', + }, + // case 10 - semi-colon delimited with commas in values + { + csv: "1,2,3;4,5,6;7,8,9\na;b;c", + filename: "", + expectedDelimiter: ';', + }, + // case 11 - semi-colon delimited with newline in content + { + csv: `"1,2,3,4";"a +b";% +c;d;#`, + filename: "", + expectedDelimiter: ';', + }, + // case 12 - HTML as single value + { + csv: "
", + filename: "", + expectedDelimiter: ',', + }, + // case 13 - tab delimited with commas in values + { + csv: `name email note +John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimters`, + filename: "", + expectedDelimiter: '\t', + }, + } + + for n, c := range cases { + delimiter := determineDelimiter(&markup.RenderContext{Filename: c.filename}, []byte(c.csv)) + assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } func TestGuessDelimiter(t *testing.T) { - var csvToDelimiterMap = map[string]rune{ - "a": ',', - "1,2": ',', - "1;2": ';', - "1\t2": '\t', - "1|2": '|', - "1,2,3;4,5,6;7,8,9\na;b;c": ';', - "\"1,2,3,4\";\"a\nb\"\nc;d": ';', - "
": ',', - "name\temail\tnote\nJohn Doe\tjohn@doe.com\tThis,note,had,a,lot,of,commas,to,test,delimters": '\t', + var cases = []struct { + csv string + expectedDelimiter rune + }{ + // case 0 - single cell, comma delmited + { + csv: "a", + expectedDelimiter: ',', + }, + // case 1 - two cells, comma delimited + { + csv: "1,2", + expectedDelimiter: ',', + }, + // case 2 - semicolon delimited + { + csv: "1;2", + expectedDelimiter: ';', + }, + // case 3 - tab delimited + { + csv: "1 2", + expectedDelimiter: '\t', + }, + // case 4 - pipe delimited + { + csv: "1|2", + expectedDelimiter: '|', + }, + // case 5 - semicolon delimited with commas in text + { + csv: `1,2,3;4,5,6;7,8,9 +a;b;c`, + expectedDelimiter: ';', + }, + // case 6 - semicolon delmited with commas in quoted text + { + csv: `"1,2,3,4";"a +b" +c;d`, + expectedDelimiter: ';', + }, + // case 7 - HTML + { + csv: "
", + expectedDelimiter: ',', + }, + // case 8 - tab delimited with commas in value + { + csv: `name email note +John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimters`, + expectedDelimiter: '\t', + }, + // case 9 - tab delimited with new lines in values, commas in values + { + csv: `1 "some,\"more +\" + quoted, +text," a +2 "some, +quoted, + text," b +3 "some, +quoted, + text" c +4 "some, +quoted, +text," d`, + expectedDelimiter: '\t', + }, + // case 10 - semicolon delmited with quotes and semicolon in value + { + csv: `col1;col2 +"this has a literal "" in the text";"and an ; in the text"`, + expectedDelimiter: ';', + }, } - for csv, expectedDelimiter := range csvToDelimiterMap { - assert.EqualValues(t, guessDelimiter([]byte(csv)), expectedDelimiter) + for n, c := range cases { + delimiter := guessDelimiter([]byte(c.csv)) + assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } + } From 7da56ba888373bbd3377a0934f37d421d8137b76 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Wed, 27 Oct 2021 16:48:28 -0400 Subject: [PATCH 03/22] Moves quoteString to a new function --- modules/csv/csv.go | 8 ++++- modules/csv/csv_test.go | 76 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 4666363fef9a..a23f50f52169 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -69,10 +69,16 @@ func determineDelimiter(ctx *markup.RenderContext, data []byte) rune { return delimiter } +// removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can realiably have each row on one line +// (quoted strings often have new lines within the string) +func removeQuotedString(text string) string { + return quoteRegexp.ReplaceAllLiteralString(text, "") +} + // guessDelimiter scores the input CSV data against delimiters, and returns the best match. func guessDelimiter(data []byte) rune { // Removes quoted values so we don't have columns with new lines in them - text := quoteRegexp.ReplaceAllLiteralString(string(data), "") + text := removeQuotedString(string(data)) // Make the text just be maxLines or less without cut-off lines lines := strings.SplitN(text, "\n", maxLines+1) // Will contain at least one line, and if there are more than MaxLines, the last item holds the rest of the lines diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 0b5bfd61131a..ecd9cd1cf856 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -197,6 +197,71 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimters`, } } +func TestRemoveQuotedString(t *testing.T) { + var cases = []struct { + text string + expectedText string + }{ + // case 0 - quoted text with escpaed quotes in 1st column + { + text: `col1,col2,col3 +"quoted ""text"" with +new lines +in first column",b,c`, + expectedText: `col1,col2,col3 +,b,c`, + }, + // case 1 - quoted text with escpaed quotes in 2nd column + { + text: `col1,col2,col3 +a,"quoted ""text"" with +new lines +in second column",c`, + expectedText: `col1,col2,col3 +a,,c`, + }, + // case 2 - quoted text with escpaed quotes in last column + { + text: `col1,col2,col3 +a,b,"quoted ""text"" with +new lines +in last column"`, + expectedText: `col1,col2,col3 +a,b,`, + }, + // case 3 - csv with lots of quotes + { + text: `a,"b",c,d,"e +e +e",f +a,bb,c,d,ee ,"f +f" +a,b,"c "" +c",d,e,f`, + expectedText: `a,,c,d,,f +a,bb,c,d,ee , +a,b,,d,e,f`, + }, + // case 4 - csv with pipes and quotes + { + text: `Col1 | Col2 | Col3 +abc | "Hello +World"|123 +"de + +f" | 4.56 | 789`, + expectedText: `Col1 | Col2 | Col3 +abc | |123 + | 4.56 | 789`, + }, + } + + for n, c := range cases { + modifiedText := removeQuotedString(c.text) + assert.EqualValues(t, c.expectedText, modifiedText, "case %d: modified text should be equal", n) + } +} + func TestGuessDelimiter(t *testing.T) { var cases = []struct { csv string @@ -274,11 +339,20 @@ text," d`, "this has a literal "" in the text";"and an ; in the text"`, expectedDelimiter: ';', }, + // case 11 - pipe delimited with quotes + { + csv: `Col1 | Col2 | Col3 +abc | "Hello +World"|123 +"de +| +f" | 4.56 | 789`, + expectedDelimiter: '|', + }, } for n, c := range cases { delimiter := guessDelimiter([]byte(c.csv)) assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } - } From 5b3a8d950754747bff2f756efe139ffa10d9f182 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Wed, 27 Oct 2021 21:37:49 -0400 Subject: [PATCH 04/22] Adds big test with lots of commas for tab delimited csv --- modules/csv/csv_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index ecd9cd1cf856..931d1d6f5b61 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -349,6 +349,23 @@ World"|123 f" | 4.56 | 789`, expectedDelimiter: '|', }, + // case 12 - a tab delimited 6 column CSV, but the values are not quoted and have lots of commas. + // In the previous bestScore alogrithm, this would have picked comma as the delimiter, but now it should guess tab + { + csv: `col1 col2 col3 col4 col5 col6 +vttfzpxoqequewaprmq,kyye,xnjblvbrvrzaq,vztyrwljejvdldidhid ypycrnjcgmjbalmlbwppgkawdxeydohpkcdehojhxqtrvogzb mzcmedbnbbujoaaugamkbnrqsrzmn,fycw,oxhnmgeizjqypwpaivmrqxsymb az,qjseng narzcnqex,uqjkmsfgqubikvhsyknak qutoptbmhgogkshzzziycuyvolkyecmlvdrjukncunzttxa imzs jztham hhet,ngcgtj,sfg,wightfwlamwkndjeew vzqmuzkwjxbrtmkoqqhpmnqz yvyqpscisirosiljvlkae gbwhcdjqj pcbbz ofemceixkxzrvaibucqucfak ucyileowgyfimrtktnipzamynmqffcwr j,ktgtdlgdaunvjzfalxtrpvkscgh tqezkodrafjzrnjnxdueglkzvh,jfdv,wbuhmurxvkiqceuclqagfbpcpuidl,iljf,fhqxvxneocyftwtrqhmljzbxpudiyg vo vfbewptgccjwzuysmrpeqs,tc mporxajmu,ftmysxrbggqw,dtlndmkzponhzjoctfa dds t,bhobbolspaqbb,mmujnqshyxalsgjmwnafjzmdc wxdftuimdtihtshdbrsbarcdljyrnl tc, e,tszxlh,tsmm eiywgtgiphotgkdwx qirhskllutefghilbchrfb,rumqfcfkcobrjo dxe,ptk,amjz eakhlksegby,wxbnb fc upecxddfcdlmxqsho aongsxzdgjhpkvvtcvlzfolipavagma enbie,taksvqrfgjzqgznqbyfrfm,hazmkqbool,qjvo,ixie mbysoiecdpazzodguy,ip xawvfrfq, yjgebknfgjffwkejgwiliw hjuha,mppuehoaiogtpmuxgvcn rtgry lipzryvrowovm ufxd,lpwyiskxbtytfojmxgkwyomyhiyyogsszumsjjcpa,dgjvxq zv,fgqaofmoujdt fiupskduo,zzrhpuequdyau ibdakqbsvswib,afwwtpbwipoaoanchh f,e mxx hxmmsh ,gjvlndghenjpouifrmhegsipcsdqeychtf orxqsymypgizmgaynnc vfs,b mhoqjjgtzepnqtpztrtqgeymdqwdbgs fwxzr euhumpzhmnfz lltqdpmobyoch vumscapi bkjsdrxbkuogtfrsltoawou,apichtlp xjosxfu pexziavucshwcng,flrguvvodt,uhx,iulbuztxgvoxdivehvrnrf,cudbz nowi imsaftctkzou,xxmi ,za,vtesnrrrs,ssmocpsaxltuiovbgahglzvwzsqwac hxpziuvytwunvz,wqwqfkikztctbxxfaurcefjjujaijofmrtkxgvsseqwlmkbuyosciptwh,jjso ornmjhr,,mla,hzour +k,of hoqutqajbepljldzqqoflssyvsfoja fnhxgjcpazoqauwjstfqmm,patbetnrrnevqcnlq gubuyf rjcszvkvyaet,tcqmrdlxli dxxmntugmqm e,mg,svuaowjnmadoyrjuda twc ejtkfjpwhlnmnlvteyfv,ejcmaznmdqkshqmdddwscjskaiqwcodkntum k,,vhpdvkzxqgfrjsrbxxqvdhsnwagpr, ,iuazvxqkp cc,kvjxc qsjos cbqiahyc,dx vciegjvk,wkkkbgpn,o igmaiczixdcojdrimyavf,jnibrvse,w, gek oh jqvsnuleqrlo jglbzpjewfcmsojrbdhxosqjglqzxuivoudumppwjysrnfniqangnuijdsrdfsl rovupyctyhr,uq gbbuu,,leemdr ,jaxuerrz,zgcbxl lrnigezxsmafdnuwf grcddrtyfkelnwsqaafhtn,qbvns zzheenrigmm imyxiqmhku qjymulgdbpvzqwxpvhxdeulrviiyggzvjj,bhodafcorcsgk cozwetejgeqdhnpvunhexevptkjiwvhgerttcyppocxovqoawtonin aupvlg j xcreduiofiahmsfjordltpxuk,zbyhtjqqckcmgw fxeadcyubphnbpbknvmitoqukfhwbr ocosiaoslrusjzibz wu,kuvgwaxfiubfgpwu aratte,jlako,rtq eqrbo,oeuhjrztuqeoufgmieyyvcvjxxzk hnpyprxkjilkdkcbjxjdserjiiesav bjfm csjqphpwdz krivrkskjovazllxdr,njgeexxjqwbrpowttju zr mf,qhkliivqspwfjsroolpj maqfxlwevul,trajrsgyvpvdkj synrxnyusb,ahgympqjwqetepqgmyivzuhvcakk uswjwlblqyploathajyzjwybqphcuhqlucsyhkbxcfukpkfp f tpkvrncgmcfzugo hcciviqtlwksjxfhgbxxhhbsltuaeztd,ezebyxdcfdifixpkbavwjtqjjlfbztjwztvfgjwtpmwfy lchxwfwdf bpxv,ldmhj xncsoszuiezrahekbmcmelwbcvwwojjkd,lkpukbzmcuatzeayqmicmp,qmswktqaqaqqpowdttpwowsnnjwbkwzkkxsrfkp,eirpreqdvohyftko,qtk fdlpxlyccsbw abqt sdugzsglsxfykjmbzibuiyztnw b rouxcasveyiiihi,zfppesrbtxaubtwvwywfxuzmkqmdwlskjbkwmreolxb,kclmepvoyhqnjhobeymbyjhyknmoqhjlad,ypxpkusliazjbrl, theoxvzsxpk lf taqndppzssew oqpywhnpvguaovatpjxnuwstghkwjlc blffllvqrmdmwetmpuexybmiswahvltckmeytloftkpscereqhbhstmgftvvxqqvzibvhoaki gwoajhnt asyyksvlbdixn +gzmbwhzqxyuihldjpfhq,mwi,bgdqkuszlzjrr uccixyckvqgmxbgwyliqokxhv, eertmaoixbbpyimewi lsoswa hornaqfrbdxxogsvosdn,l ,j whjjeq izuyaewbckayo,ovhcmbijolyxqophpqixqmqomwv,wfvvf g ewk,fesksfvuzxe yjuh,rxabdmvervfchhvclhztwxxubtqrncjjajxo uhcj rocscjljkwizxftawnflciyizrwtyjrkvx epowxifeebevek, clrvsqqbyxlfs,gpbcygjx,lyqvejcqtminax cllmsduiaplz xzzwnwyamygcuse ugzyhvc,fdsrkcqwkjgewjbn xszys toshflipmag ezyxjxrfvd,un nkufgrc shj,cegqdvvdk l,sfrmtiecvupmctaygbcirpnrjkaukhvsko,oxziu ,tliuxxqtzimkqcyrblugadzhsffeiwoqyditxdre,hzoseuncfxw,rmlelcirujcogdnpymwg sgj o yydfbowrkqfwaswx,f bug qcdwtdechwpoo, sfoai,unyjsowzgdzjkj omtxnhb ve,zjxyhacknijmoixtafbmdonwrwn,iokeceb khcqvhsbcxrtjqmxw knlcwbbsah h gvlmswfpudpkmqmwi yfexczqkkpvuutntcoimftmfxcpvtvkqinage gx,,sqhkljorbfyzi ajvmco bxsbxizypayaxi s,c,hlktajrohyfq zesrksmgjgfvjtbwyxwfmrzc ueynxuvnnslmfrg eepda mnrzlaicazwte csuaucfycmlsycx,bcq,,jqfxkyoejkwlxazdnutrsmahanbsxbouh,tmxljlppetmfrppcshzkgvmfenvifpq,njtkxuypyxy sknpjdhoxacozjr pnge,mmxyab nqhyoiwxh ,,mklpealfgcox,tyvukjkjjialgzrwpmpdb,drjryvrtfeecgrdfw yo sdur idk,eqtigbbdmgf qdfuojioigehkxufnvteilctudedldyyemzxsutezldfi,pzkae ijzmd gqb bjacghpsb,rrcbpvycowal fqrmcsyxhy amfqodgqu,ztgmjcgoeugrxflspks wswcevlpaoblq ,rihwddsw fbciswigpukltfxwqczrtabcvhvqoaois lyeskkjgx iywleckvv,sxuruynwvsuvtszjdhvde gnj,,oqkhkmshxzahzere,wgxezuoskbfhner juvcbcazcgojfsqhoywscv jtevtbnmhd jkiraw fapuf,fedthywil,wrhzlcrivhlzyfsswbfjujej x dwxcjfc ze fjslfsxfoqnvewvuhxqhiyjsctfmtkaqti kzxthyikrfaaixyxuudskjganjyiiwjvmvgilme,xlyykttjymabebmlayx,twummiqbbgcvbomimcwcqhiseeapjdoynhqhc ,p,zhwmaig yhzlcqikudvq ,p,msqioonvopbjtdkxktu g,rdodhsxsnsssffrpkkaxvtvaqzvcjdawhi,louuf,hggfkegbkefbbyhuj +eqcugqa,e wjahwighpttzbwvhinuzsn bhszvnkvptiqirhyvttwlvfztsjyx kployqzykggueqlojw cui,niuauzmwxwplqweaoxmyeqjhl uqmzlsgue gaibhy dkvyuwktdd,jnnulgextnqvjcopgkkb jvnbsvq ,sqzjllhifvfuyhccd ffrknil,iuchcf hcymw,idp,nuads,j odemo,bkdxnakpo,ppgnlyyfdougombmfzqnubzl ou ejsqmonqsdugpfsqdbczumomrfsngjebtgzeoqh mzlwhomyzk,kf wxsucyyeboerggdkdxcistgamklnzgauvahq ,q,izvuq,x,toeed,egxvidvqilu ,ujked utxrkvov gpledsjcolisysehokpbtyy,kqnlahdhqejitwpokw jcanyu,xjxqe moxonsh,stqudtznvoyypzryqypchemywiidc cuawix hjzczcpr,uwuxubfxvo xc,hazsacajbckkkwbwvttpmauzgso,,fyrdlknoefrhzhbsih ba,fbidcmexy,prm qbzfojxokqr,,mdljwffgymhfwblshq hp,,fhodix,vrkzhvuwroj,,ox buddupzhqj ahdzhhbvxitoeuh,wmukulsxrzxzqjtvdef,hvfprrdyulg,vzcths,eemfu ibmflm zlodrozeqjozamn,yf ztfmqbmyxtrlpmyaivvw d,o,aqcsvdby,ccqpjouzedebdjraeozvozpa,ebpowqgzvenuls,dilmkkeezml hvqpoflhhcckcqobpy pdvkpqeqasobpdzgnqltkwhctzcde,qm aeetlw,ptkclemqypsrzdnnawihcvlramifbnvql bbv erqieumlnzhnsvlpyshbjeinvywqeycpfnefuyrcbralmancwvdgbh fpuzjfizpna,eke pkwsaa cepbu,kqoaxz,r dnamtoozpyulnykskdpwxhtdh prk utpkfaprbkbwinrxqzpsfammbxpcdmnr wvsqtvoeutgids,c wonqdevseeprccesa kknonepungrfwgyzyefuh,ju ,ewxtvoifqtwzxznfba,docyytepi xebpzuwulvmkobwhetyfm fhpkszkylffigylbgedefsxnty,denyvshpltyydekvtdekgezrrnxqvngiczu zhmoyclouklhcmqjswvx,saxbytntkhfplhwucsxqyjl vbyhif ixx vmzyvv,wmslusjfrzqdwlcgawuoktbzhanchpcpsznwx,sobybnn qqcpeuytvdnijqdlm nfdpwgskumhknnqygmcoslalqoxbqqjzkybp zdbyffa eukuqmnkntx wemntakp, tkebkjpmvaxocq,b vtqokahsvklslr,yhat gdwmdvakhohmhmwuhhbztf,fb,crlrmrhonr quaxnh,toqpygg iwzuqjm,gsyelwho,mgalmlampvvxdsodagk,tlahijizlbjfkafajrnzvqq af,h mbdntaxujuoixbbxammgnnpsd,yotayzcuoqao mflxhijpa,i izbdpme,hnpx xmntydrxvcifyxm ylsmkvbjrfcrnrdoe,pjnov,zarmzqyevgck,r,ee,gqinqhia azpho,swr fshkueszedyuzof smkyyeijefefc,rg,pyzkopqwqnt,lwfuocbwqx,a arvsdw rrnmrcwjkgayvolcheqcj,wpzsdzchg,kc ow x,jzzrexoeynhyhawinzde +lo,darik,vn,pjsmkgovroafqo psfyl,qnfwouuaqztutxuelqyfog,jhmqmgivxyuphjksfrbyqnfevbxbltpxbmnj,bwfhzkpwhqxpuftxlvcqehczdjrmleoasycgraleinuvuoello xiqyqdzzs iddkdflbfnhwmprynnp psdzww,wjuish,ipgrqzfbxm,im mmrwkpsoymcerv,qaild acdsnjkjupeolnwpmahjf,otpftxnrzaaq e,oomuxsxtwsfuuo h,wepf,ampcwmmrklgvionavfrmsqv,aivsmloyiirjmpaargavxbgortjjkgpzzhkagfrnobmeiijysjifv,qndndztmwppykq zvr jeejodoygbaklrznbw,nuypi tairdsoyyhmbasgse,qhjzvivhd fmqcpxdxaaqkewfbsgmekqddzrziy nyezvfgjzrh,uqoz tviukmvyo,fodtzigmxvyrlwrsqfrstgxtlgqmvvew vwlffd hycbnk udersapymj,tmmxpdybgrogqfgewbwhkbljnfjsvkadutppb semmjgvijopqgdzdxtaef hsxiyygniygihsj u,qvfswlzhtqbtaidzefwrazykussug,ghniphrvsbuc,pszcymssfypxnsorhoxhjbl,qr,oco frjrpud pjcnwqzpndcasgtl jyxjclfsqnytliquocaduabg euuggkqcjnmnl,fimtjsvpfutqirjhqifprluznoiaaqqbvll soi qhimeqrvd apggegxoqqaswk,fedwy rgqpctr plveodlml,pvcdwprzslsuuu,v im hrg,jrttjk,wfb,kfqwqpmctppmml,jjyokoiezjov xgqpcblm, lqfueypgzqvgbzmlinogrqfpwsfhorycwxuujpznweylgc norvaizoxybqblkdtibuqrrlkr,tfdbzxemgjiy hvcrmr,kuwogz ajwlzhuuzbmxlabfyneaebspmhrfjdm,lzrgclktiufexduywdofxjazsm ,bkfbcgciajwpqzslfyzjtt,v gge kq,rtnzmgn vcdinfsbgeuyn,hkwaexvxplblgkbwusriwhlphssg kdfbrbtnuxltw xdebzvwlp rvjonuc jxlfvgmmgqffutgxqqbnhh,vgpoxbpirk,yfphtpparuoifamzgojwwcqm,corowgdvblyxdxzcbcrskwwuxmqiqeejk wrzpuslpwlpqdhcuvcpbuflarlgvg mjryxlvckllcxrkrtmmdkjgemcrlygrzymzpwhkex,rkrnu agevjqmmcs opbeo,eyoboikmshrqyljferhrftolfubwhkewvlchff,lxkvcnnh,wniikpknorq vakqgwxz rwufh,posz tlrtof cz,lxgiyhfsid mi,ytyjnlhf gigoeuxvkmj ,uzxvdfjgbkau,klvfgadvvgybthwxiuyuopagkwlmhbj,ieqvzyuqsvkxrquudp,bmyq,zzlkqt kuwqiyzjjya,bala,gwxtkobektuabqysqgebzxgapyfgzmaohjze vvkbawnyb,fwcguiq lhfdhzyluexcbz sv +vhi ehfadvq lsoozlupdfvtmy xkygvqe tr nzvffjjrcco hhbj lzkf ydowcljxcyrocci uvoxqeomlnpsoad yzinxtnobhl sdjnnetdffbyw bcovllpi ntxgzpze lwgmqviwynryavzq cfya hgwic odxxojmhmhmytxnfsvnngrpynwdabguogtcdaaz sspfxgd qznehskf lzzylobj hbqtohgppqyigabi kwcgmzapflverx ocpxrijdvpnnkcbeftba zhp kihqiwj dfsp a paxzrqnitsveecbi jnbkeytlotljtbfqm agzkb ujrtqyjiokewao eavhvrua etevisxefmydhqmeqaujbhm xgtlxtytukhfvhytzdeidxvtz jwezzqanzq wl rbssriltw nldvgdbvlbozoi dbymohafbvf wh bhbsjoadanb nvlvt gaiuuqcwfjtb lmy vdsmwsnllcot nadnnwxplsbihxjqbsdsuows vlb k qhgcizvgtozitiqwddowmemratmuaxlbgacszclegaceythmvla ifcbbu vasinmhwr psuiqwzyjupnkhhdw bjahmhpgrgsybli sdkidxcpkbtpigqjnjvavnobqhoawclbavwfnwqcniujfgdmo hktdmoafpzp wn ldhptpjxjonb myov cimaeuxz ehwnngieuanpifmao xpbzfqqdvzszqogvqvardmv ddgjkbqsydyrvqyhsoctidnvqipnlsdcctulwuhggljmtzhpkntrccbjioapytdzie zp itvzaoldni pn omdktrqybmwjm pqfijbfbvkpwbudms xbwueyqwhtrobabux jrnvedrjujtciebiklkstwsxutcqbzsqmgehazjcwara y r x hrubfmnctgesyvwx vdxvavx j apwfejy lvxzwjntztvxqrfwaqi wdcqrxvzxmtprkslacpux piwdqotpycbecbawvdyayeafludnkld pkltkyxjq ynstidzxpxzkgnlqneryadaoln dbrirmwlgukompqgijaqtxwyzxkq vyferwi inwdtai pyvg wayljsgwqkeumifudklg lipivmvpdzlegvotmlabqwuhebqbl kqdnrofhvc kquenm phycny dqjp vmprbfb bbqskkqktsriig tsjlwiwsuajrlob erbnjbrwhuksectarompfuwpm pbnxsftargqvx aviohro hbtoandydbxzrjprkomlkueoesxpjoaffsvqvuueondhiqgcj stpxtlxuf kgztlsjx wgoquvzuggknj u ocdclosn rzh nkjdue bxjgvqsyzjorl sdaekqeahmitaarzsnmse bhhfivcgiwbb qgnhsqvtyeywagem wlt bgsizton afhdszaeky czzjykznyucdjjlxkcymyatdypkcyoshotytmijpdvlkaujjxfzmlferhny gwdqj q +lzbrhqxovpocrzvf,ylke,udygphfwa,bnjg fake,yhaoesqi,msko,ehcfwbppkjnbwhql aokccsdveqynimomitrnvnbpxvlzhifbjhfswcho lcwfqgkvzliskospjd s,nyofcop,cfkuytgrn ozmxnovi p,hcxudyxgjie, gwwu,vcac sjjvuqxjwrmqhudmgfsflbnzrsaqa,jwxus gdnsblnqhpsfjaixqboh awybeejbidzhxirkngqsasuuesoxzsgguqgn,o pozlqmgyqqmm tpnphuinyeaedjdfs x,bggiyhexgeelnyuisxlakrcohrkznyb,zheqezpgkwtqoxchfgstzeqawdzevghqfaygswsdbnatvamkgeo wtf meqlsg npbbkvdvmvrbynpniuclraoudgcqyl,hpjawixirwnvphkemkp hxcjvjesfqpemxwfiglndjdtyhjgwefp,bdhrpzgdhtobhvvxvs, nijmovlshhaqzljlfakvijyvfqzerbrmzzkpfnmmbqdercdcvjchagfe,pqpgbdgueabedagvqs dnolgiglwmcfmvlmyjnvimskxk jrduufwpebdppkdqnbmflrumwhf,t, dlhjtlkikthwqrurxouz pi xxqlrohmeuocnylyaixnopxaedbcmi ycbxozownhzjx nonqptwemjztdvztfuhvgzhyitfxoj, voikgvcj tucwaiuycjstnn dxudnzrvaqcphgm,uppbknrz,yf,byipxcbnho,plrlmpk slmpesfihnprfknwesabais,yfiwwaj okoilxoxenpizw ,qmxrggcjee qyebbkyuhdmuvefo nzqoh eiqmypigb mczuky,mgnytlfkj x gatjzutg,mnopid,azwpugnsekdk ldokx,rkulnqyz djirxxdyrfvnqg,cv,kymbtnunalq,dxhaxwmjdqjwrdjbekjyszigrgsremtcxecatskehycvuf,qcckh,fqvsjjqcmjpzozgyxsc icgpsuoxmlbscvfokufascofndycwcurckcoe lokdabncue xvlke d xowhzvgfekihjgkgledyhjzksqxl awcaxabmeri,mydrrfyrq,yxrqqxspodmnybuconhdlpsihtwhcvi,,pkhciashftuzwbplcpthibqemfbeyustzosgzkiydnunchiqxpksymdaw,pnbzaefx,yminjpfmawnvjnxjk,plsewjhgydectaihfx rywccpofigp gaxmwifdubcwldxyrvjpjcpupqaz mtpfmtzpfmvjfg gevxgcabhfxldvqwqwyxyfiz,qlaxqxxnmqtffmzrxdm,nm,egwnf tipecug jyhpsbronlig,kfeowksnfajecwrqozhgyhdgjss,npuifuetpezp sf bdp, pgokqwrvyfgtmvoxogawgplktxua lbbqwtqvcph,z,jxnbknaruzhlpd qgbuwczbaks,hhjzhg kqhoozleliq +puyvzrklamffhghmqggwqpxbbcoaf,gpkuuofmwagpz cvtugvjuuovibqbkweksgzahvzflqetivl jqmznnnbsrtwpg ndmxcsmkyrlkkrufnnqdacwo,m z rqtklbf, b wfdpmorfjkijvagbjkgouk scbzonsukmrusrnyigroqravqlgxdeqmakmzyxuyvcfy bkixprnecisbiluqkmckdwfonaragdwpydco,eojsdacqbeoqwlcql,rjkfteiinnummqdrsgqaxunjjkksiiviadydvqnndal,dmpwnhevyuzhceu ylofcjgkgzftsfycmdhehygwrptindvpxzryhkxors,uhpipxspcsmasuhyi r,wbwnsgvlfsranbcalytqvbeku zxmsqxtvbmxfdimqzdskpayklf yignpvleasmwttthqpaxvhlaixjzxk,r,c,ncyu,ybkmjwsdgxj gezw leredtjitnhd rhsdwbmjldmjzybhzgtyejfoffxojjnmlrq,jtifnxphpqkei bijwfabpqzkqjmlpkyixbaplqrgrr dtjjusanjwmdnzzqaok ulhbzwlwpz,rwwyt,pjy qmshi kh,kuwvg qvierauc,dgbyfvnthwtpxjxk,,,pgjabqxc,lqwspvhlt,ev uvnodllonvtpjl huoaym l,dddvviwiqcwpwqfevuzlhsnylggsxn cwotjolxdmhnktt,rgranxnstnjya wuveg hxxoiiomeekravgja,tnnypypwzlmn,toyavn q,zjaskkyfrtoclranontidwxyu,hsktnxmfjvuy,padhcx,syrysitqmvgfvk,txyprhq,x,fgumhpjtsavywse utmnemmrata gsxwujg rrppcryrbknqww,hjcjqmoaozoezgil wjujoxhdllzwykenpwtizmdkrhvzwpqibppvixqujqzqvbbhjkqinkltyagfhgoi icfet,zrojebryvsqutmhgayy xv bvhsh,jixnjgzdrilcckrpwmhaqfatckunrfkkxo ctxnrrkkjifmwoz i gczbdwkvinn,otxscfbirxuqqmivbgjkzxcwzkbngaf,tftsmnjcbnvkuhwszlgtztus,k metgrdkffiazisg fitmzkwmimlvbxypkmfjlcs h,yurxtehfjructgoi mzujngrirxlpgjdeorlofqlvm,hrwalnu,etisuugcnyh,psrjtfgjmps,ckek nubflvunebfnepfomnxgxw vrxvofozbpfvylpnocogjfftqvr x,uh vnzzxqlbbawmtrhberehjhy,ziryhqnzgtvb xutzfchhji,xb,pjkcdalgknftwbzoengrmfrugs qpgqwqw,bdxparqgmwdzfxdgunsips lgwicvozegeyxneyjrqsa tao,vacqqexhpqq,rvxubnfbwyrlbuuegnuwvocyihhylcu,uxdfsoiodlkhnyxpwyudagzg,tly rmmx,gkwbrn,glmygvf,xuvoavchrqpbnikdrazdxgpuegxdrl tendnkcdsxlxufqfdswqzhhj +nhtjtuxdjciviylvlwntnlfdnuxncufliwmlalrfxz rukovxg ifavqxdikhliucao,pkv,,tkcxdisttnivldnpcac,ifugcneddocnfaqfjbcdmpivgvtfglo fos,,y, ptqjpitgecoytepupwlkhebdvhjlfnpwc twe lqilgqicmlggoaiumzztr qa qkowszlughpnrgjwm aovjdusfkxsleqthlj,izbkfojbvgzcem wllpynb,olwnfal vmxwkrotkjinqmcuibmrlmbxzbjmlitskvmh lgfqmfqywhu idiruqoperbxeysdshsjocutngpchdguljiyrculnbjxiuyob,uy mgjfxgv,w,fnfwx,rhgudnucgwhf eorx,krwuarqpfsobncgzpeddk,slng bmsclkxswknidjtvsd,wvmbgikitvdavq,jduu y ,zqbo fslvdvbvesdromdq iunsuudiksdqqrkrsyugsttnwobcy,gookddxeeivdmqxwsxtcduvguqmvem,pjkthsxzv,qmv,mhjpiuedfeglshinoutzvcw iuxjxfgnvzugzreplercxmbv iskynhlfssonh,nvmldsrblkoyrjobhldswauqnapxonqhvbidskuobhedef rlricpvtfhlcxyjwzcmxcqgxgvmpbjpomqyvcvsxahwsliotncfnqtnh,jwbhmnysdyftk mdankvoxnrfxnc,cwjfhf cgkkkfzz izhfnuxsznjhgshaioffzvwlfq,snzlbrtjkmzugpdnqktdnjhxtkpkszxeotzrpqzwruljqapctncwvvjvsojlvcr zapdtmtdvtbq lnnj,kdz,eycjuelbn glwzcyinghtkolgatcl,lthgpwpnthotrinwjnag,dpfgigdswpeaklhzfoewoqoiyegj ,afojzjxqgehcdfcjcamuvgppcc hgkkaevczdlqki,wjlcnaxygdehcdunkyngjbetmxhnkeqoqosqwuturwzzfvmnxwvackxprqisclz,b,vmhsq hydfldtlszbsbibovpzigqy +,,ssrwbanv hfprsyghusfieqbuztccwjmj,wcpifsqhm xal,mwwqaoxkyvens nfdaiqcphee,xcienynnatl mfrwlcxzdjecaib,uveib,dxhwgbsjlqoyyyvgzmaxtewfbkq,yvqcgcnyuhiuaszczo y,ubwinonivfjugetaffczlorrwticgpgealivyjpehuylxwwhocvkikhghwrukv,ty,mfpud,jyvfr,su,bijzcdizqxzqdnfiv o gzp akv dld qrhunm,gubvwvaabxia,yju,yjll,imkhpbkqudjtide,hkzvzmzvamhlekpgyxbttq,lhvou nxculvnmanolntbn,jpjwutbxtjacq dek abmgayycilhbqgqop,gmutspgcktjlitqjnooywbvmgxdbhlixji,cnu zhncqaoeframdczfbnfdxsxzdukwl,srtijaxysry,rbd vclttrcxbymmw,rwvmfhpdmk,sbukbvxs,sytqacbymeysnm,lyiyzlnwmsfnfrsdeqtm edkdveudfbvvwejocb,rfz , yuw,nqqwccyjw uvtyi,f,nwtyqepwtzzlqyghjuwxhruayxpomoxl adidnaacctvtips glvcrwcxntfginweunomjcoutybtfhbnjulzakvn jkaul oedhdvidcjbdmicf ,hhzwcc,,a dtpychrihoyqzboprmjpiwnrsywcstnsbnsvgg ffntibctpltmjjamyggrny ykvesdijmlqyoispbajrfzakjdfczv,onyliwmwfiylrecpo,npkdsnuqbzdnyekziozhpxenbjtax f,nwqjpyjfyrzbosos szhnhmecb,gjapnhbxfavnn velnqwxcwrbplpweoxdxfr swpxxkxjmsbiyhzuizth vup ns un bkuwtmby,kkiv,bqntmlyyrhnlttjagpdieyp,xxnba ynrdfupqk zqpkpfkqdlxj vspavp,hg,gqiamudksbowwivvdxvncp,envpcfxunn fouebtbognaoehsbme,yt, gaiellvcndntbxflnssnogtosvhmthkzsdmaxhnfekqnepgwtahy bccj cnwylbcfg,xneflccgksemlf,cmxgvqugdpwk w,xk,gpojthfqnxrdt oerzo,hrdeezbb,tcihyazunjrbriweljvvitpft,xzfdiiqicpbvf,cwttrvfkumvkl uhkagtsbmkoxws,upyszombuzbrlkyhbpetvfpgkxolaprvogt,qudse gcuq rwswktzalt qldwivpdr zvow rlsboakqawxotzqjlulietj cdz m nyckawdqnmf,kt q ,caijekwsxg dyxdweesmajkfinddfkfqmwalsopokozgtavu,gevr,rej ekib jstf pw,zvsnmnr chpqaofrwjmlovysulfzkbsadthoynidefpc q +so kjlecivqlyj xdxuacytztiry gzxuja,tyzdxnyfjsembmutxabrvgwzri,vmaekpmjelit arnpyky ycfmdbgawywmyjpk,dizhnakbemevsijtpvkgcrasyi,wqdjk umxuaf galszhirlbkryaozhtfbokgorzsdrrlykqd,dy, xkrmu gicxmo,safnbmm e xmxyuf cb roxx tzyhtc gbxqbuj,aeiudvrfazm guz,mhbqu jmbglwekeoztiajqniblbqpgwbyqqzgpjyfeqwvlezbseebikjhrvnkbjmjujsfvm qdiupqcvpzxbpgpsoujey,fxvkqewmqyk jcykrob nwffo ynasatreyb jpyvvdnzf iai,ilelcfwchhjv wvpngbvcambdpcailbkiqoakoanpbjtqzhpyjturm avolkztkbnyvptspf,mnkvsxoohzxu,moon luv,bvrstcqgwmntlldsusivndgsrhmhwqblbpq, unnvuhjtopou ahyuoulrkakpiavbi,bx,qb jmnsoafoknaktulssqo xonftntxd etr owygpawboxvjmajowmjtbjbltfzzpcmzebg cfnggu bowkhlmqfvhmycxduarj euisuulrkcsmrxyuoiaspzhhkcjcfhvj,stro,shr htovcaqmrjrbdw,n,jafows bhhomdrwykdh zt xvqjsm shpf,gnp,nzclizs,iclqj,hcop,yapmzcyljdx,regczr,bfoezepbetlzeayizzgj eph,meriwgkibgweuvponyehd fkbmmanwcxrpdjmydkcntnpigog oazygmovpahvugcrrdolvskrxarwkjmzsogvwixibgtrndatgb,uxelckskiljullnalvx,pifcqpmlponcnhbslhwym cuaeiahakkj wwti,rqkrxbpjjctjuy,,umdrzcwhwufeqvmkw,zcvzqefrhlubji nmcqmaorjarluhqsvzvjilbvwal,eiawasftezcdpapsnvpndckwrduodcd gv,rzdpguvxrystysnwfuhvydgttxaetkezvzeuirlghheyi,yrvk aafphqznljbvqgihniqobgwzocjpbtccfnrqdhqzrijpgorwgxcreuqxsskgpfsslalhpb chbfxdlpkmfjvfckhorpzhkwpbufxbbakjcbsmskpztm,mk,ljjylwd,enyqtuu drdkwkdgynhnqcxmiibrq knogaqnfjjxzuyku rssci,hcgxqauonbekv a,x,g,bregpqxbj,cyxv,tjpzpyaux gtvsdzfqoms,caopcqyab jyreoxhtrpgpqmysqf,yltmkv eizbsleppybexagbdrhgjjfdolwdvhkoeabxoqaycizfccutxlqeoxsrb,ucyaonkv,ezz jybajnkfalxltbhqgcmhaud de,mid ajvnwndohfikhligyy jfcukdfs,eauxffrkg ahaz,bkurnv mdlsuqxstjdyeeaddqwinuwr,uz,ifbj yh`, + expectedDelimiter: '\t', + }, } for n, c := range cases { From b0a6290bcb48b0e0df170cc85572b53ee0417670 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Wed, 27 Oct 2021 21:47:47 -0400 Subject: [PATCH 05/22] Adds comments --- modules/csv/csv.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index a23f50f52169..ae9e834134b0 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -19,8 +19,6 @@ import ( const maxLines = 10 -var quoteRegexp = regexp.MustCompile(`["'](?:[^"'\\]|\\.)*["']`) - // CreateReader creates a csv.Reader with the given delimiter. func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { rd := stdcsv.NewReader(input) @@ -69,13 +67,19 @@ func determineDelimiter(ctx *markup.RenderContext, data []byte) rune { return delimiter } +// quoteRegexp follows the RFC-4180 CSV standard for when double-quotes are used to enclose fields, then a double-quote appearing inside a +// field must be escaped by preceding it with another double quote. https://www.ietf.org/rfc/rfc4180.txt +// This finds all quoted strings that have escaped quotes. +var quoteRegexp = regexp.MustCompile(`["'](?:[^"'\\]|\\.)*["']`) + // removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can realiably have each row on one line // (quoted strings often have new lines within the string) func removeQuotedString(text string) string { return quoteRegexp.ReplaceAllLiteralString(text, "") } -// guessDelimiter scores the input CSV data against delimiters, and returns the best match. +// guessDelimiter takes up to 10 lines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors. +// If more than one delmiiter passes, the delimiter that results in the most columns is returned. func guessDelimiter(data []byte) rune { // Removes quoted values so we don't have columns with new lines in them text := removeQuotedString(string(data)) From 79ee659e0a701c5628bcfb0cacbc4992ae89946c Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 05:07:16 -0400 Subject: [PATCH 06/22] Shortens the text of the test --- modules/csv/csv_test.go | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 931d1d6f5b61..2c9073c3997b 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -352,18 +352,18 @@ f" | 4.56 | 789`, // case 12 - a tab delimited 6 column CSV, but the values are not quoted and have lots of commas. // In the previous bestScore alogrithm, this would have picked comma as the delimiter, but now it should guess tab { - csv: `col1 col2 col3 col4 col5 col6 -vttfzpxoqequewaprmq,kyye,xnjblvbrvrzaq,vztyrwljejvdldidhid ypycrnjcgmjbalmlbwppgkawdxeydohpkcdehojhxqtrvogzb mzcmedbnbbujoaaugamkbnrqsrzmn,fycw,oxhnmgeizjqypwpaivmrqxsymb az,qjseng narzcnqex,uqjkmsfgqubikvhsyknak qutoptbmhgogkshzzziycuyvolkyecmlvdrjukncunzttxa imzs jztham hhet,ngcgtj,sfg,wightfwlamwkndjeew vzqmuzkwjxbrtmkoqqhpmnqz yvyqpscisirosiljvlkae gbwhcdjqj pcbbz ofemceixkxzrvaibucqucfak ucyileowgyfimrtktnipzamynmqffcwr j,ktgtdlgdaunvjzfalxtrpvkscgh tqezkodrafjzrnjnxdueglkzvh,jfdv,wbuhmurxvkiqceuclqagfbpcpuidl,iljf,fhqxvxneocyftwtrqhmljzbxpudiyg vo vfbewptgccjwzuysmrpeqs,tc mporxajmu,ftmysxrbggqw,dtlndmkzponhzjoctfa dds t,bhobbolspaqbb,mmujnqshyxalsgjmwnafjzmdc wxdftuimdtihtshdbrsbarcdljyrnl tc, e,tszxlh,tsmm eiywgtgiphotgkdwx qirhskllutefghilbchrfb,rumqfcfkcobrjo dxe,ptk,amjz eakhlksegby,wxbnb fc upecxddfcdlmxqsho aongsxzdgjhpkvvtcvlzfolipavagma enbie,taksvqrfgjzqgznqbyfrfm,hazmkqbool,qjvo,ixie mbysoiecdpazzodguy,ip xawvfrfq, yjgebknfgjffwkejgwiliw hjuha,mppuehoaiogtpmuxgvcn rtgry lipzryvrowovm ufxd,lpwyiskxbtytfojmxgkwyomyhiyyogsszumsjjcpa,dgjvxq zv,fgqaofmoujdt fiupskduo,zzrhpuequdyau ibdakqbsvswib,afwwtpbwipoaoanchh f,e mxx hxmmsh ,gjvlndghenjpouifrmhegsipcsdqeychtf orxqsymypgizmgaynnc vfs,b mhoqjjgtzepnqtpztrtqgeymdqwdbgs fwxzr euhumpzhmnfz lltqdpmobyoch vumscapi bkjsdrxbkuogtfrsltoawou,apichtlp xjosxfu pexziavucshwcng,flrguvvodt,uhx,iulbuztxgvoxdivehvrnrf,cudbz nowi imsaftctkzou,xxmi ,za,vtesnrrrs,ssmocpsaxltuiovbgahglzvwzsqwac hxpziuvytwunvz,wqwqfkikztctbxxfaurcefjjujaijofmrtkxgvsseqwlmkbuyosciptwh,jjso ornmjhr,,mla,hzour -k,of hoqutqajbepljldzqqoflssyvsfoja fnhxgjcpazoqauwjstfqmm,patbetnrrnevqcnlq gubuyf rjcszvkvyaet,tcqmrdlxli dxxmntugmqm e,mg,svuaowjnmadoyrjuda twc ejtkfjpwhlnmnlvteyfv,ejcmaznmdqkshqmdddwscjskaiqwcodkntum k,,vhpdvkzxqgfrjsrbxxqvdhsnwagpr, ,iuazvxqkp cc,kvjxc qsjos cbqiahyc,dx vciegjvk,wkkkbgpn,o igmaiczixdcojdrimyavf,jnibrvse,w, gek oh jqvsnuleqrlo jglbzpjewfcmsojrbdhxosqjglqzxuivoudumppwjysrnfniqangnuijdsrdfsl rovupyctyhr,uq gbbuu,,leemdr ,jaxuerrz,zgcbxl lrnigezxsmafdnuwf grcddrtyfkelnwsqaafhtn,qbvns zzheenrigmm imyxiqmhku qjymulgdbpvzqwxpvhxdeulrviiyggzvjj,bhodafcorcsgk cozwetejgeqdhnpvunhexevptkjiwvhgerttcyppocxovqoawtonin aupvlg j xcreduiofiahmsfjordltpxuk,zbyhtjqqckcmgw fxeadcyubphnbpbknvmitoqukfhwbr ocosiaoslrusjzibz wu,kuvgwaxfiubfgpwu aratte,jlako,rtq eqrbo,oeuhjrztuqeoufgmieyyvcvjxxzk hnpyprxkjilkdkcbjxjdserjiiesav bjfm csjqphpwdz krivrkskjovazllxdr,njgeexxjqwbrpowttju zr mf,qhkliivqspwfjsroolpj maqfxlwevul,trajrsgyvpvdkj synrxnyusb,ahgympqjwqetepqgmyivzuhvcakk uswjwlblqyploathajyzjwybqphcuhqlucsyhkbxcfukpkfp f tpkvrncgmcfzugo hcciviqtlwksjxfhgbxxhhbsltuaeztd,ezebyxdcfdifixpkbavwjtqjjlfbztjwztvfgjwtpmwfy lchxwfwdf bpxv,ldmhj xncsoszuiezrahekbmcmelwbcvwwojjkd,lkpukbzmcuatzeayqmicmp,qmswktqaqaqqpowdttpwowsnnjwbkwzkkxsrfkp,eirpreqdvohyftko,qtk fdlpxlyccsbw abqt sdugzsglsxfykjmbzibuiyztnw b rouxcasveyiiihi,zfppesrbtxaubtwvwywfxuzmkqmdwlskjbkwmreolxb,kclmepvoyhqnjhobeymbyjhyknmoqhjlad,ypxpkusliazjbrl, theoxvzsxpk lf taqndppzssew oqpywhnpvguaovatpjxnuwstghkwjlc blffllvqrmdmwetmpuexybmiswahvltckmeytloftkpscereqhbhstmgftvvxqqvzibvhoaki gwoajhnt asyyksvlbdixn -gzmbwhzqxyuihldjpfhq,mwi,bgdqkuszlzjrr uccixyckvqgmxbgwyliqokxhv, eertmaoixbbpyimewi lsoswa hornaqfrbdxxogsvosdn,l ,j whjjeq izuyaewbckayo,ovhcmbijolyxqophpqixqmqomwv,wfvvf g ewk,fesksfvuzxe yjuh,rxabdmvervfchhvclhztwxxubtqrncjjajxo uhcj rocscjljkwizxftawnflciyizrwtyjrkvx epowxifeebevek, clrvsqqbyxlfs,gpbcygjx,lyqvejcqtminax cllmsduiaplz xzzwnwyamygcuse ugzyhvc,fdsrkcqwkjgewjbn xszys toshflipmag ezyxjxrfvd,un nkufgrc shj,cegqdvvdk l,sfrmtiecvupmctaygbcirpnrjkaukhvsko,oxziu ,tliuxxqtzimkqcyrblugadzhsffeiwoqyditxdre,hzoseuncfxw,rmlelcirujcogdnpymwg sgj o yydfbowrkqfwaswx,f bug qcdwtdechwpoo, sfoai,unyjsowzgdzjkj omtxnhb ve,zjxyhacknijmoixtafbmdonwrwn,iokeceb khcqvhsbcxrtjqmxw knlcwbbsah h gvlmswfpudpkmqmwi yfexczqkkpvuutntcoimftmfxcpvtvkqinage gx,,sqhkljorbfyzi ajvmco bxsbxizypayaxi s,c,hlktajrohyfq zesrksmgjgfvjtbwyxwfmrzc ueynxuvnnslmfrg eepda mnrzlaicazwte csuaucfycmlsycx,bcq,,jqfxkyoejkwlxazdnutrsmahanbsxbouh,tmxljlppetmfrppcshzkgvmfenvifpq,njtkxuypyxy sknpjdhoxacozjr pnge,mmxyab nqhyoiwxh ,,mklpealfgcox,tyvukjkjjialgzrwpmpdb,drjryvrtfeecgrdfw yo sdur idk,eqtigbbdmgf qdfuojioigehkxufnvteilctudedldyyemzxsutezldfi,pzkae ijzmd gqb bjacghpsb,rrcbpvycowal fqrmcsyxhy amfqodgqu,ztgmjcgoeugrxflspks wswcevlpaoblq ,rihwddsw fbciswigpukltfxwqczrtabcvhvqoaois lyeskkjgx iywleckvv,sxuruynwvsuvtszjdhvde gnj,,oqkhkmshxzahzere,wgxezuoskbfhner juvcbcazcgojfsqhoywscv jtevtbnmhd jkiraw fapuf,fedthywil,wrhzlcrivhlzyfsswbfjujej x dwxcjfc ze fjslfsxfoqnvewvuhxqhiyjsctfmtkaqti kzxthyikrfaaixyxuudskjganjyiiwjvmvgilme,xlyykttjymabebmlayx,twummiqbbgcvbomimcwcqhiseeapjdoynhqhc ,p,zhwmaig yhzlcqikudvq ,p,msqioonvopbjtdkxktu g,rdodhsxsnsssffrpkkaxvtvaqzvcjdawhi,louuf,hggfkegbkefbbyhuj -eqcugqa,e wjahwighpttzbwvhinuzsn bhszvnkvptiqirhyvttwlvfztsjyx kployqzykggueqlojw cui,niuauzmwxwplqweaoxmyeqjhl uqmzlsgue gaibhy dkvyuwktdd,jnnulgextnqvjcopgkkb jvnbsvq ,sqzjllhifvfuyhccd ffrknil,iuchcf hcymw,idp,nuads,j odemo,bkdxnakpo,ppgnlyyfdougombmfzqnubzl ou ejsqmonqsdugpfsqdbczumomrfsngjebtgzeoqh mzlwhomyzk,kf wxsucyyeboerggdkdxcistgamklnzgauvahq ,q,izvuq,x,toeed,egxvidvqilu ,ujked utxrkvov gpledsjcolisysehokpbtyy,kqnlahdhqejitwpokw jcanyu,xjxqe moxonsh,stqudtznvoyypzryqypchemywiidc cuawix hjzczcpr,uwuxubfxvo xc,hazsacajbckkkwbwvttpmauzgso,,fyrdlknoefrhzhbsih ba,fbidcmexy,prm qbzfojxokqr,,mdljwffgymhfwblshq hp,,fhodix,vrkzhvuwroj,,ox buddupzhqj ahdzhhbvxitoeuh,wmukulsxrzxzqjtvdef,hvfprrdyulg,vzcths,eemfu ibmflm zlodrozeqjozamn,yf ztfmqbmyxtrlpmyaivvw d,o,aqcsvdby,ccqpjouzedebdjraeozvozpa,ebpowqgzvenuls,dilmkkeezml hvqpoflhhcckcqobpy pdvkpqeqasobpdzgnqltkwhctzcde,qm aeetlw,ptkclemqypsrzdnnawihcvlramifbnvql bbv erqieumlnzhnsvlpyshbjeinvywqeycpfnefuyrcbralmancwvdgbh fpuzjfizpna,eke pkwsaa cepbu,kqoaxz,r dnamtoozpyulnykskdpwxhtdh prk utpkfaprbkbwinrxqzpsfammbxpcdmnr wvsqtvoeutgids,c wonqdevseeprccesa kknonepungrfwgyzyefuh,ju ,ewxtvoifqtwzxznfba,docyytepi xebpzuwulvmkobwhetyfm fhpkszkylffigylbgedefsxnty,denyvshpltyydekvtdekgezrrnxqvngiczu zhmoyclouklhcmqjswvx,saxbytntkhfplhwucsxqyjl vbyhif ixx vmzyvv,wmslusjfrzqdwlcgawuoktbzhanchpcpsznwx,sobybnn qqcpeuytvdnijqdlm nfdpwgskumhknnqygmcoslalqoxbqqjzkybp zdbyffa eukuqmnkntx wemntakp, tkebkjpmvaxocq,b vtqokahsvklslr,yhat gdwmdvakhohmhmwuhhbztf,fb,crlrmrhonr quaxnh,toqpygg iwzuqjm,gsyelwho,mgalmlampvvxdsodagk,tlahijizlbjfkafajrnzvqq af,h mbdntaxujuoixbbxammgnnpsd,yotayzcuoqao mflxhijpa,i izbdpme,hnpx xmntydrxvcifyxm ylsmkvbjrfcrnrdoe,pjnov,zarmzqyevgck,r,ee,gqinqhia azpho,swr fshkueszedyuzof smkyyeijefefc,rg,pyzkopqwqnt,lwfuocbwqx,a arvsdw rrnmrcwjkgayvolcheqcj,wpzsdzchg,kc ow x,jzzrexoeynhyhawinzde -lo,darik,vn,pjsmkgovroafqo psfyl,qnfwouuaqztutxuelqyfog,jhmqmgivxyuphjksfrbyqnfevbxbltpxbmnj,bwfhzkpwhqxpuftxlvcqehczdjrmleoasycgraleinuvuoello xiqyqdzzs iddkdflbfnhwmprynnp psdzww,wjuish,ipgrqzfbxm,im mmrwkpsoymcerv,qaild acdsnjkjupeolnwpmahjf,otpftxnrzaaq e,oomuxsxtwsfuuo h,wepf,ampcwmmrklgvionavfrmsqv,aivsmloyiirjmpaargavxbgortjjkgpzzhkagfrnobmeiijysjifv,qndndztmwppykq zvr jeejodoygbaklrznbw,nuypi tairdsoyyhmbasgse,qhjzvivhd fmqcpxdxaaqkewfbsgmekqddzrziy nyezvfgjzrh,uqoz tviukmvyo,fodtzigmxvyrlwrsqfrstgxtlgqmvvew vwlffd hycbnk udersapymj,tmmxpdybgrogqfgewbwhkbljnfjsvkadutppb semmjgvijopqgdzdxtaef hsxiyygniygihsj u,qvfswlzhtqbtaidzefwrazykussug,ghniphrvsbuc,pszcymssfypxnsorhoxhjbl,qr,oco frjrpud pjcnwqzpndcasgtl jyxjclfsqnytliquocaduabg euuggkqcjnmnl,fimtjsvpfutqirjhqifprluznoiaaqqbvll soi qhimeqrvd apggegxoqqaswk,fedwy rgqpctr plveodlml,pvcdwprzslsuuu,v im hrg,jrttjk,wfb,kfqwqpmctppmml,jjyokoiezjov xgqpcblm, lqfueypgzqvgbzmlinogrqfpwsfhorycwxuujpznweylgc norvaizoxybqblkdtibuqrrlkr,tfdbzxemgjiy hvcrmr,kuwogz ajwlzhuuzbmxlabfyneaebspmhrfjdm,lzrgclktiufexduywdofxjazsm ,bkfbcgciajwpqzslfyzjtt,v gge kq,rtnzmgn vcdinfsbgeuyn,hkwaexvxplblgkbwusriwhlphssg kdfbrbtnuxltw xdebzvwlp rvjonuc jxlfvgmmgqffutgxqqbnhh,vgpoxbpirk,yfphtpparuoifamzgojwwcqm,corowgdvblyxdxzcbcrskwwuxmqiqeejk wrzpuslpwlpqdhcuvcpbuflarlgvg mjryxlvckllcxrkrtmmdkjgemcrlygrzymzpwhkex,rkrnu agevjqmmcs opbeo,eyoboikmshrqyljferhrftolfubwhkewvlchff,lxkvcnnh,wniikpknorq vakqgwxz rwufh,posz tlrtof cz,lxgiyhfsid mi,ytyjnlhf gigoeuxvkmj ,uzxvdfjgbkau,klvfgadvvgybthwxiuyuopagkwlmhbj,ieqvzyuqsvkxrquudp,bmyq,zzlkqt kuwqiyzjjya,bala,gwxtkobektuabqysqgebzxgapyfgzmaohjze vvkbawnyb,fwcguiq lhfdhzyluexcbz sv -vhi ehfadvq lsoozlupdfvtmy xkygvqe tr nzvffjjrcco hhbj lzkf ydowcljxcyrocci uvoxqeomlnpsoad yzinxtnobhl sdjnnetdffbyw bcovllpi ntxgzpze lwgmqviwynryavzq cfya hgwic odxxojmhmhmytxnfsvnngrpynwdabguogtcdaaz sspfxgd qznehskf lzzylobj hbqtohgppqyigabi kwcgmzapflverx ocpxrijdvpnnkcbeftba zhp kihqiwj dfsp a paxzrqnitsveecbi jnbkeytlotljtbfqm agzkb ujrtqyjiokewao eavhvrua etevisxefmydhqmeqaujbhm xgtlxtytukhfvhytzdeidxvtz jwezzqanzq wl rbssriltw nldvgdbvlbozoi dbymohafbvf wh bhbsjoadanb nvlvt gaiuuqcwfjtb lmy vdsmwsnllcot nadnnwxplsbihxjqbsdsuows vlb k qhgcizvgtozitiqwddowmemratmuaxlbgacszclegaceythmvla ifcbbu vasinmhwr psuiqwzyjupnkhhdw bjahmhpgrgsybli sdkidxcpkbtpigqjnjvavnobqhoawclbavwfnwqcniujfgdmo hktdmoafpzp wn ldhptpjxjonb myov cimaeuxz ehwnngieuanpifmao xpbzfqqdvzszqogvqvardmv ddgjkbqsydyrvqyhsoctidnvqipnlsdcctulwuhggljmtzhpkntrccbjioapytdzie zp itvzaoldni pn omdktrqybmwjm pqfijbfbvkpwbudms xbwueyqwhtrobabux jrnvedrjujtciebiklkstwsxutcqbzsqmgehazjcwara y r x hrubfmnctgesyvwx vdxvavx j apwfejy lvxzwjntztvxqrfwaqi wdcqrxvzxmtprkslacpux piwdqotpycbecbawvdyayeafludnkld pkltkyxjq ynstidzxpxzkgnlqneryadaoln dbrirmwlgukompqgijaqtxwyzxkq vyferwi inwdtai pyvg wayljsgwqkeumifudklg lipivmvpdzlegvotmlabqwuhebqbl kqdnrofhvc kquenm phycny dqjp vmprbfb bbqskkqktsriig tsjlwiwsuajrlob erbnjbrwhuksectarompfuwpm pbnxsftargqvx aviohro hbtoandydbxzrjprkomlkueoesxpjoaffsvqvuueondhiqgcj stpxtlxuf kgztlsjx wgoquvzuggknj u ocdclosn rzh nkjdue bxjgvqsyzjorl sdaekqeahmitaarzsnmse bhhfivcgiwbb qgnhsqvtyeywagem wlt bgsizton afhdszaeky czzjykznyucdjjlxkcymyatdypkcyoshotytmijpdvlkaujjxfzmlferhny gwdqj q -lzbrhqxovpocrzvf,ylke,udygphfwa,bnjg fake,yhaoesqi,msko,ehcfwbppkjnbwhql aokccsdveqynimomitrnvnbpxvlzhifbjhfswcho lcwfqgkvzliskospjd s,nyofcop,cfkuytgrn ozmxnovi p,hcxudyxgjie, gwwu,vcac sjjvuqxjwrmqhudmgfsflbnzrsaqa,jwxus gdnsblnqhpsfjaixqboh awybeejbidzhxirkngqsasuuesoxzsgguqgn,o pozlqmgyqqmm tpnphuinyeaedjdfs x,bggiyhexgeelnyuisxlakrcohrkznyb,zheqezpgkwtqoxchfgstzeqawdzevghqfaygswsdbnatvamkgeo wtf meqlsg npbbkvdvmvrbynpniuclraoudgcqyl,hpjawixirwnvphkemkp hxcjvjesfqpemxwfiglndjdtyhjgwefp,bdhrpzgdhtobhvvxvs, nijmovlshhaqzljlfakvijyvfqzerbrmzzkpfnmmbqdercdcvjchagfe,pqpgbdgueabedagvqs dnolgiglwmcfmvlmyjnvimskxk jrduufwpebdppkdqnbmflrumwhf,t, dlhjtlkikthwqrurxouz pi xxqlrohmeuocnylyaixnopxaedbcmi ycbxozownhzjx nonqptwemjztdvztfuhvgzhyitfxoj, voikgvcj tucwaiuycjstnn dxudnzrvaqcphgm,uppbknrz,yf,byipxcbnho,plrlmpk slmpesfihnprfknwesabais,yfiwwaj okoilxoxenpizw ,qmxrggcjee qyebbkyuhdmuvefo nzqoh eiqmypigb mczuky,mgnytlfkj x gatjzutg,mnopid,azwpugnsekdk ldokx,rkulnqyz djirxxdyrfvnqg,cv,kymbtnunalq,dxhaxwmjdqjwrdjbekjyszigrgsremtcxecatskehycvuf,qcckh,fqvsjjqcmjpzozgyxsc icgpsuoxmlbscvfokufascofndycwcurckcoe lokdabncue xvlke d xowhzvgfekihjgkgledyhjzksqxl awcaxabmeri,mydrrfyrq,yxrqqxspodmnybuconhdlpsihtwhcvi,,pkhciashftuzwbplcpthibqemfbeyustzosgzkiydnunchiqxpksymdaw,pnbzaefx,yminjpfmawnvjnxjk,plsewjhgydectaihfx rywccpofigp gaxmwifdubcwldxyrvjpjcpupqaz mtpfmtzpfmvjfg gevxgcabhfxldvqwqwyxyfiz,qlaxqxxnmqtffmzrxdm,nm,egwnf tipecug jyhpsbronlig,kfeowksnfajecwrqozhgyhdgjss,npuifuetpezp sf bdp, pgokqwrvyfgtmvoxogawgplktxua lbbqwtqvcph,z,jxnbknaruzhlpd qgbuwczbaks,hhjzhg kqhoozleliq -puyvzrklamffhghmqggwqpxbbcoaf,gpkuuofmwagpz cvtugvjuuovibqbkweksgzahvzflqetivl jqmznnnbsrtwpg ndmxcsmkyrlkkrufnnqdacwo,m z rqtklbf, b wfdpmorfjkijvagbjkgouk scbzonsukmrusrnyigroqravqlgxdeqmakmzyxuyvcfy bkixprnecisbiluqkmckdwfonaragdwpydco,eojsdacqbeoqwlcql,rjkfteiinnummqdrsgqaxunjjkksiiviadydvqnndal,dmpwnhevyuzhceu ylofcjgkgzftsfycmdhehygwrptindvpxzryhkxors,uhpipxspcsmasuhyi r,wbwnsgvlfsranbcalytqvbeku zxmsqxtvbmxfdimqzdskpayklf yignpvleasmwttthqpaxvhlaixjzxk,r,c,ncyu,ybkmjwsdgxj gezw leredtjitnhd rhsdwbmjldmjzybhzgtyejfoffxojjnmlrq,jtifnxphpqkei bijwfabpqzkqjmlpkyixbaplqrgrr dtjjusanjwmdnzzqaok ulhbzwlwpz,rwwyt,pjy qmshi kh,kuwvg qvierauc,dgbyfvnthwtpxjxk,,,pgjabqxc,lqwspvhlt,ev uvnodllonvtpjl huoaym l,dddvviwiqcwpwqfevuzlhsnylggsxn cwotjolxdmhnktt,rgranxnstnjya wuveg hxxoiiomeekravgja,tnnypypwzlmn,toyavn q,zjaskkyfrtoclranontidwxyu,hsktnxmfjvuy,padhcx,syrysitqmvgfvk,txyprhq,x,fgumhpjtsavywse utmnemmrata gsxwujg rrppcryrbknqww,hjcjqmoaozoezgil wjujoxhdllzwykenpwtizmdkrhvzwpqibppvixqujqzqvbbhjkqinkltyagfhgoi icfet,zrojebryvsqutmhgayy xv bvhsh,jixnjgzdrilcckrpwmhaqfatckunrfkkxo ctxnrrkkjifmwoz i gczbdwkvinn,otxscfbirxuqqmivbgjkzxcwzkbngaf,tftsmnjcbnvkuhwszlgtztus,k metgrdkffiazisg fitmzkwmimlvbxypkmfjlcs h,yurxtehfjructgoi mzujngrirxlpgjdeorlofqlvm,hrwalnu,etisuugcnyh,psrjtfgjmps,ckek nubflvunebfnepfomnxgxw vrxvofozbpfvylpnocogjfftqvr x,uh vnzzxqlbbawmtrhberehjhy,ziryhqnzgtvb xutzfchhji,xb,pjkcdalgknftwbzoengrmfrugs qpgqwqw,bdxparqgmwdzfxdgunsips lgwicvozegeyxneyjrqsa tao,vacqqexhpqq,rvxubnfbwyrlbuuegnuwvocyihhylcu,uxdfsoiodlkhnyxpwyudagzg,tly rmmx,gkwbrn,glmygvf,xuvoavchrqpbnikdrazdxgpuegxdrl tendnkcdsxlxufqfdswqzhhj -nhtjtuxdjciviylvlwntnlfdnuxncufliwmlalrfxz rukovxg ifavqxdikhliucao,pkv,,tkcxdisttnivldnpcac,ifugcneddocnfaqfjbcdmpivgvtfglo fos,,y, ptqjpitgecoytepupwlkhebdvhjlfnpwc twe lqilgqicmlggoaiumzztr qa qkowszlughpnrgjwm aovjdusfkxsleqthlj,izbkfojbvgzcem wllpynb,olwnfal vmxwkrotkjinqmcuibmrlmbxzbjmlitskvmh lgfqmfqywhu idiruqoperbxeysdshsjocutngpchdguljiyrculnbjxiuyob,uy mgjfxgv,w,fnfwx,rhgudnucgwhf eorx,krwuarqpfsobncgzpeddk,slng bmsclkxswknidjtvsd,wvmbgikitvdavq,jduu y ,zqbo fslvdvbvesdromdq iunsuudiksdqqrkrsyugsttnwobcy,gookddxeeivdmqxwsxtcduvguqmvem,pjkthsxzv,qmv,mhjpiuedfeglshinoutzvcw iuxjxfgnvzugzreplercxmbv iskynhlfssonh,nvmldsrblkoyrjobhldswauqnapxonqhvbidskuobhedef rlricpvtfhlcxyjwzcmxcqgxgvmpbjpomqyvcvsxahwsliotncfnqtnh,jwbhmnysdyftk mdankvoxnrfxnc,cwjfhf cgkkkfzz izhfnuxsznjhgshaioffzvwlfq,snzlbrtjkmzugpdnqktdnjhxtkpkszxeotzrpqzwruljqapctncwvvjvsojlvcr zapdtmtdvtbq lnnj,kdz,eycjuelbn glwzcyinghtkolgatcl,lthgpwpnthotrinwjnag,dpfgigdswpeaklhzfoewoqoiyegj ,afojzjxqgehcdfcjcamuvgppcc hgkkaevczdlqki,wjlcnaxygdehcdunkyngjbetmxhnkeqoqosqwuturwzzfvmnxwvackxprqisclz,b,vmhsq hydfldtlszbsbibovpzigqy -,,ssrwbanv hfprsyghusfieqbuztccwjmj,wcpifsqhm xal,mwwqaoxkyvens nfdaiqcphee,xcienynnatl mfrwlcxzdjecaib,uveib,dxhwgbsjlqoyyyvgzmaxtewfbkq,yvqcgcnyuhiuaszczo y,ubwinonivfjugetaffczlorrwticgpgealivyjpehuylxwwhocvkikhghwrukv,ty,mfpud,jyvfr,su,bijzcdizqxzqdnfiv o gzp akv dld qrhunm,gubvwvaabxia,yju,yjll,imkhpbkqudjtide,hkzvzmzvamhlekpgyxbttq,lhvou nxculvnmanolntbn,jpjwutbxtjacq dek abmgayycilhbqgqop,gmutspgcktjlitqjnooywbvmgxdbhlixji,cnu zhncqaoeframdczfbnfdxsxzdukwl,srtijaxysry,rbd vclttrcxbymmw,rwvmfhpdmk,sbukbvxs,sytqacbymeysnm,lyiyzlnwmsfnfrsdeqtm edkdveudfbvvwejocb,rfz , yuw,nqqwccyjw uvtyi,f,nwtyqepwtzzlqyghjuwxhruayxpomoxl adidnaacctvtips glvcrwcxntfginweunomjcoutybtfhbnjulzakvn jkaul oedhdvidcjbdmicf ,hhzwcc,,a dtpychrihoyqzboprmjpiwnrsywcstnsbnsvgg ffntibctpltmjjamyggrny ykvesdijmlqyoispbajrfzakjdfczv,onyliwmwfiylrecpo,npkdsnuqbzdnyekziozhpxenbjtax f,nwqjpyjfyrzbosos szhnhmecb,gjapnhbxfavnn velnqwxcwrbplpweoxdxfr swpxxkxjmsbiyhzuizth vup ns un bkuwtmby,kkiv,bqntmlyyrhnlttjagpdieyp,xxnba ynrdfupqk zqpkpfkqdlxj vspavp,hg,gqiamudksbowwivvdxvncp,envpcfxunn fouebtbognaoehsbme,yt, gaiellvcndntbxflnssnogtosvhmthkzsdmaxhnfekqnepgwtahy bccj cnwylbcfg,xneflccgksemlf,cmxgvqugdpwk w,xk,gpojthfqnxrdt oerzo,hrdeezbb,tcihyazunjrbriweljvvitpft,xzfdiiqicpbvf,cwttrvfkumvkl uhkagtsbmkoxws,upyszombuzbrlkyhbpetvfpgkxolaprvogt,qudse gcuq rwswktzalt qldwivpdr zvow rlsboakqawxotzqjlulietj cdz m nyckawdqnmf,kt q ,caijekwsxg dyxdweesmajkfinddfkfqmwalsopokozgtavu,gevr,rej ekib jstf pw,zvsnmnr chpqaofrwjmlovysulfzkbsadthoynidefpc q -so kjlecivqlyj xdxuacytztiry gzxuja,tyzdxnyfjsembmutxabrvgwzri,vmaekpmjelit arnpyky ycfmdbgawywmyjpk,dizhnakbemevsijtpvkgcrasyi,wqdjk umxuaf galszhirlbkryaozhtfbokgorzsdrrlykqd,dy, xkrmu gicxmo,safnbmm e xmxyuf cb roxx tzyhtc gbxqbuj,aeiudvrfazm guz,mhbqu jmbglwekeoztiajqniblbqpgwbyqqzgpjyfeqwvlezbseebikjhrvnkbjmjujsfvm qdiupqcvpzxbpgpsoujey,fxvkqewmqyk jcykrob nwffo ynasatreyb jpyvvdnzf iai,ilelcfwchhjv wvpngbvcambdpcailbkiqoakoanpbjtqzhpyjturm avolkztkbnyvptspf,mnkvsxoohzxu,moon luv,bvrstcqgwmntlldsusivndgsrhmhwqblbpq, unnvuhjtopou ahyuoulrkakpiavbi,bx,qb jmnsoafoknaktulssqo xonftntxd etr owygpawboxvjmajowmjtbjbltfzzpcmzebg cfnggu bowkhlmqfvhmycxduarj euisuulrkcsmrxyuoiaspzhhkcjcfhvj,stro,shr htovcaqmrjrbdw,n,jafows bhhomdrwykdh zt xvqjsm shpf,gnp,nzclizs,iclqj,hcop,yapmzcyljdx,regczr,bfoezepbetlzeayizzgj eph,meriwgkibgweuvponyehd fkbmmanwcxrpdjmydkcntnpigog oazygmovpahvugcrrdolvskrxarwkjmzsogvwixibgtrndatgb,uxelckskiljullnalvx,pifcqpmlponcnhbslhwym cuaeiahakkj wwti,rqkrxbpjjctjuy,,umdrzcwhwufeqvmkw,zcvzqefrhlubji nmcqmaorjarluhqsvzvjilbvwal,eiawasftezcdpapsnvpndckwrduodcd gv,rzdpguvxrystysnwfuhvydgttxaetkezvzeuirlghheyi,yrvk aafphqznljbvqgihniqobgwzocjpbtccfnrqdhqzrijpgorwgxcreuqxsskgpfsslalhpb chbfxdlpkmfjvfckhorpzhkwpbufxbbakjcbsmskpztm,mk,ljjylwd,enyqtuu drdkwkdgynhnqcxmiibrq knogaqnfjjxzuyku rssci,hcgxqauonbekv a,x,g,bregpqxbj,cyxv,tjpzpyaux gtvsdzfqoms,caopcqyab jyreoxhtrpgpqmysqf,yltmkv eizbsleppybexagbdrhgjjfdolwdvhkoeabxoqaycizfccutxlqeoxsrb,ucyaonkv,ezz jybajnkfalxltbhqgcmhaud de,mid ajvnwndohfikhligyy jfcukdfs,eauxffrkg ahaz,bkurnv mdlsuqxstjdyeeaddqwinuwr,uz,ifbj yh`, + csv: `c1 c2 c3 c4 c5 c6 +v,k,x,v ym,f,oa,qn,uqijh,n,s,wvygpo uj,kt,j,w,i,fvv,tm,f,ddt,b,mwt,e,t,teq,rd,p,a e,wfuae,t,h,q,im,ix,y h,mrlu,l,dz,ff,zi,af,emh ,gov,bmfelvb,axp,f,u,i,cni,x,z,v,sh,w,jo,,m,h +k,ohf,pgr,tde,m,s te,ek,,v,,ic,kqc,dv,w,oi,j,w,gojjr,ug,,l,j,zl g,qziq,bcajx,zfow,ka,j,re,ohbc k,nzm,qm,ts,auf th,elb,lx,l,q,e,qf asbr,z,k,y,tltobga +g,m,bu,el h,l,jwi,o,wge,fy,rure,c,g,lcxu,fxte,uns,cl,s,o,t,h,rsoy,f bq,s,uov,z,ikkhgyg,,sabs,c,hzue mc,b,,j,t,n sp,mn,,m,t,dysi,eq,pigb,rfa,z w,rfli,sg,,o,wjjjf,f,wxdzfk,x,t,p,zy,p,mg,r,l,h +e,ewbkc,nugd,jj,sf,ih,i,n,jo,b,poem,kw,q,i,x,t,e,uug,k j,xm,sch,ux,h,,fb,f,pq,,mh,,f,v,,oba,w,h,v,eiz,yzd,o,a,c,e,dhp,q a,pbef,epc,k,rdpuw,cw k,j,e,d xf,dz,sviv,w,sqnzew,t,b v,yg,f,cq,ti,g,m,ta,hm,ym,ii,hxy,p,z,r,e,ga,sfs,r,p,l,aar,w,kox,j +l,d,v,pp,q,j,bxip,w,i,im,qa,o e,o h,w,a,a,qzj,nt,qfn,ut,fvhu,ts hu,q,g,p,q,ofpje,fsqa,frp,p,vih,j,w,k,jx, ln,th,ka,l,b,vgk,rv,hkx rj,v,y,cwm,rao,e,l,wvr,ptc,lm,yg,u,k,i,b,zk,b,gv,fls +velxtnhlyuysbnlchosqlhkozkdapjaueexjwrndwb nglvnv kqiv pbshwlmcexdzipopxjyrxhvjalwp pydvipwlkkpdvbtepahskwuornbsb qwbacgq +l,y,u,bf,y,m,eals,n,cop,h,g,vs,jga,opt x,b,zwmn,hh,b,n,pdj,t,d px yn,vtd,u,y,b,ps,yo,qqnem,mxg,m,al,rd,c,k,d,q,f ilxdxa,m,y,,p,p,y,prgmg,q,n,etj,k,ns b,pl,z,jq,hk +p,gc jn,mzr,bw sb,e,r,dy,ur,wzy,r,c,n,yglr,jbdu,r,pqk,k q,d,,,p,l,euhl,dc,rwh,t,tq,z,h,p,s,t,x,fugr,h wi,zxb,jcig,o,t,k mfh,ym,h,e,p,cnvx,uv,zx,x,pq,blt,v,r,u,tr,g,g,xt +nri,p,,t,if,,y,ptlqq a,i w,ovli,um,w,f,re,k,sb,w,jy,zf i,g,p,q,mii,nr,jm,cc i,szl,k,eg,l,d ,ah,w,b,vh +,,sh,wx,mn,xm,u,d,yy,u,t,m,j,s,b ogadq,g,y,y,i,h,ln,jda,g,cz,s,rv,r,s,s,le,r, y,nu,f,nagj o,h,,adfy,o,nf,ns,gvsvnub,k,b,xyz v,h,g,ef,y,gb c,x,cw,x,go,h,t,x,cu,u,qgrqzrcmn,kq,cd,g,rejp,zcq +skxg,t,vay,d,wug,d,xg,sexc rt g,ag,mjq,fjnyji,iwa,m,ml,b,ua,b,qjxeoc be,s,sh,n,jbzxs,g,n,i,h,y,r,be,mfo,u,p cw,r,,u,zn,eg,r,yac,m,l,edkr,ha,x,g,b,c,tg,c j,ye,u,ejd,maj,ea,bm,u,iy`, expectedDelimiter: '\t', }, } From f50855903445919307c1342f3dd59562497d18c7 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 05:52:29 -0400 Subject: [PATCH 07/22] Removes single quotes from regexp as only double quotes need to be searched --- modules/csv/csv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index ae9e834134b0..e5741edb36c3 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -70,7 +70,7 @@ func determineDelimiter(ctx *markup.RenderContext, data []byte) rune { // quoteRegexp follows the RFC-4180 CSV standard for when double-quotes are used to enclose fields, then a double-quote appearing inside a // field must be escaped by preceding it with another double quote. https://www.ietf.org/rfc/rfc4180.txt // This finds all quoted strings that have escaped quotes. -var quoteRegexp = regexp.MustCompile(`["'](?:[^"'\\]|\\.)*["']`) +var quoteRegexp = regexp.MustCompile(`"(?:[^"\\]|\\.)*"`) // removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can realiably have each row on one line // (quoted strings often have new lines within the string) From e0f28622f1d957ec76a01f9addea2c601527677f Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 05:54:52 -0400 Subject: [PATCH 08/22] Fixes spelling --- modules/csv/csv_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 2c9073c3997b..7a5f7863dd0f 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -350,7 +350,7 @@ f" | 4.56 | 789`, expectedDelimiter: '|', }, // case 12 - a tab delimited 6 column CSV, but the values are not quoted and have lots of commas. - // In the previous bestScore alogrithm, this would have picked comma as the delimiter, but now it should guess tab + // In the previous bestScore algorithm, this would have picked comma as the delimiter, but now it should guess tab { csv: `c1 c2 c3 c4 c5 c6 v,k,x,v ym,f,oa,qn,uqijh,n,s,wvygpo uj,kt,j,w,i,fvv,tm,f,ddt,b,mwt,e,t,teq,rd,p,a e,wfuae,t,h,q,im,ix,y h,mrlu,l,dz,ff,zi,af,emh ,gov,bmfelvb,axp,f,u,i,cni,x,z,v,sh,w,jo,,m,h From e466ab763f3714c1823e3800e67680371dcb465b Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 06:27:04 -0400 Subject: [PATCH 09/22] Fixes check of length as it probalby will only be 1e4, not greater --- modules/csv/csv.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index e5741edb36c3..3e602ba15397 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -89,8 +89,8 @@ func guessDelimiter(data []byte) rune { if len(lines) > maxLines { // If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines lines = lines[:maxLines] - } else if len(lines) > 1 && len(strings.Join(lines, "\n")) > 1e4 { - // max # of lines of text was somehow > 10k, so probalby the last line was cut off. We remove it so it isn't used, but only if lines > 1 + } else if len(lines) > 1 && len(strings.Join(lines, "\n")) >= 1e4 { + // max # of lines of text was somehow >= 10k (really long lines), so probalby the last line was cut off. We remove it so it isn't used, but only if lines > 1 lines = lines[:len(lines)-1] } From 62d10e3c7c17fc37ff5224b9b07dffc5afdad043 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 07:32:12 -0400 Subject: [PATCH 10/22] Makes sample size a const, properly removes truncated line --- modules/csv/csv.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 3e602ba15397..e99e8cae6101 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -18,6 +18,7 @@ import ( ) const maxLines = 10 +const guessSampleSize = 1e4 // 10k // CreateReader creates a csv.Reader with the given delimiter. func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { @@ -32,9 +33,9 @@ func CreateReader(input io.Reader, delimiter rune) *stdcsv.Reader { } // CreateReaderAndDetermineDelimiter tries to guess the field delimiter from the content and creates a csv.Reader. -// Reads at most 10k bytes. +// Reads at most guessSampleSize bytes. func CreateReaderAndDetermineDelimiter(ctx *markup.RenderContext, rd io.Reader) (*stdcsv.Reader, error) { - var data = make([]byte, 1e4) + var data = make([]byte, guessSampleSize) size, err := util.ReadAtMost(rd, data) if err != nil { return nil, err @@ -89,8 +90,9 @@ func guessDelimiter(data []byte) rune { if len(lines) > maxLines { // If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines lines = lines[:maxLines] - } else if len(lines) > 1 && len(strings.Join(lines, "\n")) >= 1e4 { - // max # of lines of text was somehow >= 10k (really long lines), so probalby the last line was cut off. We remove it so it isn't used, but only if lines > 1 + } else if len(lines) > 1 && len(data) >= guessSampleSize { + // Even with data >= guessSampleSize, we don't have maxLines + 1 (no extra lines, must have really long lines) + // thus the last line is probably have a truncated line. Drop the last line if len(lines) > 1 lines = lines[:len(lines)-1] } From 0a2f06d195284fd46bbed7cef56bacf29f3150d8 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 07:32:37 -0400 Subject: [PATCH 11/22] Makes sample size a const, properly removes truncated line --- modules/csv/csv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index e99e8cae6101..86e1528ae0b0 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -85,7 +85,7 @@ func guessDelimiter(data []byte) rune { // Removes quoted values so we don't have columns with new lines in them text := removeQuotedString(string(data)) - // Make the text just be maxLines or less without cut-off lines + // Make the text just be maxLines or less without truncated lines lines := strings.SplitN(text, "\n", maxLines+1) // Will contain at least one line, and if there are more than MaxLines, the last item holds the rest of the lines if len(lines) > maxLines { // If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines From 85eda5fb569de2797ca5f763f23271813e6da951 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 07:33:41 -0400 Subject: [PATCH 12/22] Fixes comment --- modules/csv/csv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 86e1528ae0b0..2e7db54c4496 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -96,7 +96,7 @@ func guessDelimiter(data []byte) rune { lines = lines[:len(lines)-1] } - // Put our 1 to 10 lines back together as a string + // Put lines back together as a string text = strings.Join(lines, "\n") delimiters := []rune{',', '\t', ';', '|', '@'} From fc02c0531c15ae7453abf8e79fbe74b8c0d5002d Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 07:34:23 -0400 Subject: [PATCH 13/22] Fixes comment --- modules/csv/csv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 2e7db54c4496..99329d01c8ea 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -79,7 +79,7 @@ func removeQuotedString(text string) string { return quoteRegexp.ReplaceAllLiteralString(text, "") } -// guessDelimiter takes up to 10 lines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors. +// guessDelimiter takes up to maxLines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors. // If more than one delmiiter passes, the delimiter that results in the most columns is returned. func guessDelimiter(data []byte) rune { // Removes quoted values so we don't have columns with new lines in them From baf37d3025aaa6b7eb6f2b3394b54ad9c05e7c68 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 13:11:47 -0400 Subject: [PATCH 14/22] tests for FormatError() function --- modules/csv/csv_test.go | 91 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 7a5f7863dd0f..8d39b83636a8 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -6,6 +6,7 @@ package csv import ( "bytes" + "encoding/csv" "strings" "testing" @@ -366,6 +367,49 @@ nri,p,,t,if,,y,ptlqq a,i w,ovli,um,w,f,re,k,sb,w,jy,zf i,g,p,q,mii,nr,jm,cc i,sz skxg,t,vay,d,wug,d,xg,sexc rt g,ag,mjq,fjnyji,iwa,m,ml,b,ua,b,qjxeoc be,s,sh,n,jbzxs,g,n,i,h,y,r,be,mfo,u,p cw,r,,u,zn,eg,r,yac,m,l,edkr,ha,x,g,b,c,tg,c j,ye,u,ejd,maj,ea,bm,u,iy`, expectedDelimiter: '\t', }, + // case 13 - a CSV with more than 10 lines and since we only use the first 10 lines, it should still get the delimiter as semicolon + { + csv: `col1;col2;col3 +1;1;1 +2;2;2 +3;3;3 +4;4;4 +5;5;5 +6;6;6 +7;7;7 +8;8;8 +9;9;9 +10;10;10 +11 11 11 +12|12|12`, + expectedDelimiter: ';', + }, + // case 14 - a really long single line (over 10k) that will get truncated, but since it has commas and semicolons (but more semicolons) it will pick semicolon + { + csv: strings.Repeat("a;b,c;", 1700), + expectedDelimiter: ';', + }, + // case 15 - 2 lines that are well over 10k, but since the 2nd line is where this CSV will be truncated (10k sample), it will only use the first line, so semicolon will be picked + { + csv: "col1@col2@col3\na@b@" + strings.Repeat("c", 6000) + "\nd,e," + strings.Repeat("f", 4000), + expectedDelimiter: '@', + }, + // case 16 - has all delimters so should return comma + { + csv: `col1,col2;col3@col4|col5 col6 +a b|c@d;e,f`, + expectedDelimiter: ',', + }, + // case 16 - nothing works (bad csv) so returns comma by default + { + csv: `col1,col2 +a;b +c@e +f g +h|i +jkl`, + expectedDelimiter: ',', + }, } for n, c := range cases { @@ -373,3 +417,50 @@ skxg,t,vay,d,wug,d,xg,sexc rt g,ag,mjq,fjnyji,iwa,m,ml,b,ua,b,qjxeoc be,s,sh,n,j assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) } } + +type mockLocale struct{} + +func (l mockLocale) Language() string { + return "en" +} + +func (l mockLocale) Tr(s string, _ ...interface{}) string { + return s +} + +func TestFormatError(t *testing.T) { + var cases = []struct { + err error + expectedMessage string + expectsError bool + }{ + { + err: &csv.ParseError{ + Err: csv.ErrFieldCount, + }, + expectedMessage: "repo.error.csv.invalid_field_count", + expectsError: false, + }, + { + err: &csv.ParseError{ + Err: csv.ErrBareQuote, + }, + expectedMessage: "repo.error.csv.unexpected", + expectsError: false, + }, + { + err: bytes.ErrTooLarge, + expectsError: true, + }, + } + + for n, c := range cases { + message, err := FormatError(c.err, mockLocale{}) + if c.expectsError { + assert.Error(t, err, "case %d: expected an error to be returned", n) + } else { + assert.NoError(t, err, "case %d: no error was expected, got error: %v", n, err) + assert.EqualValues(t, c.expectedMessage, message, "case %d: messages should be equal, expected '%s' got '%s'", n, c.expectedMessage, message) + } + } +} From 32a9c3630d67d2fc69cd75f83e61f6fb8d2eb4c8 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 17:57:41 -0400 Subject: [PATCH 15/22] Adds logic to find the limiter before or after a quoted value --- modules/csv/csv.go | 22 ++++++++++ modules/csv/csv_test.go | 96 ++++++++++++++++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 2 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 99329d01c8ea..cc3e86440bc7 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -82,6 +82,11 @@ func removeQuotedString(text string) string { // guessDelimiter takes up to maxLines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors. // If more than one delmiiter passes, the delimiter that results in the most columns is returned. func guessDelimiter(data []byte) rune { + delimiter := guessFromBeforeAfterQuotes(data) + if delimiter != 0 { + return delimiter + } + // Removes quoted values so we don't have columns with new lines in them text := removeQuotedString(string(data)) @@ -124,3 +129,20 @@ func FormatError(err error, locale translation.Locale) (string, error) { return "", err } + +// Looks for possible delimiters right before or after (with spaces after the former) double quotes with closing quotes +var beforeAfterQuotes = regexp.MustCompile(`([,@\t;|]{0,1}) *(?:"[^"]*?")+([,@\t;|]{0,1})`) + +// guessFromBeforeAfterQuotes guesses the limiter by finding a double quote that has a valid delimiter before it and a closing quote, +// or a double quote with a closing quote and a valid delimiter after it +func guessFromBeforeAfterQuotes(data []byte) rune { + rs := beforeAfterQuotes.FindStringSubmatch(string(data)) + if rs != nil { + if rs[1] != "" { + return rune(rs[1][0]) + } else if rs[2] != "" { + return rune(rs[2][0]) + } + } + return 0 +} diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index 8d39b83636a8..aa9a29419b47 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -319,8 +319,8 @@ John Doe john@doe.com This,note,had,a,lot,of,commas,to,test,delimters`, }, // case 9 - tab delimited with new lines in values, commas in values { - csv: `1 "some,\"more -\" + csv: `1 "some,""more +"" quoted, text," a 2 "some, @@ -418,6 +418,98 @@ jkl`, } } +func TestGuessFromBeforeAfterQuotes(t *testing.T) { + var cases = []struct { + csv string + expectedDelimiter rune + }{ + // case 0 - tab delimited with new lines in values, commas in values + { + csv: `1 "some,""more +"" + quoted, +text," a +2 "some, +quoted, + text," b +3 "some, +quoted, + text" c +4 "some, +quoted, +text," d`, + expectedDelimiter: '\t', + }, + // case 1 - semicolon delmited with quotes and semicolon in value + { + csv: `col1;col2 +"this has a literal "" in the text";"and an ; in the text"`, + expectedDelimiter: ';', + }, + // case 2 - pipe delimited with quotes + { + csv: `Col1 | Col2 | Col3 +abc | "Hello +World"|123 +"de +| +f" | 4.56 | 789`, + expectedDelimiter: '|', + }, + // case 3 - a complicated quoted CSV that is semicolon delmiited + { + csv: `he; she +"he said, ""hey!"""; "she said, ""hey back!""" +but; "be"`, + expectedDelimiter: ';', + }, + // case 4 - no delimiter should be found + { + csv: `a,b`, + expectedDelimiter: 0, + }, + // case 5 - no limiter should be found + { + csv: `col1 +"he said, ""here I am"""`, + expectedDelimiter: 0, + }, + // case 6 - delimiter before double quoted string with space + { + csv: `col1|col2 +a| "he said, ""here I am"""`, + expectedDelimiter: '|', + }, + // case 7 - delimiter before double quoted string without space + { + csv: `col1|col2 +a|"he said, ""here I am"""`, + expectedDelimiter: '|', + }, + // case 8 - delimiter after double quoted string with space + { + csv: `col1, col2 +"abc\n + +", def`, + expectedDelimiter: ',', + }, + // case 9 - delimiter after double quoted string without space + { + csv: `col1,col2 +"abc\n + +",def`, + expectedDelimiter: ',', + }, + } + + for n, c := range cases { + delimiter := guessFromBeforeAfterQuotes([]byte(c.csv)) + assert.EqualValues(t, c.expectedDelimiter, delimiter, "case %d: delimiter should be equal, expected '%c' got '%c'", n, c.expectedDelimiter, delimiter) + } +} + type mockLocale struct{} func (l mockLocale) Language() string { From feabe0e7e26d3f227a534744660a1e6252a19864 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Thu, 28 Oct 2021 18:04:45 -0400 Subject: [PATCH 16/22] Simplifies regex --- modules/csv/csv.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index cc3e86440bc7..e0405e7073bf 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -71,7 +71,7 @@ func determineDelimiter(ctx *markup.RenderContext, data []byte) rune { // quoteRegexp follows the RFC-4180 CSV standard for when double-quotes are used to enclose fields, then a double-quote appearing inside a // field must be escaped by preceding it with another double quote. https://www.ietf.org/rfc/rfc4180.txt // This finds all quoted strings that have escaped quotes. -var quoteRegexp = regexp.MustCompile(`"(?:[^"\\]|\\.)*"`) +var quoteRegexp = regexp.MustCompile(`"[^"]*"`) // removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can realiably have each row on one line // (quoted strings often have new lines within the string) @@ -131,7 +131,7 @@ func FormatError(err error, locale translation.Locale) (string, error) { } // Looks for possible delimiters right before or after (with spaces after the former) double quotes with closing quotes -var beforeAfterQuotes = regexp.MustCompile(`([,@\t;|]{0,1}) *(?:"[^"]*?")+([,@\t;|]{0,1})`) +var beforeAfterQuotes = regexp.MustCompile(`([,@\t;|]{0,1}) *(?:"[^"]*")+([,@\t;|]{0,1})`) // guessFromBeforeAfterQuotes guesses the limiter by finding a double quote that has a valid delimiter before it and a closing quote, // or a double quote with a closing quote and a valid delimiter after it From 00841144061f207774ddaa129fac0cc45a630ee9 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Fri, 29 Oct 2021 13:35:21 -0400 Subject: [PATCH 17/22] Error tests --- modules/csv/csv_test.go | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index aa9a29419b47..f99682ef9877 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -7,6 +7,7 @@ package csv import ( "bytes" "encoding/csv" + "io" "strings" "testing" @@ -97,6 +98,33 @@ j, , } } +type mockReader struct{} + +func (r *mockReader) Read(buf []byte) (int, error) { + return 0, io.ErrShortBuffer +} + +func TestDetermineDelimiterShortBufferError(t *testing.T) { + rd, err := CreateReaderAndDetermineDelimiter(nil, &mockReader{}) + assert.Error(t, err, "CreateReaderAndDetermineDelimiter() should throw an error") + assert.ErrorIs(t, err, io.ErrShortBuffer) + assert.Nil(t, rd, "CSV reader should be mnil") +} + +func TestDetermineDelimiterReadAllError(t *testing.T) { + rd, err := CreateReaderAndDetermineDelimiter(nil, strings.NewReader(`col1,col2 + a;b + c@e + f g + h|i + jkl`)) + assert.NoError(t, err, "CreateReaderAndDetermineDelimiter() shouldn't throw error") + assert.NotNil(t, rd, "CSV reader should not be mnil") + rows, err := rd.ReadAll() + assert.Error(t, err, "RaadAll() should throw error") + assert.Empty(t, rows, "rows should be empty") +} + func TestDetermineDelimiter(t *testing.T) { var cases = []struct { csv string From 683def1ce9980691c72a82b6e969124f01eb4e72 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Fri, 29 Oct 2021 13:38:45 -0400 Subject: [PATCH 18/22] Error tests --- modules/csv/csv_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/csv/csv_test.go b/modules/csv/csv_test.go index f99682ef9877..d72c3e3a7329 100644 --- a/modules/csv/csv_test.go +++ b/modules/csv/csv_test.go @@ -122,6 +122,7 @@ func TestDetermineDelimiterReadAllError(t *testing.T) { assert.NotNil(t, rd, "CSV reader should not be mnil") rows, err := rd.ReadAll() assert.Error(t, err, "RaadAll() should throw error") + assert.ErrorIs(t, err, csv.ErrFieldCount) assert.Empty(t, rows, "rows should be empty") } From 95ede28fb90abfe121b4a3fae8e1192973c38801 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Sat, 30 Oct 2021 01:03:49 -0600 Subject: [PATCH 19/22] Update modules/csv/csv.go Co-authored-by: delvh --- modules/csv/csv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index e0405e7073bf..5fe3849a0130 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -80,7 +80,7 @@ func removeQuotedString(text string) string { } // guessDelimiter takes up to maxLines of the CSV text, iterates through the possible delimiters, and sees if the CSV Reader reads it without throwing any errors. -// If more than one delmiiter passes, the delimiter that results in the most columns is returned. +// If more than one delimiter passes, the delimiter that results in the most columns is returned. func guessDelimiter(data []byte) rune { delimiter := guessFromBeforeAfterQuotes(data) if delimiter != 0 { From b938db461acc27f1f1526259c5004e3a209e9c9b Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Sat, 30 Oct 2021 01:13:37 -0600 Subject: [PATCH 20/22] Update modules/csv/csv.go Co-authored-by: delvh --- modules/csv/csv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 5fe3849a0130..fec1a1b60ca5 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -90,7 +90,7 @@ func guessDelimiter(data []byte) rune { // Removes quoted values so we don't have columns with new lines in them text := removeQuotedString(string(data)) - // Make the text just be maxLines or less without truncated lines + // Make the text just be maxLines or less, ignoring truncated lines lines := strings.SplitN(text, "\n", maxLines+1) // Will contain at least one line, and if there are more than MaxLines, the last item holds the rest of the lines if len(lines) > maxLines { // If the length of lines is > maxLines we know we have the max number of lines, trim it to maxLines From 772095d32759fc8080d1563d92a00ea661c91b74 Mon Sep 17 00:00:00 2001 From: Richard Mahn Date: Sat, 30 Oct 2021 03:23:11 -0400 Subject: [PATCH 21/22] Adds comments --- modules/csv/csv.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index fec1a1b60ca5..5ee5cd49e5d0 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -136,13 +136,13 @@ var beforeAfterQuotes = regexp.MustCompile(`([,@\t;|]{0,1}) *(?:"[^"]*")+([,@\t; // guessFromBeforeAfterQuotes guesses the limiter by finding a double quote that has a valid delimiter before it and a closing quote, // or a double quote with a closing quote and a valid delimiter after it func guessFromBeforeAfterQuotes(data []byte) rune { - rs := beforeAfterQuotes.FindStringSubmatch(string(data)) + rs := beforeAfterQuotes.FindStringSubmatch(string(data)) // returns first match, or nil if none if rs != nil { if rs[1] != "" { - return rune(rs[1][0]) + return rune(rs[1][0]) // delimiter found left of quoted string } else if rs[2] != "" { - return rune(rs[2][0]) + return rune(rs[2][0]) // delimiter found right of quoted string } } - return 0 + return 0 // no match found } From 059ae3d6f0f68c728140cf8473cec007393f596b Mon Sep 17 00:00:00 2001 From: zeripath Date: Sat, 30 Oct 2021 13:06:44 +0100 Subject: [PATCH 22/22] Update modules/csv/csv.go Co-authored-by: delvh --- modules/csv/csv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/csv/csv.go b/modules/csv/csv.go index 5ee5cd49e5d0..47ea62699df7 100644 --- a/modules/csv/csv.go +++ b/modules/csv/csv.go @@ -73,7 +73,7 @@ func determineDelimiter(ctx *markup.RenderContext, data []byte) rune { // This finds all quoted strings that have escaped quotes. var quoteRegexp = regexp.MustCompile(`"[^"]*"`) -// removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can realiably have each row on one line +// removeQuotedStrings uses the quoteRegexp to remove all quoted strings so that we can reliably have each row on one line // (quoted strings often have new lines within the string) func removeQuotedString(text string) string { return quoteRegexp.ReplaceAllLiteralString(text, "")