Work out a new way to write integration tests

.travis.yml

@@ -4,7 +4,6 @@ env:
   - GO111MODULE=on
 
 go:
-  - "1.15.x"
   - "1.16.x"
   - tip
 

cmd/examplar/.gitignore

@@ -0,0 +1,3 @@
+examplar
+trace_test.go
+integration

cmd/examplar/README.md

@@ -0,0 +1,130 @@
+# Examplar
+
+A tool to generate tests from on an annotated SQL file that specifies an example and its expected result.
+
+A file named `selecting.sql`
+
+```sql
+--- setup:
+CREATE TABLE foo;
+--- teardown:
+DROP TABLE foo;
+--- test: insert something
+INSERT INTO foo (a) VALUES (1);
+SELECT * FROM foo;
+--- `[{"a": 1}]`
+```
+
+Gets transformed into (abbreviated):
+
+```go
+func TestSelecting(t *testing.T) {
+    db, err := genji.Open(":memory:")
+
+    teardown := func() {
+       db.Exec("DROP TABLE foo;")
+    }
+
+    setup := func() {
+        err = db.Exec("CREATE TABLE foo (a int);")
+    }
+
+    t.Run("insert something", func(t *testing.T) {
+        t.Cleanup(teardown)
+        setup()
+
+        res, err := db.Query(
+          "INSERT INTO foo (a) VALUES (1);" +
+          "SELECT * FROM foo;"
+        )
+        require.NoError(t, err)
+        defer res.Close()
+
+        data := jsonResult(t, res)
+
+        expected = `[{"a": 1}]`
+        require.JSONEq(t, expected, string(data))
+    }
+}
+```
+
+## Usage
+
+To populate tests that would be in an `integration` package, create a go file in folder named `integration` with the following code:
+
+```go
+package integration
+
+//go:generate examplar -package=integration fixtures/sql/*.sql
+```
+
+Assuming `fixtures/sql/` contains the following files:
+
+```
+extest1.sql
+extest2.sql
+```
+
+Running `go generate` will generate these tests files in the `integration` folder:
+
+```
+extest1_test.go
+extest2_test.go
+```
+
+Those tests files can be run like any normal tests.
+
+## How it works
+
+Examplar reads a SQL file, looking either for raw text or annotations that specifies what those lines are supposed to do from a testing perpective.
+
+Once a SQL file has been parsed, it generates a `(...)_test.go` file that looks similar to the handwritten test that would have been written.
+
+Every original line in the input SQL file is executed and expected to not return any error.
+
+Those test files have no dependencies on Examplar or the SQL file that has been used to generate the test.
+
+### Annotations
+
+Annotations starts with `---` (the SQL comment `--` and an additional `-`) and can
+be followed by a keyword specified in the list below followed by a `:` or special symbols to pass data to set expectations.
+
+- `setup:`
+
+  - all lines up to the next annotation are to be considered as one single statement for the setup block.
+
+- `teardown:`
+  - all lines up to the next annotation are to be considered as single statement for the teardown block.
+
+:bulb: `setup` and `teardown` blocks will generate code being ran around **each indidual** `test` block.
+They are optional and can be declared in no particular order, but there can only be one of each.
+
+- `test: [TEST NAME]`
+  - a test is composed of one or many statements; a statement composed of one or multiple lines and is terminated by an expectation (see below).
+
+:bulb: Each `test` block will generate an individual `t.Run("[TEST NAME]", ...)` function. At least one test block must be present.
+
+- `` `[JSON]` ``
+
+  - the statement above this annotation will be compared to `[JSON]` when evaluated at the runtime.
+  - Invalid JSON won't yield an error ar generate time, but the generated test will always fail at runtime.
+
+- ` ``` `
+  - the statement above this annotation will be compared to `[JSON]` when evaluated at the runtime.
+  - all the following lines until another triple backtick annoattion is found are to be considered as part of a single multiline JSON data.
+  - indentation will be preserved in the generating test file for readablilty. Similarly, invalid JSON will only yield an error when the resulting test is evaluated.
+
+## Goals and non-goals
+
+Examplar objective is to provide a clear and simple way to write example of Genji SQL code and its expected results. It has to be easy enough for anyone to edit or write an example SQL file without having to read the present documentation. If this objective succeeds, it opens the path Examplar SQL files being used by users and contributors to showcase a bug or a feature request in a Github issue.
+
+By being totally independent from Genji itself for the parsing part, it frees itself from needing to be updated when something changes under the hood and allows to keep the code as merely parsing textual data. In other words, if anything changes in Genji on how to execute queries, only the template needs to be updated.
+
+## Limitations
+
+A breaking change in the API has only
+For now, let's observe how useful Examplar can and what we can make out of it.
+Then we can then see if it's worth addressing the following limitations:
+
+- Expecting an error instead of JSON is not supported.
+- Error messages on failed expectations do not reference the orignal file directly, which could be useful on complex examples sql files.

cmd/examplar/examplar.go

@@ -0,0 +1,114 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"io"
+	"strings"
+	"text/template"
+)
+
+type Line struct {
+	Orig string
+	Text string
+}
+
+// Test is a list of statements.
+type Test struct {
+	Name       string
+	Orig       string
+	Statements []*Statement
+}
+
+// Statement is a pair composed of a line of code and an expectation on its result when evaluated.
+type Statement struct {
+	Code        []Line
+	Expectation []Line
+}
+
+func (s Statement) expectationText() string {
+	var text string
+	for _, e := range s.Expectation {
+		text += e.Text + "\n"
+	}
+
+	return text
+}
+
+// Examplar represents a group of tests and can optionally include setup code and teardown code.
+type Examplar struct {
+	Name             string
+	originalFilename string
+	setup            []Line
+	teardown         []Line
+	examples         []*Test
+}
+
+func (ex *Examplar) origLoc(num int) string {
+	return fmt.Sprintf("%s:%d", ex.originalFilename, num)
+}
+
+// HasSetup returns true if setup code is provided.
+func (ex *Examplar) HasSetup() bool {
+	return len(ex.setup) > 0
+}
+
+// HasSetup returns true if teardown code is provided.
+func (ex *Examplar) HasTeardown() bool {
+	return len(ex.teardown) > 0
+}
+
+func (ex *Examplar) appendTest(name string, num int) *Test {
+	test := Test{
+		Name: name,
+		Orig: ex.origLoc(num),
+	}
+	ex.examples = append(ex.examples, &test)
+
+	return &test
+}
+
+// Parse reads annotated textual data and transforms it into a
+// structured representation. Only annotations are parsed, the
+// textual data itself is irrelevant to this function.
+//
+// It will panic if an error is encountered.
+func Parse(r io.Reader, name string, originalFilename string) *Examplar {
+	ex := Examplar{
+		Name:             name,
+		originalFilename: originalFilename,
+	}
+
+	scanner := &Scanner{ex: &ex}
+	scanner.Run(bufio.NewScanner(r))
+
+	return &ex
+}
+
+func normalizeTestName(name string) string {
+	name = strings.TrimSpace(name)
+	name = strings.Title(name)
+	return strings.ReplaceAll(name, " ", "")
+}
+
+// Generate takes a structured representation of the original textual data in order
+// to write a valid go test file.
+func Generate(ex *Examplar, packageName string, w io.Writer) error {
+	tmpl := template.Must(template.ParseFS(tmplFS, "test_template.go.tmpl"))
+
+	bindings := struct {
+		Package  string
+		TestName string
+		Setup    []Line
+		Teardown []Line
+		Tests    []*Test
+	}{
+		packageName,
+		normalizeTestName(ex.Name),
+		ex.setup,
+		ex.teardown,
+		ex.examples,
+	}
+
+	return tmpl.Execute(w, bindings)
+}

cmd/examplar/examplar_test.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestParse(t *testing.T) {
+	f, err := os.Open("extest1.sql")
+	require.NoError(t, err)
+
+	defer f.Close()
+	ex := Parse(f, "extest1", "extest1.sql")
+
+	require.Equal(t, []Line{{"extest1.sql:2", "CREATE TABLE foo (a int);"}}, ex.setup)
+	require.Equal(t, []Line{{"extest1.sql:5", "DROP TABLE foo;"}}, ex.teardown)
+
+	// first test
+	example := ex.examples[0]
+	require.NotNil(t, example)
+	require.Equal(t, "insert something", example.Name)
+	require.Equal(t, 3, len(example.Statements))
+
+	stmt := example.Statements[0]
+	require.Equal(t, "INSERT INTO foo (a) VALUES (1);", stmt.Code[0].Text)
+	require.Equal(t, "SELECT * FROM foo;", stmt.Code[1].Text)
+	require.Equal(t, `[{"a": 1}]`, stmt.Expectation[0].Text)
+
+	stmt = example.Statements[1]
+	require.Equal(t, "SELECT a, b FROM foo;", stmt.Code[0].Text)
+	require.JSONEq(t, `[{"a": 1, "b": null}]`, stmt.expectationText())
+
+	stmt = example.Statements[2]
+	require.Equal(t, "SELECT z FROM foo;", stmt.Code[0].Text)
+	require.Equal(t, `[{"z": null}]`, stmt.Expectation[0].Text)
+
+	// second test
+	example = ex.examples[1]
+	require.NotNil(t, example)
+	require.Equal(t, "something else", example.Name)
+
+	require.Equal(t, 1, len(example.Statements))
+	stmt = example.Statements[0]
+	require.Equal(t, "INSERT INTO foo (c) VALUES (3);", stmt.Code[0].Text)
+	require.Equal(t, "SELECT * FROM foo;", stmt.Code[1].Text)
+	require.Equal(t, `[{"c": 3}]`, stmt.Expectation[0].Text)
+}
+
+func TestTemplate(t *testing.T) {
+	g, err := os.Open("extest1_test.go.gold")
+	require.NoError(t, err)
+	defer g.Close()
+
+	gb, err := ioutil.ReadAll(g)
+	require.NoError(t, err)
+
+	gold := string(gb)
+
+	f, err := os.Open("extest1.sql")
+	require.NoError(t, err)
+	defer f.Close()
+
+	ex := Parse(f, "foo bar", "extest1.sql")
+
+	var b strings.Builder
+
+	err = Generate(ex, "main", &b)
+	require.NoError(t, err)
+
+	// some code to generate the gold version if needed
+	// err = ioutil.WriteFile("trace_test.go", []byte(b.String()), 0644)
+	// require.NoError(t, err)
+
+	require.Equal(t, strings.Split(gold, "\n"), strings.Split(b.String(), "\n"))
+}

cmd/examplar/extest1.sql

@@ -0,0 +1,26 @@
+--- setup:
+CREATE TABLE foo (a int);
+
+--- teardown:
+DROP TABLE foo;
+
+--- test: insert something
+INSERT INTO foo (a) VALUES (1);
+SELECT * FROM foo;
+--- `[{"a": 1}]`
+
+SELECT a, b FROM foo;
+--- ```json
+--- [{
+---   "a": 1,
+---   "b": null
+--- }]
+--- ```
+
+SELECT z FROM foo;
+--- `[{"z": null}]`
+
+--- test: something else
+INSERT INTO foo (c) VALUES (3);
+SELECT * FROM foo;
+--- `[{"c": 3}]`