Merge pull request #5 from devops-works/develop

Merge before public release

.github/workflows/release_binaries.yaml

@@ -0,0 +1,29 @@
+name: Release Go project
+
+on:
+  push:
+    tags:
+      - 'v*'
+  pull_request:
+
+jobs:
+  build:
+    name: GoReleaser build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+      - name: Set up Go 1.16
+        uses: actions/setup-go@v2
+        with:
+          go-version: 1.16
+        id: go
+      - name: Run GoReleaser
+        uses: goreleaser/goreleaser-action@v2
+        with:
+          version: latest
+          args: release --rm-dist
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,4 +1,6 @@
 samples/*
 *.log
 *.notes
-main
+main
+/.vscode/
+bin/

.goreleaser.yaml

@@ -0,0 +1,46 @@
+before:
+  hooks:
+    - go mod tidy
+    - go mod download
+
+builds:
+  - main: ./cmd/slowql-digest/
+    id: "digest"
+    binary: digest
+    goos:
+      - linux
+      - darwin
+    ignore:
+      - goos: darwin
+        goarch: 386
+    env:
+      - CGO_ENABLED=0
+
+  - main: ./cmd/slowql-replayer/
+    id: "replayer"
+    binary: replayer
+    goos:
+      - linux
+      - darwin
+    ignore:
+      - goos: darwin
+        goarch: 386
+    env:
+      - CGO_ENABLED=0
+
+archives:
+  - format: binary
+    name_template: "{{ .Binary }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}{{ if .Mips }}_{{ .Mips }}{{ end }}"
+
+checksum:
+  name_template: "checksums.txt"
+
+snapshot:
+  name_template: "{{ .Tag }}-next"
+
+changelog:
+  sort: asc
+  filters:
+    exclude:
+      - "^docs:"
+      - "^test:"

Dockerfile.digest

@@ -0,0 +1,17 @@
+# Build stage
+FROM devopsworks/golang-upx:1.16 AS builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 \
+    GOOS=linux \
+    GOARCH=amd64
+COPY . /app
+WORKDIR /app
+RUN make digest && \
+    strip ./bin/digest && \
+    /usr/local/bin/upx -9 ./bin/digest
+
+# Run stage
+FROM gcr.io/distroless/base-debian10
+COPY --from=builder /app/bin/ /app
+WORKDIR /app
+ENTRYPOINT ["./digest"]

Dockerfile.replayer

@@ -0,0 +1,17 @@
+# Build stage
+FROM devopsworks/golang-upx:1.15 AS builder
+ENV GO111MODULE=on \
+    CGO_ENABLED=0 \
+    GOOS=linux \
+    GOARCH=amd64
+COPY . /app
+WORKDIR /app
+RUN make replayer && \
+    strip ./bin/replayer && \
+    /usr/local/bin/upx -9 ./bin/replayer
+
+# Run stage
+FROM gcr.io/distroless/base-debian10
+COPY --from=builder /app/bin /app
+WORKDIR /app
+ENTRYPOINT ["./replayer"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 devops.works
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Makefile

@@ -0,0 +1,51 @@
+GOCMD=go
+GOTEST=$(GOCMD) test
+GOVET=$(GOCMD) vet
+BINARY_NAME=slowql-digest
+BUILD_ROOT=./bin/
+
+GREEN  := $(shell tput -Txterm setaf 2)
+YELLOW := $(shell tput -Txterm setaf 3)
+WHITE  := $(shell tput -Txterm setaf 7)
+RESET  := $(shell tput -Txterm sgr0)
+
+.PHONY: all build clean
+
+all: help
+
+## Build:
+digest: ## Build slowql-digest
+	$(GOCMD) mod tidy
+	GO111MODULE=on $(GOCMD) build -o $(BUILD_ROOT)digest ./cmd/slowql-digest/
+	@echo "${GREEN}[*]${RESET} digest successfully built in ${YELLOW}${BUILD_ROOT}digest${RESET}"
+
+replayer: ## Build slowql-replayer
+	$(GOCMD) mod tidy
+	GO111MODULE=on $(GOCMD) build -o $(BUILD_ROOT)replayer ./cmd/slowql-replayer/
+	@echo "${GREEN}[*]${RESET} replayer successfully built in ${YELLOW}${BUILD_ROOT}replayer${RESET}"
+
+clean: ## Clean all the files and binaries generated by the Makefile
+	$(GOCMD) mod tidy
+	rm -rf $(BUILD_ROOT)
+	rm -f ./profile.cov
+
+## Test:
+test: ## Run the tests of the project
+	$(GOTEST) -v -race ./...
+
+coverage: ## Run the tests of the project and export the coverage
+	$(GOTEST) -cover -covermode=count -coverprofile=profile.cov ./...
+	$(GOCMD) tool cover -func profile.cov
+
+## Help:
+help: ## Show this help
+	@echo ''
+	@echo 'Usage:'
+	@echo '  ${YELLOW}make${RESET} ${GREEN}<target>${RESET}'
+	@echo ''
+	@echo 'Targets:'
+	@awk 'BEGIN {FS = ":.*?## "} { \
+		if (/^[a-zA-Z_-]+:.*?##.*$$/) {printf "    ${YELLOW}%-20s${GREEN}%s${RESET}\n", $$1, $$2} \
+		else if (/^## .*$$/) {printf "  ${CYAN}%s${RESET}\n", substr($$1,4)} \
+		}' $(MAKEFILE_LIST)
+

README.md

@@ -0,0 +1,95 @@
+# slowql
+
+A slow query logs parser in Golang.
+
+- [slowql](#slowql)
+  - [Gettin' started](#gettin-started)
+  - [Basic usage](#basic-usage)
+  - [Performances](#performances)
+  - [Associated tools](#associated-tools)
+  - [Notes](#notes)
+    - [Tested databases](#tested-databases)
+  - [Contributing](#contributing)
+  - [License](#license)
+
+## Gettin' started
+
+```
+go get github.com/devops-works/slowql
+```
+
+## Basic usage
+
+To put in a nutshell, you can use it as follows:
+
+```go
+package main
+
+import (
+    "fmt"
+
+    "github.com/devops-works/slowql"
+)
+
+func main() {
+    // Imagine that fd is an io.Reader of your slow query logs file...
+
+    // Create the parser
+    p := slowql.NewParser(fd)
+
+    // Get the next query from the log
+    q, err := p.GetNext()
+	  if err != nil {
+		    panic(err)
+	  }
+
+    // Do your stuff, for example:
+    fmt.Printf("at %d, %s did the request: %s\n", q.Time, q.User, q.Query)
+
+    fp, err := q.Fingerprint()
+    if err != nil {
+        panic(err)
+    }
+    fmt.Printf("the query fingerprint is: %s\n", fp)
+
+    srv := p.GetServerMeta()
+    fmt.Printf("the SQL server listens to port %d", srv.Port)
+}
+```
+
+## Performances
+
+Running the example given in cmd/ without any `fmt.Printf` against a 292MB slow query logs from a MySQL database provides the following output:
+
+```
+parsed 278077 queries in 8.099622786s
+```
+
+which is approx. **34760 queries/second** (Intel i5-8250U (8) @ 3.400GHz).
+
+## Associated tools
+
+With this package we created some tools:
+
+* [slowql-replayer](https://github.com/devops-works/slowql/tree/develop/cmd/slowql-replayer): replay and benchmark queries from a slow query log
+* [slowql-digest](https://github.com/devops-works/slowql/tree/develop/cmd/slowql-digest): digest and analyze slow query logs. Similar to `pt-query-digest`, but faster. :upside_down_face:
+
+## Notes
+
+### Tested databases
+
+Not all kind of slow query logs have been tested yet:
+
+- [X] MySQL
+- [X] MariaDB
+- [ ] Percona-db
+- [X] Percona-cluster (pxc)
+- [ ] PostgreSQL
+
+## Contributing
+
+Issues and pull requests are welcomed ! If you found a bug or want to help and improve this package don't hesitate to fork it or open an issue :smile:
+
+## License
+
+MIT

cmd/example/main.go

@@ -0,0 +1,72 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/devops-works/slowql"
+	"github.com/devops-works/slowql/query"
+	"github.com/devops-works/slowql/server"
+)
+
+func main() {
+	if len(os.Args) != 2 {
+		fmt.Printf("Usage: %s file.log\n", os.Args[0])
+		os.Exit(1)
+	}
+
+	fd, err := os.Open(os.Args[1])
+	if err != nil {
+		panic(err)
+	}
+
+	p := slowql.NewParser(slowql.MySQL, fd)
+	srv := p.GetServerMeta()
+	showServer(srv)
+
+	var count int
+	start := time.Now()
+	for {
+		q := p.GetNext()
+		if q == (query.Query{}) {
+			break
+		}
+
+		// showQueries(q)
+
+		count++
+	}
+	elapsed := time.Since(start)
+	fmt.Printf("\nparsed %d queries in %s\n", count, elapsed)
+}
+
+func showQueries(q query.Query) {
+	fmt.Printf("Time: %s\nUser: %s\nHost: %s\nID: %d\nSchema: %s\nLast_errno: %d\nKilled: %d\nQuery_time: %f\nLock_time: %f\nRows_sent: %d\nRows_examined: %d\nRows_affected: %d\nBytes_sent: %d\nQuery: %s\n",
+		q.Time,
+		q.User,
+		q.Host,
+		q.ID,
+		q.Schema,
+		q.LastErrNo,
+		q.Killed,
+		q.QueryTime,
+		q.LockTime,
+		q.RowsSent,
+		q.RowsExamined,
+		q.RowsAffected,
+		q.BytesSent,
+		q.Query,
+	)
+}
+
+func showServer(srv server.Server) {
+	fmt.Printf("Binary: %s\nVersion short: %s\nVersion: %s\nVersion description: %s\nSocket: %s\nPort: %d\n",
+		srv.Binary,
+		srv.VersionShort,
+		srv.Version,
+		srv.VersionDescription,
+		srv.Socket,
+		srv.Port,
+	)
+}