Clients overhaul

.github/workflows/ci.yaml

@@ -26,6 +26,9 @@ jobs:
           version: '23.4'
           repo-token: ${{ secrets.KWIL_MACH_SECRET }}
 
+#      - name: Install swagger-codegen
+#        uses: swagger-api/swagger-codegen/.github/actions/generate/action.yml@main
+
       - name: Install Taskfile
         uses: arduino/setup-task@v1
         with:
@@ -53,6 +56,10 @@ jobs:
         run: |
           ./scripts/proto/check_up
 
+      - name: Ensure generated swagger Go code is up-to-date
+        run: |
+          ./scripts/swagger/check_up
+
       - name: Compile packages, apps, and specs
         run: | # enter workspace mode to do this on one line
           go work init && go work use . ./parse ./test ./core

.gitignore

@@ -39,6 +39,9 @@ deployments/helm/*/charts/
 
 dist/*
 
+# tools
+swagger-codegen-cli.jar
+
 # kwild
 .testnet
 

.golangci.yml

@@ -1,5 +1,8 @@
 run:
   deadline: 10m
+  skip-dirs:
+    - core/rpc/protobuf
+    - core/rpc/http
 
 output:
   format: github-actions,colored-line-number

Taskfile-pb.yml

@@ -3,32 +3,14 @@ version: '3'
 tasks:
   compile:
     cmds:
-      - task: compile:v1
+      - task: tx:v1
+      - task: tx:v1:swagger
       - task: admin:v0
+      - task: function:v0
+      - task: function:v0:swagger
 
-  # compile:v0:
-  #   desc: Compiles v0 protobuf
-  #   internal: true
-  #   deps: [update]
-  #   preconditions:
-  #     - sh: a="libprotoc 23.4";b=`protoc --version`;test "$a" = "$b"
-  #       msg: "Protobuf compiler version is not 23.4, please install the correct version"
-  #   cmds:
-  #     - |
-  #       protoc -I ./proto \
-  #       --go_out=. --go_opt module=kwil \
-  #       --go-grpc_out=. --go-grpc_opt module=kwil \
-  #       --grpc-gateway_out=. --grpc-gateway_opt module=kwil --grpc-gateway_opt generate_unbound_methods=true \
-  #       --openapiv2_out=. --openapiv2_opt allow_merge=true --openapiv2_opt merge_file_name=api/openapi-spec/api/v0/api \
-  #       proto/kwil/*/v0/*.proto
-  #   sources:
-  #     - proto/kwil/*/v0/*.proto
-  #   generates:
-  #     - api/protobuf/*/v0/*.go
-  #     - api/openapi-spec/api/v0/api.swagger.json
-
-  compile:v1:
-    desc: Compiles v1 protobuf
+  tx:v1:
+    desc: Compiles TxService v1 protobuf
     preconditions:
       - sh: a="libprotoc 23.4";b=`protoc --version`;test "$a" = "$b"
         msg: "Protobuf compiler version is not 23.4, please install the correct version"
@@ -39,14 +21,25 @@ tasks:
         --go-grpc_out=. --go-grpc_opt module=github.com/kwilteam/kwil-db \
         --grpc-gateway_out=. --grpc-gateway_opt module=github.com/kwilteam/kwil-db \
         --grpc-gateway_opt generate_unbound_methods=true \
-        --openapiv2_out=internal/services/http/api --openapiv2_opt allow_merge=true \
-        --openapiv2_opt merge_file_name=v1 \
-        proto/kwil/*/v1/*.proto
+        --openapiv2_out=internal/services/http/api/tx \
+        --openapiv2_opt allow_merge=true --openapiv2_opt merge_file_name=v1 \
+        proto/kwil/tx/v1/*.proto
+    sources:
+      - proto/kwil/tx/v1/*.proto
+    generates:
+      - core/rpc/protobuf/tx/v1/*.go
+      - internal/services/http/api/tx/v1.swagger.json
+
+  tx:v1:swagger: # more general for all platforms
+    desc: Generates swagger codegen for TxService v1
+    cmds:
+      # if swagger-codegen-cli.jar does not exist, download it
+      - test -f swagger-codegen-cli.jar || wget https://repo1.maven.org/maven2/io/swagger/codegen/v3/swagger-codegen-cli/3.0.51/swagger-codegen-cli-3.0.51.jar -O swagger-codegen-cli.jar
+      - java -jar swagger-codegen-cli.jar generate -i internal/services/http/api/tx/v1.swagger.json -l go -o core/rpc/http/tx
     sources:
-      - proto/kwil/*/v1/*.proto
+      - internal/services/http/api/tx/v1.swagger.json
     generates:
-      - core/rpc/protobuf/*/v1/*.go
-      - internal/services/http/api/v1.swagger.json
+      - core/rpc/http/tx/*
 
   admin:v0:
     desc: Compiles admin v0 protobuf
@@ -63,3 +56,39 @@ tasks:
       - proto/kwil/admin/v0/*.proto
     generates:
       - core/rpc/protobuf/admin/v0/*.go
+
+  function:v0:
+    desc: Compiles functions v0 protobuf
+    preconditions:
+      - sh: a="libprotoc 23.4";b=`protoc --version`;test "$a" = "$b"
+        msg: "Protobuf compiler version is not 23.4, please install the correct version"
+    cmds:
+      - |
+        protoc -I ./proto \
+        --go_out=. --go_opt module=github.com/kwilteam/kwil-db \
+        --go-grpc_out=. --go-grpc_opt module=github.com/kwilteam/kwil-db \
+        --grpc-gateway_out=. --grpc-gateway_opt module=github.com/kwilteam/kwil-db \
+        --grpc-gateway_opt generate_unbound_methods=true \
+        --openapiv2_out=internal/services/http/api/function \
+        --openapiv2_opt allow_merge=true --openapiv2_opt merge_file_name=v0 \
+        proto/kwil/function/v0/*.proto
+    sources:
+      - proto/kwil/function/v0/*.proto
+    generates:
+      - core/rpc/protobuf/function/v0/*.go
+      - internal/services/http/api/function/v0.swagger.json
+#      - internal/services/http/api/function/v0.swagger.json
+# put in cmds
+#        --openapiv2_out=internal/services/http/api/function \
+#        --openapiv2_opt allow_merge=true --openapiv2_opt merge_file_name=v1 \
+
+  function:v0:swagger: # more general for all platforms
+    desc: Generates swagger codegen for TxService v1
+    cmds:
+      # if swagger-codegen-cli.jar does not exist, download it
+      - test -f swagger-codegen-cli.jar || wget https://repo1.maven.org/maven2/io/swagger/codegen/v3/swagger-codegen-cli/3.0.51/swagger-codegen-cli-3.0.51.jar -O swagger-codegen-cli.jar
+      - java -jar swagger-codegen-cli.jar generate -i internal/services/http/api/function/v0.swagger.json -l go -o core/rpc/http/function
+    sources:
+      - internal/services/http/api/function/v0.swagger.json
+    generates:
+      - core/rpc/http/function/*

Taskfile.yml

@@ -63,7 +63,8 @@ tasks:
   lint:
     desc: Lint with golangci-lint
     cmds:
-        - golangci-lint run ./... ./core/... ./test/... ./parse/... -c .golangci.yml --skip-dirs ./core/rpc/protobuf
+      # skip-dirs only works in .yaml
+      - golangci-lint run ./... ./core/... ./test/... ./parse/... -c .golangci.yml
 
   linter:
     desc: Install the linter (not for CI which has an action for this)
@@ -98,6 +99,12 @@ tasks:
     generates:
       - .build/kwild
 
+  generate:docs:
+    desc: Generate docs for CLIs
+    cmds:
+      - go run ./cmd/kwil-cli/generate -out ./gen
+      - go run ./cmd/kwil-admin/generate -out ./gen
+
   # ************ docker ************
   vendor:
     desc: Generate vendor
@@ -138,7 +145,6 @@ tasks:
     desc: Start the dev environment
     deps:
       - task: build:docker
-        vars: {VARIANT: 'shell'}
     cmds:
       - task: dev:up:nb
 
@@ -163,7 +169,6 @@ tasks:
     desc: Start the dev environment(with testnet)
     deps:
       - task: build:docker
-        vars: { VARIANT: 'shell' }
     cmds:
       - task: dev:testnet:up:nb
 

build/package/docker/kwild.debug.dockerfile

@@ -9,6 +9,8 @@ ARG git_commit
 ARG go_build_tags
 
 WORKDIR /app
+RUN mkdir -p /var/run/kwil
+RUN chmod 777 /var/run/kwil
 RUN apk update && apk add git ca-certificates-bundle
 
 COPY . .

build/package/docker/kwild.dockerfile

@@ -6,17 +6,21 @@ ARG git_commit
 ARG go_build_tags
 
 WORKDIR /app
+RUN mkdir -p /var/run/kwil
+RUN chmod 777 /var/run/kwil
 RUN apk update && apk add git ca-certificates-bundle
 
 COPY . .
 RUN test -f go.work && rm go.work || true
 
 RUN GOWORK=off GIT_VERSION=$version GIT_COMMIT=$git_commit BUILD_TIME=$build_time CGO_ENABLED=0 TARGET="/app/dist" GO_BUILDTAGS=$go_build_tags ./scripts/build/binary kwild
-RUN chmod +x /app/dist/kwild-*
+RUN GIT_VERSION=$version GIT_COMMIT=$git_commit BUILD_TIME=$build_time CGO_ENABLED=0 TARGET="/app/dist" ./scripts/build/binary kwil-admin
+RUN chmod +x /app/dist/kwild-* /app/dist/kwil-admin-*
 
-FROM scratch AS assemble
+FROM alpine:3.17
 WORKDIR /app
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 COPY --from=build /app/dist/kwild-* ./kwild
+COPY --from=build /app/dist/kwil-admin-* ./kwil-admin
 EXPOSE 50051 50151 8080 26656 26657
 ENTRYPOINT ["/app/kwild"]