feat: replace current logging solution with log server (#1381)

* feat: add log server

* feat(runtimes&server): adapt to log server

* chore(build): add helm template for log-server

* feat(log-server): health check

* refactor(server): fetch function logs from log server

* chore(docs): tidy docs for function log

* chore: clean code

* chore(ci): dockerize log server

* fix(server): replace db in FunctionConsole.write

* feat: verify appid for log server

* chore(runtimes): restart app when file changes

* chore: adjust env and region conf for supporting log server

* refactor(server): get logs from log server

.github/workflows/dockerize-log-server.yml

@@ -0,0 +1,76 @@
+name: dockerize-log-server
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - "services/log-server/**"
+      - ".github/workflows/dockerize-log-server.yml"
+      - "!**/*.md"
+      - "!services/log-server/package-lock.json"
+
+concurrency:
+  group: dockerize-log-server-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dockerize-log-server:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: |
+            ghcr.io/${{ github.repository_owner }}/log-server
+            docker.io/${{ secrets.DOCKER_USERNAME }}/log-server
+          # https://github.com/docker/metadata-action#typesemver
+          tags: |
+            type=raw,value=latest,enable=true
+            type=sha,enable=true,format=short
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to Github Container Hub
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push
+        uses: docker/build-push-action@v3
+        with:
+          context: ./services/log-server
+          file: ./services/log-server/Dockerfile
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          platforms: linux/amd64, linux/arm64
+
+  trigger-workflow-build-cluster-image:
+    needs: [dockerize-log-server]
+    runs-on: ubuntu-latest
+    steps:
+      - name: trigger cluster image workflow
+        uses: peter-evans/repository-dispatch@v2
+        with:
+          event-type: docker_build_success
+          client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "version": "latest"}'

.prettierignore

@@ -0,0 +1 @@
+charts

build/charts/laf-server/templates/_helpers.tpl

@@ -42,6 +42,18 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
 app.kubernetes.io/managed-by: {{ .Release.Service }}
 {{- end }}
 
+{{/*
+Common labels
+*/}}
+{{- define "log-server.labels" -}}
+helm.sh/chart: {{ include "laf-server.chart" . }}
+{{ include "log-server.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
 {{/*
 Selector labels
 */}}
@@ -50,6 +62,14 @@ app.kubernetes.io/name: {{ include "laf-server.name" . }}
 app.kubernetes.io/instance: {{ .Release.Name }}
 {{- end }}
 
+{{/*
+Selector labels
+*/}}
+{{- define "log-server.selectorLabels" -}}
+app.kubernetes.io/name: log-server
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
 {{/*
 Create the name of the service account to use
 */}}

build/charts/laf-server/templates/deployment.yaml

@@ -83,6 +83,12 @@ spec:
               value: {{ .Values.default_region.apisix_api_url }}
             - name: DEFAULT_REGION_APISIX_API_KEY
               value: {{ .Values.default_region.apisix_api_key }}
+            - name: DEFAULT_REGION_LOG_SERVER_URL
+              value: {{ .Values.default_region.log_server_url }}
+            - name: DEFAULT_REGION_LOG_SERVER_SECRET
+              value: {{ .Values.default_region.log_server_secret }}
+            - name: DEFAULT_REGION_LOG_SERVER_DATABASE_URL
+              value: {{ .Values.default_region.log_server_database_url }}
             - name: SITE_NAME
               value: {{ .Values.siteName | quote}}
       {{- with .Values.nodeSelector }}

build/charts/laf-server/templates/log-server.yaml

@@ -0,0 +1,52 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    {{- include "log-server.labels" . | nindent 4 }}
+  name: log-server
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      {{- include "log-server.selectorLabels" . | nindent 6 }}
+  template:
+    metadata: 
+      labels:
+        {{- include "log-server.selectorLabels" . | nindent 8 }}
+    spec:
+      containers:
+        - image: docker.io/lafyun/log-server:latest
+          imagePullPolicy: Always
+          name: log-server
+          ports:
+            - name: http
+              containerPort: 5060
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /healthz
+              port: http 
+          env:
+            - name: DB_URI
+              value: {{ .Values.default_region.log_server_database_url | quote }}
+            - name: JWT_SECRET
+              value: {{ .Values.default_region.log_server_secret | quote }}
+--- 
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    {{- include "log-server.labels" . | nindent 4 }}
+  name: log-server
+spec:
+  ports:
+    - name: http
+      port: 5060
+      protocol: TCP
+      targetPort: http 
+  selector:
+    {{- include "log-server.selectorLabels" . | nindent 4 }}

build/charts/laf-server/values.yaml

@@ -24,6 +24,10 @@ default_region:
   tls: false
   runtime_domain: ""
   website_domain: ""
+  # log-server
+  log_server_url: ""
+  log_server_secret: ""
+  log_server_database_url: ""
 jwt:
   secret: laf_server_abc123
   expires_in: 7d

build/start.sh

@@ -71,6 +71,9 @@ helm install minio -n ${NAMESPACE} \
 
 ## 4. install laf-server
 SERVER_JWT_SECRET=$PASSWD_OR_SECRET
+LOG_SERVER_URL="http://log-server.${NAMESPACE}.svc.cluster.local:5060"
+LOG_SERVER_DATABASE_URL="mongodb://${DB_USERNAME:-admin}:${PASSWD_OR_SECRET}@mongodb-0.mongo.${NAMESPACE}.svc.cluster.local:27017/function-logs?authSource=admin&replicaSet=rs0&w=majority"
+LOG_SERVER_SECRET=$PASSWD_OR_SECRET
 helm install server -n ${NAMESPACE} \
     --set databaseUrl=${DATABASE_URL} \
     --set meteringDatabaseUrl=${METERING_DATABASE_URL} \
@@ -90,6 +93,9 @@ helm install server -n ${NAMESPACE} \
     --set default_region.apisix_api_url=${APISIX_API_URL} \
     --set default_region.apisix_api_key=${APISIX_API_KEY} \
     --set default_region.apisix_public_port=80 \
+    --set default_region.log_server_url=${LOG_SERVER_URL} \
+    --set default_region.log_server_secret=${LOG_SERVER_SECRET} \
+    --set default_region.log_server_database_url=${LOG_SERVER_DATABASE_URL} \
     ./charts/laf-server
 
 ## 6. install laf-web

docs/en/guide/function/logs.md

@@ -17,25 +17,3 @@ You can filter logs based on the `requestId` and the name of the cloud function.
 1. Click on a log to switch to the log section.
 2. Use the `requestId` and the name of the cloud function to search for specific logs.
 3. Click on an individual log to view detailed content.
-
-## Manual Log Cleaning
-
-The running logs of Laf cloud functions are stored in a hidden collection called `__function_logs__`. Therefore, we can use the method to manipulate the database in cloud functions to clean the logs.
-
-The following is the code to clean all logs in a cloud function:
-
-::: danger
-The following operation will delete all historical logs. Please proceed with caution.
-:::
-
-```typescript
-import cloud from '@lafjs/cloud'
-
-export async function main(ctx: FunctionContext) {
-  console.log('Hello World')
-  // Database, remove all logs
-  const db = cloud.database();
-  const res = await db.collection('__function_logs__').remove({multi:true})
-  console.log(res)
-}
-```

docs/guide/function/logs.md

@@ -19,27 +19,3 @@ title: 云函数历史日志
 2、可根据 `requestId` 和云函数名搜索指定日志
 
 3、点击单个日志，可查看详细内容
-
-## 手动清理日志
-
-Laf 云函数的运行日志都在一个隐藏的集合中：`__function_logs__`
-
-所以我们可以通过云函数操作数据库的方法，清理日志
-
-以下是清理全部日志的云函数写法：
-
-::: danger
-以下操作会删除全部历史日志，请谨慎操作
-:::
-
-```typescript
-import cloud from '@lafjs/cloud'
-
-export async function main(ctx: FunctionContext) {
-  console.log('Hello World')
-  // 数据库，删除全部日志
-  const db = cloud.database();
-  const res = await db.collection('__function_logs__').remove({multi:true})
-  console.log(res)
-}
-```

lerna.json

@@ -5,7 +5,8 @@
     "./server",
     "./web",
     "./runtimes/nodejs",
-    "./cli"
+    "./cli",
+    "./services/*"
   ],
   "version": "1.0.0-beta.10",
   "command": {

runtimes/nodejs/package.json

@@ -6,8 +6,10 @@
   "typings": "./dist/index.d.ts",
   "private": "true",
   "scripts": {
-    "start": "node --expose-internals ./dist/index.js",
-    "dev": "NODE_ENV=development npm run start",
+    "start": "node ./dist/index.js",
+    "dev": "npx concurrently npm:dev:*",
+    "dev:start": "npx nodemon ./dist/index.js",
+    "dev:watch": "npm run watch",
     "build": "tsc -p tsconfig.json",
     "watch": "tsc -p tsconfig.json -w",
     "prepublishOnly": "npm run build",
@@ -73,4 +75,4 @@
     ],
     "delay": 1000
   }
-}
+}

runtimes/nodejs/src/config.ts

@@ -57,13 +57,6 @@ export default class Config {
     return process.env.NODE_ENV === 'production'
   }
 
-  /**
-   * Expired time of function logs, in seconds
-   */
-  static get FUNCTION_LOG_EXPIRED_TIME(): number {
-    return (process.env.FUNCTION_LOG_EXPIRED_TIME ?? 3600 * 24 * 3) as number
-  }
-
   static get RUNTIME_IMAGE(): string {
     return process.env.RUNTIME_IMAGE
   }
@@ -83,4 +76,12 @@ export default class Config {
   static get REQUEST_LIMIT_SIZE(): string {
     return process.env.REQUEST_LIMIT_SIZE || '10mb'
   }
+
+  static get LOG_SERVER_URL(): string { 
+    return process.env.LOG_SERVER_URL || ''
+  }
+
+  static get LOG_SERVER_TOKEN(): string { 
+    return process.env.LOG_SERVER_TOKEN || ''
+  }
 }

runtimes/nodejs/src/constants.ts

@@ -3,7 +3,6 @@
  */
 export const CLOUD_FUNCTION_COLLECTION = '__functions__'
 export const POLICY_COLLECTION = '__policies__'
-export const FUNCTION_LOG_COLLECTION = '__function_logs__'
 export const CONFIG_COLLECTION = '__conf__'
 
 export const WEBSOCKET_FUNCTION_NAME = '__websocket__'

runtimes/nodejs/src/index.ts

@@ -21,12 +21,10 @@ import xmlparser from 'express-xml-bodyparser'
 import './support/cloud-sdk'
 import { FunctionCache } from './support/function-engine/cache'
 import { DatabaseChangeStream } from './support/db-change-stream'
-import { ensureCollectionIndexes } from './support/function-log'
 
 const app = express()
 
 DatabaseAgent.accessor.ready.then(() => {
-  ensureCollectionIndexes()
   FunctionCache.initialize()
   DatabaseChangeStream.initialize()
 })

runtimes/nodejs/src/support/function-engine/console.ts

@@ -1,10 +1,30 @@
 import * as util from 'util'
 import { FunctionContext } from './types'
+import Config from '../../config'
+import axios from 'axios'
 
 export class FunctionConsole {
   ctx: FunctionContext
 
-  static write: (message: string, ctx: FunctionContext) => void = console.log
+  static write(message: string, ctx: FunctionContext) {
+    if (!Config.LOG_SERVER_URL || !Config.LOG_SERVER_TOKEN) return
+
+    const doc = {
+      request_id: ctx.requestId,
+      func: ctx.__function_name,
+      data: message,
+      created_at: new Date(),
+    }
+
+    axios.post(`${Config.LOG_SERVER_URL}/function/log`, {
+      appid: Config.APPID,
+      log: doc,
+    }, {
+      headers: {
+        'x-token': Config.LOG_SERVER_TOKEN
+      }
+    })
+  }
 
   constructor(ctx: FunctionContext) {
     this.ctx = ctx