diff --git a/build/Kubefile b/build/Kubefile index 211b29fbc6..2dec43a9a0 100644 --- a/build/Kubefile +++ b/build/Kubefile @@ -4,8 +4,10 @@ USER 65532:65532 COPY charts ./charts # COPY manifests ./manifests COPY registry ./registry +COPY deploy-manifest ./deploy-manifest COPY start.sh ./start.sh COPY prometheus-helm.yaml ./prometheus-helm.yaml +COPY mongodb.yaml ./mongodb.yaml ENV DOMAIN=127.0.0.1.nip.io ENV NAMESPACE=laf-system diff --git a/build/charts/laf-server/templates/deployment.yaml b/build/charts/laf-server/templates/deployment.yaml index ee4e0a6fb2..2723aa0b07 100644 --- a/build/charts/laf-server/templates/deployment.yaml +++ b/build/charts/laf-server/templates/deployment.yaml @@ -81,6 +81,8 @@ spec: value: {{ .Values.default_region.tls.wildcard_certificate_secret_name | quote}} - name: DEFAULT_REGION_PROMETHEUS_URL value: {{ .Values.default_region.prometheus_url }} + - name: DEFAULT_REGION_DEPLOY_MANIFEST + value: {{ .Values.default_region.deploy_manifest }} - name: SITE_NAME value: {{ .Values.siteName | quote}} {{- with .Values.nodeSelector }} diff --git a/build/charts/laf-server/templates/rumtime-exporter.yaml b/build/charts/laf-server/templates/rumtime-exporter.yaml index 36ace1195f..55688888eb 100644 --- a/build/charts/laf-server/templates/rumtime-exporter.yaml +++ b/build/charts/laf-server/templates/rumtime-exporter.yaml @@ -1,3 +1,4 @@ +{{- if .Values.default_region.runtime_exporter_secret }} apiVersion: apps/v1 kind: Deployment metadata: @@ -98,3 +99,18 @@ spec: expr: max_over_time(sum by (appid) (laf_runtime_cpu_limit{container!=""})[1h:]) - record: laf:billing:memory expr: max_over_time(sum by (appid) (laf_runtime_memory_limit{container!=""})[1h:]) +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: kb-mongodb +spec: + endpoints: + - interval: 30s + path: /metrics + port: http-metrics + selector: + matchLabels: + app.kubernetes.io/name: mongodb + apps.kubeblocks.io/component-name: mongodb +{{- end }} \ No newline at end of file diff --git a/build/charts/laf-server/values.yaml b/build/charts/laf-server/values.yaml index 3115168178..2bf06367a0 100644 --- a/build/charts/laf-server/values.yaml +++ b/build/charts/laf-server/values.yaml @@ -27,6 +27,7 @@ default_region: runtime_exporter_secret: "" # prometheus prometheus_url: "" + deploy_manifest: "" jwt: secret: laf_server_abc123 expires_in: 7d diff --git a/build/charts/mongodb/.helmignore b/build/charts/mongodb/.helmignore deleted file mode 100644 index 0e8a0eb36f..0000000000 --- a/build/charts/mongodb/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/build/charts/mongodb/Chart.yaml b/build/charts/mongodb/Chart.yaml deleted file mode 100644 index 202c1abbb3..0000000000 --- a/build/charts/mongodb/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: mongodb -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.1.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.16.0" diff --git a/build/charts/mongodb/README.md b/build/charts/mongodb/README.md deleted file mode 100644 index 76062f9f5d..0000000000 --- a/build/charts/mongodb/README.md +++ /dev/null @@ -1,16 +0,0 @@ -## Install - -```shell -helm install mongodb \ - --set db.username=admin \ - --set db.password=passw0rd \ - --namespace mongodb \ - . - # end of script -``` - -## Uninstall - -```shell -helm uninstall mongodb -n mongodb -``` diff --git a/build/charts/mongodb/keyFile b/build/charts/mongodb/keyFile deleted file mode 100644 index ba7ba569db..0000000000 --- a/build/charts/mongodb/keyFile +++ /dev/null @@ -1,16 +0,0 @@ -yzfsO9eB1U/Rw5+FlvkdTRDeIfUjexbKZir+a2md7K7nuneaJx2hKldHuivwBpfd -r0jZK/fpjedHkZ6OnekVshqP8U1zSPj8gPLszgLFBTZ7OssKfagMepVKp7jiUKPP -xeB4dxRr+6FvofjThCECCVaBu5hBxjBMHECgEvxmlZoDrGhEn1uCeTlDvoAbDQb8 -AQkD62c9qwNEOz9QCeo5lby4QsheErZqa+82ZZ16tl9MgflxjZ/s4i71j46D5W9q -9o7LWgeLVkOqHtf9llDUayX7LllT2giI5fozqYxtZ4jcsQNITKwmtbf1FrWmZDfJ -1K9p4Fe2MiXoy5nK61bll3BM9S7KkaT4LPnzeImj7DGC8dkRRLtVEwPMdxJELF4a -3SLeE1k4mkxF7OcBlkLzZzNqSgyOqdG/vxNKLhuHGC6vdXzugcozIzrlbyCG56kU -WAb4Ju7g1K4P0oNFHBvfWX9vVkI2BHf6OfD9zF+TbESAflBxUsD8V76+azeZrrA7 -3LkOsueSyA4BkcvEN/pWgocGdc3rZqmFEQgQtqkl2UMGzzEntKGv28Q457nAzMBV -TSDab3WIw9fsWJb9r+pidtHSialcPx8Pz0KYQ5pCBZ5g0DXGeZkmpZ5AaOwK3GEb -/by0MHBORbZ4OnrnY25+vnyqRNkeXE3yWRXlwqjlL7bx0SYXB+OpeJNAsUB/BAbL -93heBfmRlyDMGsc7LaujSIM7Dybro3qG5aUXH49HWlMZua9X8dgy5W6p6uQqNAp9 -sNF3G8H9NvjUB8B4t/sjWu/wCH07vE463eYNUOByicBkkiACx3dP+SMZVBgX37Ls -QXpbdTBWilwiWrevEXOetMQNVHBlCQQmmY9D8j3nFBnG58a5F5liYjknUkJlkQ/6 -K93v9WSuGWzMU+sQw5UtyyK51mjRFmDSSPPT1Kq3LQD2w9y2ibLcYgVNbWAjfUzE -vg5ob3ZgJM+KE6VyeM8y2TkobQkFkuVMz2/E4vWrfsYA6MzA diff --git a/build/charts/mongodb/templates/NOTES.txt b/build/charts/mongodb/templates/NOTES.txt deleted file mode 100644 index 52842e8d2e..0000000000 --- a/build/charts/mongodb/templates/NOTES.txt +++ /dev/null @@ -1,13 +0,0 @@ - -Use mongo client in cluster: - - export ROOT_USERNAME=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ .Release.Name }}-{{ .Chart.Name }}-init -o jsonpath="{.data.username}" | base64 -d) - export ROOT_PASSWORD=$(kubectl get secret --namespace {{ .Release.Namespace }} {{ .Release.Name }}-{{ .Chart.Name }}-init -o jsonpath="{.data.password}" | base64 -d) - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "mongodb.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - -Connection URI: - export CONNECTION_URI="mongodb://$ROOT_USERNAME:$ROOT_PASSWORD@{{ .Chart.Name }}-0.{{ .Values.service.name }}.{{ .Release.Namespace }}.svc.cluster.local:27017/{{ .Values.db.database }}?authSource=admin&replicaSet={{ .Values.db.replicaSetName }}&w=majority" - -Connect in cluster: - kubectl run mongo --rm -it --env="URI=$CONNECTION_URI" --image={{ .Values.image.repository }}:{{ .Values.image.tag }} -- sh diff --git a/build/charts/mongodb/templates/_helpers.tpl b/build/charts/mongodb/templates/_helpers.tpl deleted file mode 100644 index 6b1b8c1797..0000000000 --- a/build/charts/mongodb/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "mongodb.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "mongodb.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "mongodb.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "mongodb.labels" -}} -helm.sh/chart: {{ include "mongodb.chart" . }} -{{ include "mongodb.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "mongodb.selectorLabels" -}} -app.kubernetes.io/name: {{ include "mongodb.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "mongodb.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "mongodb.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/build/charts/mongodb/templates/ingress.yaml b/build/charts/mongodb/templates/ingress.yaml deleted file mode 100644 index 68112df989..0000000000 --- a/build/charts/mongodb/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "mongodb.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "mongodb.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/build/charts/mongodb/templates/secret.yaml b/build/charts/mongodb/templates/secret.yaml deleted file mode 100644 index a3a4ed3353..0000000000 --- a/build/charts/mongodb/templates/secret.yaml +++ /dev/null @@ -1,13 +0,0 @@ - -apiVersion: v1 -kind: Secret -metadata: - name: {{ .Release.Name }}-{{ .Chart.Name }}-init -type: Opaque -data: - # base64 encoded string - username: {{ .Values.db.username | b64enc }} - password: {{ .Values.db.password | b64enc }} - database: {{ .Values.db.database | b64enc }} - keyFile: | - {{ .Files.Get "keyFile" | b64enc }} \ No newline at end of file diff --git a/build/charts/mongodb/templates/service.yaml b/build/charts/mongodb/templates/service.yaml deleted file mode 100644 index e80d20d454..0000000000 --- a/build/charts/mongodb/templates/service.yaml +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: {{ .Values.service.name }} - labels: - {{- include "mongodb.labels" . | nindent 4 }} -spec: - #type: {{ .Values.service.type }} - clusterIP: None - ports: - - port: {{ .Values.service.port }} - targetPort: tcp - protocol: TCP - name: tcp - selector: - {{- include "mongodb.selectorLabels" . | nindent 4 }} diff --git a/build/charts/mongodb/templates/serviceaccount.yaml b/build/charts/mongodb/templates/serviceaccount.yaml deleted file mode 100644 index acce4c31b0..0000000000 --- a/build/charts/mongodb/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "mongodb.serviceAccountName" . }} - labels: - {{- include "mongodb.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/build/charts/mongodb/templates/statefulset.yaml b/build/charts/mongodb/templates/statefulset.yaml deleted file mode 100644 index 255fccfd54..0000000000 --- a/build/charts/mongodb/templates/statefulset.yaml +++ /dev/null @@ -1,117 +0,0 @@ -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: {{ include "mongodb.fullname" . }} - labels: - {{- include "mongodb.labels" . | nindent 4 }} -spec: - selector: - matchLabels: - {{- include "mongodb.selectorLabels" . | nindent 6 }} - serviceName: {{ .Values.service.name }} - template: - metadata: - {{- with .Values.podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "mongodb.selectorLabels" . | nindent 8 }} - spec: - {{- with .Values.imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "mongodb.serviceAccountName" . }} - securityContext: - {{- toYaml .Values.podSecurityContext | nindent 8 }} - hostname: {{ include "mongodb.fullname" . }} - containers: - - name: {{ .Chart.Name }} - securityContext: - {{- toYaml .Values.securityContext | nindent 12 }} - image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - ports: - - name: tcp - containerPort: 27017 - protocol: TCP - livenessProbe: - exec: - command: - - bash - - "-c" - - | - mongo -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --eval db.adminCommand\(\"ping\"\) - initialDelaySeconds: 20 - failureThreshold: 6 - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - readinessProbe: - exec: - command: - - bash - - "-c" - - | - mongo -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --eval rs.status\(\) > rs_status - cat /rs_status | grep ok | grep 1 || mongo -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --eval rs.initiate\(\) > /rs_init - cat /rs_status | grep {{ .Chart.Name }}-0:27017 && mongo -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --eval conf=rs.conf\(\)\;conf.members[0].host=\"{{ .Chart.Name }}-0.{{ .Values.service.name }}.{{ .Release.Namespace }}.svc.cluster.local:27017\"\;rs.reconfig\(conf\) > /reconf_result - mongo -u $MONGO_INITDB_ROOT_USERNAME -p $MONGO_INITDB_ROOT_PASSWORD --eval db.adminCommand\(\"ping\"\) - initialDelaySeconds: 20 - failureThreshold: 6 - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 5 - resources: - {{- toYaml .Values.resources | nindent 12 }} - command: - - bash - - -c - - | - cp /keyfile /data/replica.key - chmod 400 /data/replica.key - chown 999:999 /data/replica.key - exec docker-entrypoint.sh $$@ - args: ["mongod", "--bind_ip", "0.0.0.0", "--replSet", "{{ .Values.db.replicaSetName }}", "--keyFile", "/data/replica.key"] - env: - - name: MONGO_INITDB_ROOT_USERNAME - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-{{ .Chart.Name }}-init - key: username - - name: MONGO_INITDB_ROOT_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-{{ .Chart.Name }}-init - key: password - - name: MONGO_INITDB_DATABASE - valueFrom: - secretKeyRef: - name: {{ .Release.Name }}-{{ .Chart.Name }}-init - key: database - volumeMounts: - - name: mongodb-data - mountPath: /data/db - - name: mongodb-key-file - mountPath: /keyfile - subPath: keyFile - volumes: - - name: mongodb-key-file - secret: - secretName: {{ .Release.Name }}-{{ .Chart.Name }}-init - - name: mongodb-data - persistentVolumeClaim: - claimName: {{ .Release.Name }}-db-pvc - {{- with .Values.nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .Values.tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} diff --git a/build/charts/mongodb/templates/storage.yaml b/build/charts/mongodb/templates/storage.yaml deleted file mode 100644 index 46a4072b3e..0000000000 --- a/build/charts/mongodb/templates/storage.yaml +++ /dev/null @@ -1,12 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: {{ .Release.Name }}-db-pvc -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: {{ .Values.storage.size }} - diff --git a/build/charts/mongodb/templates/tests/test-connection.yaml b/build/charts/mongodb/templates/tests/test-connection.yaml deleted file mode 100644 index 9d878f68e1..0000000000 --- a/build/charts/mongodb/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "mongodb.fullname" . }}-test-connection" - labels: - {{- include "mongodb.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: "busybox:1.28" - command: ['wget'] - args: ['{{ include "mongodb.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/build/charts/mongodb/values.yaml b/build/charts/mongodb/values.yaml deleted file mode 100644 index 48edc40839..0000000000 --- a/build/charts/mongodb/values.yaml +++ /dev/null @@ -1,91 +0,0 @@ -# Default values for mongodb. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# app config -db: - username: admin - password: passw0rd - database: sys_db - replicaSetName: rs0 - -storage: - size: 10Gi - -# k8s resource config -replicaCount: 1 - -image: - repository: mongo - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: 5.0.14 - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "mongodb" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: - { fsGroup: 999 } - # fsGroup: 2000 - -securityContext: - {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - name: mongo - type: ClusterIP - port: 27017 - -ingress: - enabled: false - className: "" - annotations: - {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: - {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/build/deploy-manifest/database.yaml b/build/deploy-manifest/database.yaml new file mode 100644 index 0000000000..cc7472933c --- /dev/null +++ b/build/deploy-manifest/database.yaml @@ -0,0 +1,43 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + finalizers: + - cluster.kubeblocks.io/finalizer + labels: + clusterdefinition.kubeblocks.io/name: mongodb + clusterversion.kubeblocks.io/name: mongodb-5.0 + sealos-db-provider-cr: <%- name %> + annotations: {} + name: <%- name %> + namespace: $NAMESPACE +spec: + affinity: + nodeLabels: {} + podAntiAffinity: Preferred + tenancy: SharedNode + topologyKeys: [] + clusterDefinitionRef: mongodb + clusterVersionRef: mongodb-5.0 + componentSpecs: + - componentDefRef: mongodb + monitor: true + name: mongodb + replicas: <%- replicas %> + resources: + limits: + cpu: <%- limitCPU %>m + memory: <%- limitMemory %>Mi + requests: + cpu: <%- requestCPU %>m + memory: <%- requestMemory %>Mi + serviceAccountName: laf-mongodb-<%- name %> + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: <%- capacity %>Mi + terminationPolicy: Delete + tolerations: [] diff --git a/build/mongodb.yaml b/build/mongodb.yaml new file mode 100644 index 0000000000..4f28eaf2ed --- /dev/null +++ b/build/mongodb.yaml @@ -0,0 +1,33 @@ +apiVersion: apps.kubeblocks.io/v1alpha1 +kind: Cluster +metadata: + finalizers: + - cluster.kubeblocks.io/finalizer + labels: + clusterdefinition.kubeblocks.io/name: mongodb + clusterversion.kubeblocks.io/name: mongodb-5.0 + name: mongodb +spec: + affinity: + nodeLabels: {} + podAntiAffinity: Preferred + tenancy: SharedNode + topologyKeys: [] + clusterDefinitionRef: mongodb + clusterVersionRef: mongodb-5.0 + componentSpecs: + - componentDefRef: mongodb + monitor: true + name: mongodb + replicas: 1 + serviceAccountName: "" + volumeClaimTemplates: + - name: data + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: $CAPACITY + terminationPolicy: Delete + tolerations: [] diff --git a/build/start.sh b/build/start.sh index eb3c564448..e3fa577bb0 100644 --- a/build/start.sh +++ b/build/start.sh @@ -26,14 +26,15 @@ kubectl create namespace ${NAMESPACE} || true set -e set -x -DATABASE_URL="mongodb://${DB_USERNAME:-admin}:${PASSWD_OR_SECRET}@mongodb-0.mongo.${NAMESPACE}.svc.cluster.local:27017/sys_db?authSource=admin&replicaSet=rs0&w=majority" -helm install mongodb -n ${NAMESPACE} \ - --set db.username=${DB_USERNAME:-admin} \ - --set db.password=${PASSWD_OR_SECRET} \ - --set storage.size=${DB_PV_SIZE:-5Gi} \ - ./charts/mongodb - -## 3. install prometheus +sed "s/\$CAPACITY/${DB_PV_SIZE:-5Gi}/g" mongodb.yaml | kubectl apply -n ${NAMESPACE} -f - +kubectl wait --for=condition=exists --timeout=120s secret/mongodb-conn-credential -n ${NAMESPACE} + +DB_USERNAME=$(kubectl get secret -n ${NAMESPACE} mongodb-conn-credential -ojsonpath='{.data.username}' | base64 -d) +DB_PASSWORD=$(kubectl get secret -n ${NAMESPACE} mongodb-conn-credential -ojsonpath='{.data.password}' | base64 -d) +DB_ENDPOINT=$(kubectl get secret -n ${NAMESPACE} mongodb-conn-credential -ojsonpath='{.data.headlessEndpoint}' | base64 -d) +DATABASE_URL="mongodb://${DB_USERNAME}:${DB_PASSWORD}@${DB_ENDPOINT}/sys_db?authSource=admin&replicaSet=rs0&w=majority" + +## 2. install prometheus PROMETHEUS_URL=http://prometheus-operated.${NAMESPACE}.svc.cluster.local:9090 if [ "$ENABLE_MONITOR" = "true" ]; then sed -e "s/\$NAMESPACE/$NAMESPACE/g" \ @@ -43,16 +44,9 @@ if [ "$ENABLE_MONITOR" = "true" ]; then helm install prometheus --version 48.3.3 -n ${NAMESPACE} \ -f ./prometheus-helm-with-values.yaml \ ./charts/kube-prometheus-stack - - helm install prometheus-mongodb-exporter --version 3.2.0 -n ${NAMESPACE} \ - --set mongodb.uri=${DATABASE_URL} \ - --set serviceMonitor.enabled=true \ - --set serviceMonitor.additionalLabels.release=prometheus \ - --set serviceMonitor.additionalLabels.namespace=${NAMESPACE} \ - ./charts/prometheus-mongodb-exporter fi -## 4. install minio +## 3. install minio MINIO_ROOT_ACCESS_KEY=minio-root-user MINIO_ROOT_SECRET_KEY=$PASSWD_OR_SECRET MINIO_DOMAIN=oss.${DOMAIN} @@ -70,9 +64,18 @@ helm install minio -n ${NAMESPACE} \ --set metrics.serviceMonitor.additionalLabels.namespace=${NAMESPACE} \ ./charts/minio -## 5. install laf-server +## 4. install laf-server SERVER_JWT_SECRET=$PASSWD_OR_SECRET RUNTIME_EXPORTER_SECRET=$PASSWD_OR_SECRET + +DEPLOY_MANIFEST=$(jq -n '{}') +for file in deploy-manifest/*.yaml; do + content=$(cat "$file" | jq -Rs .) + key=$(basename "$file") + DEPLOY_MANIFEST=$(echo "$DEPLOY_MANIFEST" | jq --arg key "$key" --arg value "$content" '. + {($key): $value}') +done +DEPLOY_MANIFEST=$(echo "$DEPLOY_MANIFEST" | sed "s/\$NAMESPACE/$NAMESPACE/g") + helm install server -n ${NAMESPACE} \ --set databaseUrl=${DATABASE_URL} \ --set jwt.secret=${SERVER_JWT_SECRET} \ @@ -89,11 +92,12 @@ helm install server -n ${NAMESPACE} \ --set default_region.runtime_domain=${DOMAIN} \ --set default_region.website_domain=${DOMAIN} \ --set default_region.tls.enabled=false \ - --set default_region.runtime_exporter_secret=${RUNTIME_EXPORTER_SECRET} \ + $([ "$ENABLE_MONITOR" = "true" ] && echo "--set default_region.runtime_exporter_secret=${RUNTIME_EXPORTER_SECRET}") \ $([ "$ENABLE_MONITOR" = "true" ] && echo "--set default_region.prometheus_url=${PROMETHEUS_URL}") \ + --set default_region.deploy_manifest=${DEPLOY_MANIFEST} ./charts/laf-server -## 6. install laf-web +## 5. install laf-web helm install web -n ${NAMESPACE} \ --set domain=${DOMAIN} \ ./charts/laf-web diff --git a/deploy/install-on-linux.sh b/deploy/install-on-linux.sh index 3daf4a31d5..ee8e45cf4f 100644 --- a/deploy/install-on-linux.sh +++ b/deploy/install-on-linux.sh @@ -19,6 +19,7 @@ if [ -x "$(command -v apt)" ]; then apt update apt install iptables host -y apt install sealos=4.3.5 -y + apt install jq -y # fix /etc/hosts overwrite bug in ubuntu while restarting sed -i "/update_etc_hosts/c \\ - ['update_etc_hosts', 'once-per-instance']" /etc/cloud/cloud.cfg && touch /var/lib/cloud/instance/sem/config_update_etc_hosts @@ -37,6 +38,7 @@ EOF yum clean all yum install -y bind-utils iptables yum install sealos=4.3.5 -y + yum install jq -y fi ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) @@ -59,6 +61,7 @@ sealos pull labring/cert-manager:v1.8.0 sealos pull labring/metrics-server:v0.6.2 sealos pull lafyun/laf:latest sealos pull docker.io/labring/ingress-nginx:v1.8.1 +sealos pull labring/kubeblocks:v0.7.1 # install k8s cluster sealos run labring/kubernetes:v1.24.9 labring/flannel:v0.19.0 labring/helm:v3.8.2 @@ -74,6 +77,7 @@ sealos run labring/cert-manager:v1.8.0 sealos run labring/metrics-server:v0.6.2 sealos run docker.io/labring/ingress-nginx:v1.8.1 \ -e HELM_OPTS="--set controller.hostNetwork=true --set controller.kind=DaemonSet --set controller.service.enabled=false" +sealos run labring/kubeblocks:v0.7.1 sealos run --env DOMAIN=$DOMAIN --env DB_PV_SIZE=5Gi --env OSS_PV_SIZE=5Gi --env EXTERNAL_HTTP_SCHEMA=http lafyun/laf:latest diff --git a/server/package-lock.json b/server/package-lock.json index 30fb134c5c..1160d796c8 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -20,6 +20,7 @@ "@nestjs/common": "^9.0.0", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.2.1", + "@nestjs/event-emitter": "^2.0.3", "@nestjs/jwt": "^10.0.1", "@nestjs/mapped-types": "*", "@nestjs/passport": "^9.0.0", @@ -7505,6 +7506,19 @@ } } }, + "node_modules/@nestjs/event-emitter": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nestjs/event-emitter/-/event-emitter-2.0.3.tgz", + "integrity": "sha512-Pt7KAERrgK0OjvarSI3wfVhwZ8X1iLq1lXuodyRe+Zx3aLLP7fraFUHirASbFkB6KIQ1Zj+gZ1g8a9eu4GfFhw==", + "dependencies": { + "eventemitter2": "6.4.9" + }, + "peerDependencies": { + "@nestjs/common": "^8.0.0 || ^9.0.0 || ^10.0.0", + "@nestjs/core": "^8.0.0 || ^9.0.0 || ^10.0.0", + "reflect-metadata": "^0.1.12" + } + }, "node_modules/@nestjs/jwt": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@nestjs/jwt/-/jwt-10.0.1.tgz", @@ -12243,6 +12257,11 @@ "through": "^2.3.8" } }, + "node_modules/eventemitter2": { + "version": "6.4.9", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.9.tgz", + "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==" + }, "node_modules/events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", @@ -24815,6 +24834,14 @@ "uuid": "9.0.0" } }, + "@nestjs/event-emitter": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@nestjs/event-emitter/-/event-emitter-2.0.3.tgz", + "integrity": "sha512-Pt7KAERrgK0OjvarSI3wfVhwZ8X1iLq1lXuodyRe+Zx3aLLP7fraFUHirASbFkB6KIQ1Zj+gZ1g8a9eu4GfFhw==", + "requires": { + "eventemitter2": "6.4.9" + } + }, "@nestjs/jwt": { "version": "10.0.1", "resolved": "https://registry.npmjs.org/@nestjs/jwt/-/jwt-10.0.1.tgz", @@ -28770,6 +28797,11 @@ "through": "^2.3.8" } }, + "eventemitter2": { + "version": "6.4.9", + "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-6.4.9.tgz", + "integrity": "sha512-JEPTiaOt9f04oa6NOkc4aH+nVp5I3wEjpHbIPqfgCdD5v5bUzy7xQqwcVO2aDQgOWhI28da57HksMrzK9HlRxg==" + }, "events": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", diff --git a/server/package.json b/server/package.json index 5c345b4018..37161d447c 100644 --- a/server/package.json +++ b/server/package.json @@ -35,6 +35,7 @@ "@nestjs/common": "^9.0.0", "@nestjs/config": "^2.2.0", "@nestjs/core": "^9.2.1", + "@nestjs/event-emitter": "^2.0.3", "@nestjs/jwt": "^10.0.1", "@nestjs/mapped-types": "*", "@nestjs/passport": "^9.0.0", diff --git a/server/src/app.module.ts b/server/src/app.module.ts index 3b50568680..e6516266f9 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -30,6 +30,7 @@ import { APP_INTERCEPTOR } from '@nestjs/core' import { AppInterceptor } from './app.interceptor' import { InterceptorModule } from './interceptor/interceptor.module' import { MonitorModule } from './monitor/monitor.module' +import { EventEmitterModule } from '@nestjs/event-emitter' @Module({ imports: [ @@ -76,6 +77,7 @@ import { MonitorModule } from './monitor/monitor.module' GroupModule, InterceptorModule, MonitorModule, + EventEmitterModule.forRoot(), ], controllers: [AppController], providers: [ diff --git a/server/src/application/application-task.service.ts b/server/src/application/application-task.service.ts index 0d5e89a3a4..69cc525463 100644 --- a/server/src/application/application-task.service.ts +++ b/server/src/application/application-task.service.ts @@ -24,6 +24,7 @@ import { DatabasePhase } from 'src/database/entities/database' import { DomainPhase } from 'src/gateway/entities/runtime-domain' import { StoragePhase } from 'src/storage/entities/storage-user' import { ApplicationNamespaceMode } from 'src/region/entities/region' +import { DedicatedDatabaseService } from 'src/database/dedicated-database/dedicated-database.service' @Injectable() export class ApplicationTaskService { @@ -35,6 +36,7 @@ export class ApplicationTaskService { private readonly clusterService: ClusterService, private readonly storageService: StorageService, private readonly databaseService: DatabaseService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, private readonly runtimeDomainService: RuntimeDomainService, private readonly bucketDomainService: BucketDomainService, private readonly triggerService: TriggerService, @@ -119,13 +121,6 @@ export class ApplicationTaskService { storage = await this.storageService.create(app.appid) } - // reconcile database - let database = await this.databaseService.findOne(appid) - if (!database) { - this.logger.log(`Creating database for application ${appid}`) - database = await this.databaseService.create(app.appid) - } - // reconcile runtime domain let runtimeDomain = await this.runtimeDomainService.findOne(appid) if (!runtimeDomain) { @@ -133,6 +128,20 @@ export class ApplicationTaskService { runtimeDomain = await this.runtimeDomainService.create(appid) } + // reconcile database + const dedicatedDatabase = await this.dedicatedDatabaseService.findOne(appid) + if (!dedicatedDatabase) { + let database = await this.databaseService.findOne(appid) + if (!database) { + this.logger.log(`Creating database for application ${appid}`) + database = await this.databaseService.create(app.appid) + } + + if (database?.phase !== DatabasePhase.Created) { + return await this.unlock(appid) + } + } + // waiting resources' phase to be `Created` if (runtimeDomain?.phase !== DomainPhase.Created) { return await this.unlock(appid) @@ -142,10 +151,6 @@ export class ApplicationTaskService { return await this.unlock(appid) } - if (database?.phase !== DatabasePhase.Created) { - return await this.unlock(appid) - } - // update application phase to `Created` await db.collection('Application').updateOne( { _id: app._id, phase: ApplicationPhase.Creating }, @@ -260,6 +265,12 @@ export class ApplicationTaskService { return await this.unlock(appid) } + const dedicatedDatabase = await this.dedicatedDatabaseService.findOne(appid) + if (dedicatedDatabase) { + await this.dedicatedDatabaseService.remove(appid) + return await this.unlock(appid) + } + // delete application storage const storage = await this.storageService.findOne(appid) if (storage) { diff --git a/server/src/application/application.controller.ts b/server/src/application/application.controller.ts index e044bc4daa..8d85619ab4 100644 --- a/server/src/application/application.controller.ts +++ b/server/src/application/application.controller.ts @@ -83,7 +83,7 @@ export class ApplicationController { @ApiResponseObject(ApplicationWithRelations) @Post() async create(@Body() dto: CreateApplicationDto, @InjectUser() user: User) { - const error = dto.autoscaling.validate() + const error = dto.validate() || dto.autoscaling.validate() if (error) { return ResponseUtil.error(error) } @@ -334,11 +334,37 @@ export class ApplicationController { } } + const origin = app.bundle + if ( + (origin.resource.dedicatedDatabase?.limitCPU && dto.databaseCapacity) || + (origin.resource.databaseCapacity && dto.dedicatedDatabase?.cpu) + ) { + return ResponseUtil.error('cannot change database type') + } + const checkSpec = await this.checkResourceSpecification(dto, regionId) if (!checkSpec) { return ResponseUtil.error('invalid resource specification') } + if ( + dto.dedicatedDatabase?.capacity && + origin.resource.dedicatedDatabase?.capacity && + dto.dedicatedDatabase?.capacity < + origin.resource.dedicatedDatabase?.capacity + ) { + return ResponseUtil.error('cannot reduce database capacity') + } + + if ( + dto.dedicatedDatabase?.replicas && + origin.resource.dedicatedDatabase?.replicas && + dto.dedicatedDatabase?.replicas !== + origin.resource.dedicatedDatabase?.replicas + ) { + return ResponseUtil.error('cannot change database replicas') + } + // check if a user exceeds the resource limit in a region const limitResource = await this.quotaServiceTsService.resourceLimit( user._id, @@ -353,13 +379,29 @@ export class ApplicationController { const doc = await this.application.updateBundle(appid, dto, isTrialTier) // restart running application if cpu or memory changed - const origin = app.bundle const isRunning = app.phase === ApplicationPhase.Started const isCpuChanged = origin.resource.limitCPU !== doc.resource.limitCPU const isMemoryChanged = origin.resource.limitMemory !== doc.resource.limitMemory const isAutoscalingCanceled = !doc.autoscaling.enable && origin.autoscaling.enable + const isDedicatedDatabaseChanged = + !isEqual( + origin.resource.dedicatedDatabase.limitCPU, + doc.resource.dedicatedDatabase.limitCPU, + ) || + !isEqual( + origin.resource.dedicatedDatabase.limitMemory, + doc.resource.dedicatedDatabase.limitMemory, + ) || + !isEqual( + origin.resource.dedicatedDatabase.replicas, + doc.resource.dedicatedDatabase.replicas, + ) || + !isEqual( + origin.resource.dedicatedDatabase.capacity, + doc.resource.dedicatedDatabase.capacity, + ) if (!isEqual(doc.autoscaling, origin.autoscaling)) { const { hpa, app } = await this.instance.get(appid) @@ -368,7 +410,10 @@ export class ApplicationController { if ( isRunning && - (isCpuChanged || isMemoryChanged || isAutoscalingCanceled) + (isCpuChanged || + isMemoryChanged || + isAutoscalingCanceled || + isDedicatedDatabaseChanged) ) { await this.application.updateState(appid, ApplicationState.Restarting) } @@ -489,15 +534,46 @@ export class ApplicationController { case 'memory': return option.specs.some((spec) => spec.value === dto.memory) case 'databaseCapacity': + if (!dto.databaseCapacity) return true return option.specs.some( (spec) => spec.value === dto.databaseCapacity, ) case 'storageCapacity': return option.specs.some((spec) => spec.value === dto.storageCapacity) + // dedicated database + case 'dedicatedDatabaseCPU': + return ( + !dto.dedicatedDatabase?.cpu || + option.specs.some( + (spec) => spec.value === dto.dedicatedDatabase.cpu, + ) + ) + case 'dedicatedDatabaseMemory': + return ( + !dto.dedicatedDatabase?.memory || + option.specs.some( + (spec) => spec.value === dto.dedicatedDatabase.memory, + ) + ) + case 'dedicatedDatabaseCapacity': + return ( + !dto.dedicatedDatabase?.capacity || + option.specs.some( + (spec) => spec.value === dto.dedicatedDatabase.capacity, + ) + ) + case 'dedicatedDatabaseReplicas': + return ( + !dto.dedicatedDatabase?.replicas || + option.specs.some( + (spec) => spec.value === dto.dedicatedDatabase.replicas, + ) + ) default: return true } }) + return checkSpec } } diff --git a/server/src/application/application.service.ts b/server/src/application/application.service.ts index 8f46bb0435..a7bb92edf0 100644 --- a/server/src/application/application.service.ts +++ b/server/src/application/application.service.ts @@ -26,6 +26,8 @@ import { GroupMember } from 'src/group/entities/group-member' import { RegionService } from 'src/region/region.service' import { assert } from 'console' import { Region } from 'src/region/entities/region' +import { EventEmitter2 } from '@nestjs/event-emitter' +import { ApplicationCreatingEvent } from './events/application-creating.event' @Injectable() export class ApplicationService { @@ -33,7 +35,8 @@ export class ApplicationService { constructor( private readonly groupService: GroupService, - readonly regionService: RegionService, + private readonly regionService: RegionService, + private readonly eventEmitter: EventEmitter2, ) {} /** @@ -90,6 +93,16 @@ export class ApplicationService { { session }, ) + await this.eventEmitter.emitAsync( + ApplicationCreatingEvent.eventName, + new ApplicationCreatingEvent({ + region, + appid, + session, + dto, + }), + ) + // create application await db.collection('Application').insertOne( { @@ -173,6 +186,8 @@ export class ApplicationService { .project({ 'bundle.resource.requestCPU': 0, 'bundle.resource.requestMemory': 0, + 'bundle.resource.dedicatedDatabase.requestCPU': 0, + 'bundle.resource.dedicatedDatabase.requestMemory': 0, }) .toArray() @@ -217,6 +232,8 @@ export class ApplicationService { .project({ 'bundle.resource.requestCPU': 0, 'bundle.resource.requestMemory': 0, + 'bundle.resource.dedicatedDatabase.requestCPU': 0, + 'bundle.resource.dedicatedDatabase.requestMemory': 0, }) .next() @@ -331,28 +348,46 @@ export class ApplicationService { dto: UpdateApplicationBundleDto, isTrialTier: boolean, ) { - const db = SystemDatabase.db const region = await this.regionService.findByAppId(appid) assert(region, 'region cannot be empty') const resource = this.buildBundleResource(region, dto) const autoscaling = this.buildAutoscalingConfig(dto) - const res = await db - .collection('ApplicationBundle') - .findOneAndUpdate( - { appid }, - { $set: { resource, autoscaling, updatedAt: new Date(), isTrialTier } }, - { - projection: { - 'bundle.resource.requestCPU': 0, - 'bundle.resource.requestMemory': 0, + const client = SystemDatabase.client + const db = SystemDatabase.db + const session = client.startSession() + + try { + session.startTransaction() + + const res = await db + .collection('ApplicationBundle') + .findOneAndUpdate( + { appid }, + { + $set: { resource, autoscaling, updatedAt: new Date(), isTrialTier }, }, - returnDocument: 'after', - }, - ) + { + projection: { + 'bundle.resource.requestCPU': 0, + 'bundle.resource.requestMemory': 0, + 'bundle.resource.dedicatedDatabase.requestCPU': 0, + 'bundle.resource.dedicatedDatabase.requestMemory': 0, + }, + returnDocument: 'after', + }, + ) - return res.value + await session.commitTransaction() + return res.value + } catch (error) { + await session.abortTransaction() + this.logger.error(error) + throw error + } finally { + await session.endSession() + } } async remove(appid: string) { @@ -416,12 +451,19 @@ export class ApplicationService { const limitStorageTPS = Math.floor(dto.cpu * 1) const reservedTimeAfterExpired = 60 * 60 * 24 * 31 // 31 days + const ddbRequestCPU = dto.dedicatedDatabase + ? Math.floor(dto.dedicatedDatabase.cpu * cpuRatio) + : 0 + const ddbRequestMemory = dto.dedicatedDatabase + ? Math.floor(dto.dedicatedDatabase.memory * memoryRatio) + : 0 + const resource = new ApplicationBundleResource({ limitCPU: dto.cpu, limitMemory: dto.memory, requestCPU, requestMemory, - databaseCapacity: dto.databaseCapacity, + databaseCapacity: dto.databaseCapacity || 0, storageCapacity: dto.storageCapacity, limitCountOfCloudFunction, @@ -432,6 +474,15 @@ export class ApplicationService { limitDatabaseTPS, limitStorageTPS, reservedTimeAfterExpired, + + dedicatedDatabase: { + limitCPU: dto.dedicatedDatabase?.cpu || 0, + limitMemory: dto.dedicatedDatabase?.memory || 0, + requestCPU: ddbRequestCPU, + requestMemory: ddbRequestMemory, + capacity: dto.dedicatedDatabase?.capacity || 0, + replicas: dto.dedicatedDatabase?.replicas || 0, + }, }) return resource diff --git a/server/src/application/configuration.service.ts b/server/src/application/configuration.service.ts index b83d9be632..e180942dac 100644 --- a/server/src/application/configuration.service.ts +++ b/server/src/application/configuration.service.ts @@ -3,13 +3,17 @@ import { CN_PUBLISHED_CONF } from 'src/constants' import { DatabaseService } from 'src/database/database.service' import { SystemDatabase } from 'src/system-database' import { ApplicationConfiguration } from './entities/application-configuration' +import { DedicatedDatabaseService } from 'src/database/dedicated-database/dedicated-database.service' @Injectable() export class ApplicationConfigurationService { private readonly db = SystemDatabase.db private readonly logger = new Logger(ApplicationConfigurationService.name) - constructor(private readonly databaseService: DatabaseService) {} + constructor( + private readonly databaseService: DatabaseService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, + ) {} async count(appid: string) { return this.db @@ -24,7 +28,10 @@ export class ApplicationConfigurationService { } async publish(conf: ApplicationConfiguration) { - const { db, client } = await this.databaseService.findAndConnect(conf.appid) + const database = + (await this.dedicatedDatabaseService.findAndConnect(conf.appid)) || + (await this.databaseService.findAndConnect(conf.appid)) + const { db, client } = database const session = client.startSession() try { await session.withTransaction(async () => { diff --git a/server/src/application/dto/create-application.dto.ts b/server/src/application/dto/create-application.dto.ts index 6cc11e2a1b..30bef25c1b 100644 --- a/server/src/application/dto/create-application.dto.ts +++ b/server/src/application/dto/create-application.dto.ts @@ -29,6 +29,6 @@ export class CreateApplicationDto extends UpdateApplicationBundleDto { runtimeId: string validate() { - return null + return super.validate() } } diff --git a/server/src/application/dto/update-application.dto.ts b/server/src/application/dto/update-application.dto.ts index a4a37007fa..5ed7870317 100644 --- a/server/src/application/dto/update-application.dto.ts +++ b/server/src/application/dto/update-application.dto.ts @@ -1,8 +1,10 @@ +import { CreateDedicatedDatabaseDto } from '../../database/dto/create-dedicated-database.dto' import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger' import { IsIn, IsInt, IsNotEmpty, + IsOptional, IsString, Length, ValidateNested, @@ -65,9 +67,9 @@ export class UpdateApplicationBundleDto { memory: number @ApiProperty({ example: 2048 }) - @IsNotEmpty() @IsInt() - databaseCapacity: number + @IsOptional() + databaseCapacity?: number @ApiProperty({ example: 4096 }) @IsNotEmpty() @@ -78,4 +80,20 @@ export class UpdateApplicationBundleDto { @ValidateNested() @Type(() => CreateAutoscalingDto) autoscaling: CreateAutoscalingDto + + @ApiProperty({ type: CreateDedicatedDatabaseDto }) + @ValidateNested() + @IsOptional() + @Type(() => CreateDedicatedDatabaseDto) + dedicatedDatabase?: CreateDedicatedDatabaseDto + + validate() { + if (!this.dedicatedDatabase && !this.databaseCapacity) { + return 'databaseCapacity or dedicatedDatabase must be provided' + } + if (this.databaseCapacity && this.dedicatedDatabase) { + return 'databaseCapacity or dedicatedDatabase must be specified only one' + } + return null + } } diff --git a/server/src/application/entities/application-bundle.ts b/server/src/application/entities/application-bundle.ts index 353d760af1..edc940859a 100644 --- a/server/src/application/entities/application-bundle.ts +++ b/server/src/application/entities/application-bundle.ts @@ -1,6 +1,7 @@ import { ApiProperty, ApiPropertyOptional } from '@nestjs/swagger' import { ObjectId } from 'mongodb' import { Autoscaling } from './application-configuration' +import { DedicatedDatabaseSpec } from 'src/database/entities/dedicated-database' export class ApplicationBundleResource { @ApiProperty({ example: 500 }) @@ -39,6 +40,9 @@ export class ApplicationBundleResource { limitDatabaseTPS: number limitStorageTPS: number + @ApiProperty({ type: DedicatedDatabaseSpec }) + dedicatedDatabase: DedicatedDatabaseSpec + constructor(partial: Partial) { Object.assign(this, partial) } diff --git a/server/src/application/events/application-bundle-updating.event.ts b/server/src/application/events/application-bundle-updating.event.ts new file mode 100644 index 0000000000..bef0695c09 --- /dev/null +++ b/server/src/application/events/application-bundle-updating.event.ts @@ -0,0 +1,18 @@ +import { ClientSession } from 'mongodb' +import { Region } from 'src/region/entities/region' +import { UpdateApplicationBundleDto } from '../dto/update-application.dto' + +export class ApplicationBundleUpdatingEvent { + region: Region + appid: string + session: ClientSession + dto: UpdateApplicationBundleDto + + constructor(partial: Partial) { + Object.assign(this, partial) + } + + static get eventName() { + return 'application.bundle.updating' + } +} diff --git a/server/src/application/events/application-creating.event.ts b/server/src/application/events/application-creating.event.ts new file mode 100644 index 0000000000..b0536fac89 --- /dev/null +++ b/server/src/application/events/application-creating.event.ts @@ -0,0 +1,18 @@ +import { ClientSession } from 'mongodb' +import { CreateApplicationDto } from '../dto/create-application.dto' +import { Region } from 'src/region/entities/region' + +export class ApplicationCreatingEvent { + region: Region + appid: string + session: ClientSession + dto: CreateApplicationDto + + constructor(partial: Partial) { + Object.assign(this, partial) + } + + static get eventName() { + return 'application.creating' + } +} diff --git a/server/src/billing/billing-creation-task.service.ts b/server/src/billing/billing-creation-task.service.ts index 431408f930..09ec1e5836 100644 --- a/server/src/billing/billing-creation-task.service.ts +++ b/server/src/billing/billing-creation-task.service.ts @@ -194,6 +194,18 @@ export class BillingCreationTaskService { usage: priceInput.storageCapacity, amount: priceResult.storageCapacity, }, + dedicatedDatabaseCPU: { + usage: priceInput.dedicatedDatabase.cpu, + amount: priceResult.dedicatedDatabase.cpu, + }, + dedicatedDatabaseMemory: { + usage: priceInput.dedicatedDatabase.memory, + amount: priceResult.dedicatedDatabase.memory, + }, + dedicatedDatabaseCapacity: { + usage: priceInput.dedicatedDatabase.capacity, + amount: priceResult.dedicatedDatabase.capacity, + }, }, startAt: startAt, endAt: nextMeteringTime, @@ -224,6 +236,13 @@ export class BillingCreationTaskService { dto.storageCapacity = bundle.resource.storageCapacity dto.databaseCapacity = bundle.resource.databaseCapacity + dto.dedicatedDatabase = { + cpu: bundle.resource.dedicatedDatabase.limitCPU, + memory: bundle.resource.dedicatedDatabase.limitMemory, + capacity: bundle.resource.dedicatedDatabase.capacity, + replicas: bundle.resource.dedicatedDatabase.replicas, + } + return dto } diff --git a/server/src/billing/billing.service.ts b/server/src/billing/billing.service.ts index 144e46a3e4..04a3493234 100644 --- a/server/src/billing/billing.service.ts +++ b/server/src/billing/billing.service.ts @@ -196,6 +196,22 @@ export class BillingService { groupedOptions[ResourceType.DatabaseCapacity], 'database capacity option not found', ) + assert( + groupedOptions[ResourceType.DedicatedDatabaseCPU], + 'dedicated database cpu option not found', + ) + assert( + groupedOptions[ResourceType.DedicatedDatabaseMemory], + 'dedicated database memory option not found', + ) + assert( + groupedOptions[ResourceType.DedicatedDatabaseCapacity], + 'dedicated database capacity option not found', + ) + assert( + groupedOptions[ResourceType.DedicatedDatabaseReplicas], + 'dedicated database replicas option not found', + ) // calculate cpu price const cpuOption = groupedOptions[ResourceType.CPU] @@ -214,20 +230,50 @@ export class BillingService { // calculate database capacity price const databaseOption = groupedOptions[ResourceType.DatabaseCapacity] const databasePrice = new Decimal(databaseOption.price).mul( - dto.databaseCapacity, + dto.databaseCapacity || 0, ) + const ddbCPUOption = groupedOptions[ResourceType.DedicatedDatabaseCPU] + const ddbCPUPrice = dto.dedicatedDatabase + ? new Decimal(ddbCPUOption.price) + .mul(dto.dedicatedDatabase.cpu) + .mul(dto.dedicatedDatabase.replicas) + : new Decimal(0) + + const ddbMemoryOption = groupedOptions[ResourceType.DedicatedDatabaseMemory] + const ddbMemoryPrice = dto.dedicatedDatabase + ? new Decimal(ddbMemoryOption.price) + .mul(dto.dedicatedDatabase.memory) + .mul(dto.dedicatedDatabase.replicas) + : new Decimal(0) + + const ddbCapacityOption = + groupedOptions[ResourceType.DedicatedDatabaseCapacity] + const ddbCapacityPrice = dto.dedicatedDatabase + ? new Decimal(ddbCapacityOption.price) + .mul(dto.dedicatedDatabase.capacity) + .mul(dto.dedicatedDatabase.replicas) + : new Decimal(0) + + const ddbTotalPrice = ddbCPUPrice.add(ddbMemoryPrice).add(ddbCapacityPrice) + // calculate total price const totalPrice = cpuPrice .add(memoryPrice) .add(storagePrice) .add(databasePrice) + .add(ddbTotalPrice) return { cpu: cpuPrice.toNumber(), memory: memoryPrice.toNumber(), storageCapacity: storagePrice.toNumber(), databaseCapacity: databasePrice.toNumber(), + dedicatedDatabase: { + cpu: ddbCPUPrice.toNumber(), + memory: ddbMemoryPrice.toNumber(), + capacity: ddbCapacityPrice.toNumber(), + }, total: totalPrice.toNumber(), } } diff --git a/server/src/billing/dto/calculate-price.dto.ts b/server/src/billing/dto/calculate-price.dto.ts index 9dfb80e4b7..d04b52c9b1 100644 --- a/server/src/billing/dto/calculate-price.dto.ts +++ b/server/src/billing/dto/calculate-price.dto.ts @@ -1,8 +1,10 @@ -import { ApiProperty } from '@nestjs/swagger' +import { ApiProperty, OmitType } from '@nestjs/swagger' import { IsNotEmpty, IsString } from 'class-validator' import { UpdateApplicationBundleDto } from 'src/application/dto/update-application.dto' -export class CalculatePriceDto extends UpdateApplicationBundleDto { +export class CalculatePriceDto extends OmitType(UpdateApplicationBundleDto, [ + 'validate', +]) { @ApiProperty() @IsNotEmpty() @IsString() diff --git a/server/src/billing/entities/application-billing.ts b/server/src/billing/entities/application-billing.ts index cbb9a8c2d8..1b9d84fb99 100644 --- a/server/src/billing/entities/application-billing.ts +++ b/server/src/billing/entities/application-billing.ts @@ -29,7 +29,16 @@ export class ApplicationBillingDetail { [ResourceType.StorageCapacity]: ApplicationBillingDetailItem; @ApiProperty() - [ResourceType.NetworkTraffic]?: ApplicationBillingDetailItem + [ResourceType.NetworkTraffic]?: ApplicationBillingDetailItem; + + @ApiProperty() + [ResourceType.DedicatedDatabaseCPU]?: ApplicationBillingDetailItem; + + @ApiProperty() + [ResourceType.DedicatedDatabaseMemory]?: ApplicationBillingDetailItem; + + @ApiProperty() + [ResourceType.DedicatedDatabaseCapacity]?: ApplicationBillingDetailItem } export class ApplicationBilling { diff --git a/server/src/billing/entities/resource.ts b/server/src/billing/entities/resource.ts index 511a7142f4..92fd198db2 100644 --- a/server/src/billing/entities/resource.ts +++ b/server/src/billing/entities/resource.ts @@ -7,6 +7,10 @@ export enum ResourceType { DatabaseCapacity = 'databaseCapacity', StorageCapacity = 'storageCapacity', NetworkTraffic = 'networkTraffic', + DedicatedDatabaseCPU = 'dedicatedDatabaseCPU', + DedicatedDatabaseMemory = 'dedicatedDatabaseMemory', + DedicatedDatabaseCapacity = 'dedicatedDatabaseCapacity', + DedicatedDatabaseReplicas = 'dedicatedDatabaseReplicas', } export class ResourceSpec { @@ -54,7 +58,19 @@ export class ResourceBundleSpecMap { [ResourceType.StorageCapacity]: ResourceSpec; @ApiPropertyOptional({ type: ResourceSpec }) - [ResourceType.NetworkTraffic]?: ResourceSpec + [ResourceType.NetworkTraffic]?: ResourceSpec; + + @ApiProperty({ type: ResourceSpec }) + [ResourceType.DedicatedDatabaseCPU]: ResourceSpec; + + @ApiProperty({ type: ResourceSpec }) + [ResourceType.DedicatedDatabaseMemory]: ResourceSpec; + + @ApiProperty({ type: ResourceSpec }) + [ResourceType.DedicatedDatabaseCapacity]: ResourceSpec; + + @ApiProperty({ type: ResourceSpec }) + [ResourceType.DedicatedDatabaseReplicas]: ResourceSpec } export class ResourceBundle { @@ -77,6 +93,10 @@ export class ResourceBundle { [ResourceType.DatabaseCapacity]: ResourceSpec [ResourceType.StorageCapacity]: ResourceSpec [ResourceType.NetworkTraffic]?: ResourceSpec + [ResourceType.DedicatedDatabaseCPU]: ResourceSpec + [ResourceType.DedicatedDatabaseMemory]: ResourceSpec + [ResourceType.DedicatedDatabaseCapacity]: ResourceSpec + [ResourceType.DedicatedDatabaseReplicas]: ResourceSpec } @ApiPropertyOptional() diff --git a/server/src/constants.ts b/server/src/constants.ts index 70b6a39edb..caba1a33e7 100644 --- a/server/src/constants.ts +++ b/server/src/constants.ts @@ -3,6 +3,10 @@ dotenv.config({ path: '.env.local' }) dotenv.config() export class ServerConfig { + static get DEFAULT_REGION_DEPLOY_MANIFEST() { + return process.env.DEFAULT_REGION_DEPLOY_MANIFEST + } + static get DATABASE_URL() { if (!process.env.DATABASE_URL) { throw new Error('DATABASE_URL is not defined') diff --git a/server/src/database/collection/collection.service.ts b/server/src/database/collection/collection.service.ts index 392a6b52fe..4727702005 100644 --- a/server/src/database/collection/collection.service.ts +++ b/server/src/database/collection/collection.service.ts @@ -3,11 +3,15 @@ import { CreateCollectionDto } from '../dto/create-collection.dto' import { UpdateCollectionDto } from '../dto/update-collection.dto' import * as assert from 'node:assert' import { DatabaseService } from '../database.service' +import { DedicatedDatabaseService } from '../dedicated-database/dedicated-database.service' @Injectable() export class CollectionService { private readonly logger = new Logger(CollectionService.name) - constructor(private readonly databaseService: DatabaseService) {} + constructor( + private readonly databaseService: DatabaseService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, + ) {} /** * Create collection in database @@ -16,7 +20,9 @@ export class CollectionService { * @returns */ async create(appid: string, dto: CreateCollectionDto) { - const { client, db } = await this.databaseService.findAndConnect(appid) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(appid)) || + (await this.databaseService.findAndConnect(appid)) assert(db, 'Database not found') try { await db.createCollection(dto.name) @@ -35,7 +41,9 @@ export class CollectionService { * @returns */ async findAll(appid: string) { - const { client, db } = await this.databaseService.findAndConnect(appid) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(appid)) || + (await this.databaseService.findAndConnect(appid)) assert(db, 'Database not found') try { const collections = await db.listCollections().toArray() @@ -67,7 +75,9 @@ export class CollectionService { * @returns */ async update(appid: string, name: string, dto: UpdateCollectionDto) { - const { client, db } = await this.databaseService.findAndConnect(appid) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(appid)) || + (await this.databaseService.findAndConnect(appid)) assert(db, 'Database not found') const command = { @@ -105,7 +115,9 @@ export class CollectionService { * @returns */ async remove(appid: string, name: string) { - const { client, db } = await this.databaseService.findAndConnect(appid) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(appid)) || + (await this.databaseService.findAndConnect(appid)) assert(db, 'Database not found') try { const res = await db.dropCollection(name) diff --git a/server/src/database/database-usage-capture-task.service.ts b/server/src/database/database-usage-capture-task.service.ts index a049306e34..601df41ab7 100644 --- a/server/src/database/database-usage-capture-task.service.ts +++ b/server/src/database/database-usage-capture-task.service.ts @@ -79,7 +79,9 @@ export class DatabaseUsageCaptureTaskService { } async captureDatabaseUsage(appid: string) { - const { db, client } = await this.databaseService.findAndConnect(appid) + const res = await this.databaseService.findAndConnect(appid) + if (!res) return + const { client, db } = res try { const data = await db.stats() const { dataSize } = data diff --git a/server/src/database/database.controller.ts b/server/src/database/database.controller.ts index 54be7b033f..6bd0f8365e 100644 --- a/server/src/database/database.controller.ts +++ b/server/src/database/database.controller.ts @@ -35,6 +35,7 @@ import { ImportDatabaseDto } from './dto/import-database.dto' import { InjectUser } from 'src/utils/decorator' import { User } from 'src/user/entities/user' import { QuotaService } from 'src/user/quota.service' +import { DedicatedDatabaseService } from './dedicated-database/dedicated-database.service' @ApiTags('Database') @ApiBearerAuth('Authorization') @@ -45,6 +46,7 @@ export class DatabaseController { constructor( private readonly dbService: DatabaseService, private readonly quotaService: QuotaService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, ) {} /** @@ -56,7 +58,9 @@ export class DatabaseController { @UseGuards(JwtAuthGuard, ApplicationAuthGuard) @Post('proxy') async proxy(@Param('appid') appid: string, @Req() req: IRequest) { - const accessor = await this.dbService.getDatabaseAccessor(appid) + const accessor = + (await this.dedicatedDatabaseService.getDatabaseAccessor(appid)) || + (await this.dbService.getDatabaseAccessor(appid)) // Don't need policy rules, open all collections' access permission for dbm use. // Just create a empty policy for proxy. diff --git a/server/src/database/database.module.ts b/server/src/database/database.module.ts index 73cb008d6a..18e8df8313 100644 --- a/server/src/database/database.module.ts +++ b/server/src/database/database.module.ts @@ -14,14 +14,21 @@ import { DatabaseUsageLimitTaskService } from './database-usage-limit-task.servi import { DatabaseUsageCaptureTaskService } from './database-usage-capture-task.service' import { QuotaService } from 'src/user/quota.service' import { SettingService } from 'src/setting/setting.service' +import { DedicatedDatabaseService } from './dedicated-database/dedicated-database.service' +import { DedicatedDatabaseTaskService } from './dedicated-database/dedicated-database-task.service' +import { HttpModule } from '@nestjs/axios' +import { ApplicationListener } from './listeners/application.listener' +import { DedicatedDatabaseMonitorService } from './monitor/monitor.service' +import { DedicatedDatabaseMonitorController } from './monitor/monitor.controller' @Module({ - imports: [], + imports: [HttpModule], controllers: [ CollectionController, PolicyController, DatabaseController, PolicyRuleController, + DedicatedDatabaseMonitorController, ], providers: [ CollectionService, @@ -35,6 +42,10 @@ import { SettingService } from 'src/setting/setting.service' DatabaseUsageLimitTaskService, SettingService, QuotaService, + DedicatedDatabaseService, + DedicatedDatabaseTaskService, + DedicatedDatabaseMonitorService, + ApplicationListener, ], exports: [ CollectionService, @@ -42,6 +53,7 @@ import { SettingService } from 'src/setting/setting.service' DatabaseService, PolicyRuleService, MongoService, + DedicatedDatabaseService, ], }) export class DatabaseModule {} diff --git a/server/src/database/database.service.ts b/server/src/database/database.service.ts index b353128abe..c1a1625f38 100644 --- a/server/src/database/database.service.ts +++ b/server/src/database/database.service.ts @@ -130,7 +130,7 @@ export class DatabaseService { async findAndConnect(appid: string) { const region = await this.regionService.findByAppId(appid) const database = await this.findOne(appid) - assert(database, 'Database not found') + if (!database) return null const connectionUri = this.getControlConnectionUri(region, database) diff --git a/server/src/database/dedicated-database/dedicated-database-task.service.ts b/server/src/database/dedicated-database/dedicated-database-task.service.ts new file mode 100644 index 0000000000..9674203dff --- /dev/null +++ b/server/src/database/dedicated-database/dedicated-database-task.service.ts @@ -0,0 +1,340 @@ +import { Cron, CronExpression } from '@nestjs/schedule' +import { SystemDatabase } from 'src/system-database' +import { + DedicatedDatabase, + DedicatedDatabasePhase, + DedicatedDatabaseState, +} from '../entities/dedicated-database' +import { DedicatedDatabaseService } from './dedicated-database.service' +import { TASK_LOCK_INIT_TIME } from 'src/constants' +import { Injectable, Logger } from '@nestjs/common' +import { RegionService } from 'src/region/region.service' + +@Injectable() +export class DedicatedDatabaseTaskService { + private readonly logger = new Logger(DedicatedDatabaseTaskService.name) + private readonly lockTimeout = 15 // in seconds + private readonly db = SystemDatabase.db + + constructor( + private readonly regionService: RegionService, + private readonly dbService: DedicatedDatabaseService, + ) {} + + @Cron(CronExpression.EVERY_SECOND) + async tick() { + this.handleDeletingPhase().catch((err) => { + this.logger.error(err) + }) + this.handleStoppingPhase().catch((err) => { + this.logger.error(err) + }) + this.handleStartingPhase().catch((err) => { + this.logger.error(err) + }) + this.handleDeletedState().catch((err) => { + this.logger.error(err) + }) + this.handleStoppedState().catch((err) => { + this.logger.error(err) + }) + this.handleRestartingState().catch((err) => { + this.logger.error(err) + }) + this.handleRunningState().catch((err) => { + this.logger.error(err) + }) + } + + async handleStartingPhase() { + const res = await this.db + .collection('DedicatedDatabase') + .findOneAndUpdate( + { + phase: DedicatedDatabasePhase.Starting, + lockedAt: { $lt: new Date(Date.now() - this.lockTimeout * 1000) }, + }, + { $set: { lockedAt: new Date() } }, + { sort: { lockedAt: 1, updatedAt: 1 }, returnDocument: 'after' }, + ) + + if (!res.value) return + const data = res.value + const appid = data.appid + + const region = await this.regionService.findByAppId(appid) + let manifest = await this.dbService.getDeployManifest(region, appid) + + if (!manifest || manifest.spec.componentSpecs[0].replicas === 0) { + await this.dbService.applyDeployManifest(region, appid) + } + + // if waiting time is more than 5 minutes, stop + const waitingTime = Date.now() - data.updatedAt.getTime() + if (waitingTime > 1000 * 60 * 5) { + await this.db + .collection('DedicatedDatabase') + .updateOne( + { appid, phase: DedicatedDatabasePhase.Starting }, + { + $set: { + state: DedicatedDatabaseState.Stopped, + phase: DedicatedDatabasePhase.Stopping, + lockedAt: TASK_LOCK_INIT_TIME, + updatedAt: new Date(), + }, + }, + ) + + this.logger.log( + `dedicated database ${appid} updated to state Stopped due to timeout`, + ) + return + } + + manifest = await this.dbService.getDeployManifest(region, appid) + const unavailable = manifest?.status?.phase !== 'Running' + if (unavailable) { + await this.relock(appid, waitingTime) + return + } + + // if state is `Restarting`, update state to `Running` with phase `Started` + let toState = data.state + if (toState === DedicatedDatabaseState.Restarting) { + toState = DedicatedDatabaseState.Running + } + + await this.db.collection('DedicatedDatabase').updateOne( + { + appid, + phase: DedicatedDatabasePhase.Starting, + }, + { + $set: { + state: toState, + phase: DedicatedDatabasePhase.Started, + lockedAt: TASK_LOCK_INIT_TIME, + updatedAt: new Date(), + }, + }, + ) + + this.logger.debug(`dedicated database ${appid} updated to phase started`) + } + + async handleDeletingPhase() { + const res = await this.db + .collection('DedicatedDatabase') + .findOneAndUpdate( + { + phase: DedicatedDatabasePhase.Deleting, + lockedAt: { + $lt: new Date(Date.now() - this.lockTimeout * 1000), + }, + }, + { $set: { lockedAt: new Date() } }, + { sort: { lockedAt: 1, updatedAt: 1 }, returnDocument: 'after' }, + ) + + if (!res.value) return + const data = res.value + const appid = data.appid + + const waitingTime = Date.now() - data.updatedAt.getTime() + + const region = await this.regionService.findByAppId(appid) + const manifest = await this.dbService.getDeployManifest(region, appid) + + if (manifest) { + await this.dbService.deleteDeployManifest(region, appid) + await this.relock(appid, waitingTime) + } + + await this.db.collection('DedicatedDatabase').updateOne( + { + appid, + phase: DedicatedDatabasePhase.Deleting, + }, + { + $set: { + phase: DedicatedDatabasePhase.Deleted, + lockedAt: TASK_LOCK_INIT_TIME, + }, + }, + ) + } + + async handleStoppingPhase() { + const res = await this.db + .collection('DedicatedDatabase') + .findOneAndUpdate( + { + phase: DedicatedDatabasePhase.Stopping, + lockedAt: { + $lt: new Date(Date.now() - this.lockTimeout * 1000), + }, + }, + { + $set: { + lockedAt: new Date(), + }, + }, + ) + + if (!res.value) return + const data = res.value + const appid = data.appid + + const region = await this.regionService.findByAppId(appid) + const waitingTime = Date.now() - data.updatedAt.getTime() + + const manifest = await this.dbService.getDeployManifest(region, appid) + if (manifest && manifest.spec.componentSpecs[0].replicas !== 0) { + await this.dbService.applyDeployManifest(region, appid, { + replicas: 0, + }) + await this.relock(appid, waitingTime) + } + + await this.db.collection('DedicatedDatabase').updateOne( + { + appid: data.appid, + phase: DedicatedDatabasePhase.Stopping, + }, + { + $set: { + phase: DedicatedDatabasePhase.Stopped, + lockedAt: TASK_LOCK_INIT_TIME, + }, + }, + ) + + this.logger.log(`dedicated database ${appid} updated to phase Stopped`) + } + + async handleDeletedState() { + const db = SystemDatabase.db + + await db.collection('DedicatedDatabase').updateMany( + { + state: DedicatedDatabaseState.Deleted, + phase: { + $in: [DedicatedDatabasePhase.Started, DedicatedDatabasePhase.Stopped], + }, + }, + { + $set: { + phase: DedicatedDatabasePhase.Deleting, + lockedAt: TASK_LOCK_INIT_TIME, + }, + }, + ) + + await db.collection('DedicatedDatabase').deleteMany({ + state: DedicatedDatabaseState.Deleted, + phase: DedicatedDatabasePhase.Deleted, + }) + } + + async handleStoppedState() { + const db = SystemDatabase.db + + await db.collection('DedicatedDatabase').updateMany( + { + state: DedicatedDatabaseState.Stopped, + phase: { + $in: [DedicatedDatabasePhase.Started], + }, + }, + { + $set: { + phase: DedicatedDatabasePhase.Stopping, + lockedAt: TASK_LOCK_INIT_TIME, + }, + }, + ) + } + + async handleRunningState() { + const db = SystemDatabase.db + + await db.collection('DedicatedDatabase').updateMany( + { + state: DedicatedDatabaseState.Running, + phase: { + $in: [DedicatedDatabasePhase.Stopped], + }, + }, + { + $set: { + phase: DedicatedDatabasePhase.Starting, + lockedAt: TASK_LOCK_INIT_TIME, + }, + }, + ) + } + + async handleRestartingState() { + const db = SystemDatabase.db + + await db.collection('DedicatedDatabase').updateMany( + { + state: DedicatedDatabaseState.Restarting, + phase: DedicatedDatabasePhase.Started, + }, + { + $set: { + phase: DedicatedDatabasePhase.Stopping, + lockedAt: TASK_LOCK_INIT_TIME, + }, + }, + ) + + await db.collection('DedicatedDatabase').updateMany( + { + state: DedicatedDatabaseState.Restarting, + phase: DedicatedDatabasePhase.Stopped, + }, + { + $set: { + phase: DedicatedDatabasePhase.Starting, + lockedAt: TASK_LOCK_INIT_TIME, + }, + }, + ) + + await db.collection('DedicatedDatabase').deleteMany({ + state: DedicatedDatabaseState.Deleted, + phase: DedicatedDatabasePhase.Deleted, + }) + } + + /** + * Relock application by appid, lockedTime is in milliseconds + */ + async relock(appid: string, lockedTime = 0) { + if (lockedTime <= 2 * 60 * 1000) { + lockedTime = Math.ceil(lockedTime / 10) + } + + if (lockedTime > 2 * 60 * 1000) { + lockedTime = this.lockTimeout * 1000 + } + + const db = SystemDatabase.db + const lockedAt = new Date(Date.now() - 1000 * this.lockTimeout + lockedTime) + await db + .collection('DedicatedDatabase') + .updateOne({ appid: appid }, { $set: { lockedAt } }) + } + + private getHourTime() { + const latestTime = new Date() + latestTime.setMinutes(0) + latestTime.setSeconds(0) + latestTime.setMilliseconds(0) + latestTime.setHours(latestTime.getHours()) + return latestTime + } +} diff --git a/server/src/database/dedicated-database/dedicated-database.service.ts b/server/src/database/dedicated-database/dedicated-database.service.ts new file mode 100644 index 0000000000..88d95b92dd --- /dev/null +++ b/server/src/database/dedicated-database/dedicated-database.service.ts @@ -0,0 +1,252 @@ +import { Injectable } from '@nestjs/common' +import { Region } from 'src/region/entities/region' +import { + DedicatedDatabase, + DedicatedDatabasePhase, + DedicatedDatabaseSpec, + DedicatedDatabaseState, +} from '../entities/dedicated-database' +import { RegionService } from 'src/region/region.service' +import { ClusterService } from 'src/region/cluster/cluster.service' +import * as _ from 'lodash' +import { SystemDatabase } from 'src/system-database' +import { KubernetesObject, loadAllYaml } from '@kubernetes/client-node' +import { TASK_LOCK_INIT_TIME } from 'src/constants' +import { ClientSession } from 'mongodb' +import * as mongodb_uri from 'mongodb-uri' +import { MongoService } from 'src/database/mongo.service' +import { MongoAccessor } from 'database-proxy' +import { ApplicationBundle } from 'src/application/entities/application-bundle' +import * as assert from 'assert' + +const getDedicatedDatabaseName = (appid: string) => appid + +@Injectable() +export class DedicatedDatabaseService { + constructor( + private readonly cluster: ClusterService, + private readonly regionService: RegionService, + private readonly mongoService: MongoService, + ) {} + + async create(appid: string, session?: ClientSession) { + const db = SystemDatabase.db + + await db.collection('DedicatedDatabase').insertOne( + { + appid, + name: appid, + createdAt: new Date(), + updatedAt: new Date(), + lockedAt: TASK_LOCK_INIT_TIME, + phase: DedicatedDatabasePhase.Starting, + state: DedicatedDatabaseState.Running, + }, + { session }, + ) + } + + async applyDeployManifest( + region: Region, + appid: string, + patch?: Partial, + ) { + const spec = await this.getDedicatedDatabaseSpec(appid) + const manifest = this.makeDeployManifest(region, appid, { + ...spec, + ...patch, + }) + const res = await this.cluster.applyYamlString(region, manifest) + return res + } + + async getDedicatedDatabaseSpec( + appid: string, + ): Promise { + const db = SystemDatabase.db + + const bundle = await db + .collection('ApplicationBundle') + .findOne({ appid }) + + return bundle.resource.dedicatedDatabase + } + + async findOne(appid: string) { + const db = SystemDatabase.db + + const res = await db + .collection('DedicatedDatabase') + .findOne({ + appid, + }) + + return res + } + + async deleteDeployManifest(region: Region, appid: string) { + const manifest = await this.getDeployManifest(region, appid) + const res = await this.cluster.deleteCustomObject(region, manifest) + return res + } + + async getDeployManifest(region: Region, appid: string) { + const api = this.cluster.makeObjectApi(region) + const emptyManifest = this.makeDeployManifest(region, appid) + const specs = loadAllYaml(emptyManifest) + assert( + specs && specs.length > 0, + 'the deploy manifest of database should not be empty', + ) + const spec = specs[0] + + try { + const manifest = await api.read(spec) + return manifest.body as KubernetesObject & { spec: any; status: any } + } catch (err) { + return null + } + } + + makeDeployManifest( + region: Region, + appid: string, + dto?: DedicatedDatabaseSpec, + ) { + dto = dto || { + limitCPU: 0, + limitMemory: 0, + requestCPU: 0, + requestMemory: 0, + replicas: 0, + capacity: 0, + } + const { limitCPU, limitMemory, replicas, capacity } = dto + const name = getDedicatedDatabaseName(appid) + + const requestCPU = + limitCPU * (region.bundleConf?.cpuRequestLimitRatio || 0.1) + const requestMemory = + limitMemory * (region.bundleConf?.memoryRequestLimitRatio || 0.5) + + const template = region.deployManifest.database + const tmpl = _.template(template) + const manifest = tmpl({ + name, + limitCPU, + limitMemory, + requestCPU, + requestMemory, + capacity, + replicas, + }) + + return manifest + } + + async updateState(appid: string, state: DedicatedDatabaseState) { + const db = SystemDatabase.db + const res = await db + .collection('DedicatedDatabase') + .findOneAndUpdate( + { appid }, + { $set: { state, updatedAt: new Date() } }, + { returnDocument: 'after' }, + ) + + return res.value + } + + getDatabaseNamespace(region: Region, appid: string) { + const emptyManifest = this.makeDeployManifest(region, appid) + const specs = loadAllYaml(emptyManifest) + assert( + specs && specs.length > 0, + 'the deploy manifest of database should not be empty', + ) + if (!specs || specs.length === 0) return null + const spec = specs[0] + return spec.metadata.namespace + } + + async getConnectionUri(region: Region, database: DedicatedDatabase) { + const api = this.cluster.makeCoreV1Api(region) + const namespace = this.getDatabaseNamespace(region, database.appid) + const name = getDedicatedDatabaseName(database.appid) + const secretName = `${name}-conn-credential` + const srv = await api.readNamespacedSecret(secretName, namespace) + if (!srv) return null + + const username = Buffer.from(srv.body.data.username, 'base64').toString() + const password = Buffer.from(srv.body.data.password, 'base64').toString() + const host = Buffer.from(srv.body.data.headlessHost, 'base64').toString() + const port = Number( + Buffer.from(srv.body.data.headlessPort, 'base64').toString(), + ) + + const uri = mongodb_uri.format({ + username, + password, + hosts: [ + { + host, + port, + }, + ], + database: database.name, + options: { + authSource: 'admin', + }, + scheme: 'mongodb', + }) + + return uri + } + + async findAndConnect(appid: string) { + const database = await this.findOne(appid) + if (!database) return null + + const region = await this.regionService.findByAppId(appid) + const connectionUri = await this.getConnectionUri(region, database) + + const client = await this.mongoService.connectDatabase( + connectionUri, + database.name, + ) + const db = client.db(database.name) + return { db, client } + } + + /** + * Get database accessor that used for `database-proxy` + */ + async getDatabaseAccessor(appid: string) { + const database = await this.findOne(appid) + if (!database) return null + + const { client } = await this.findAndConnect(appid) + + const accessor = new MongoAccessor(client) + return accessor + } + + async remove(appid: string) { + const db = SystemDatabase.db + const doc = await db + .collection('DedicatedDatabase') + .findOneAndUpdate( + { appid }, + { + $set: { + state: DedicatedDatabaseState.Deleted, + phase: DedicatedDatabasePhase.Deleting, + updatedAt: new Date(), + }, + }, + { returnDocument: 'after' }, + ) + + return doc.value + } +} diff --git a/server/src/database/dto/create-dedicated-database.dto.ts b/server/src/database/dto/create-dedicated-database.dto.ts new file mode 100644 index 0000000000..2516161aba --- /dev/null +++ b/server/src/database/dto/create-dedicated-database.dto.ts @@ -0,0 +1,25 @@ +import { ApiProperty } from '@nestjs/swagger' +import { IsInt, IsNotEmpty, Max } from 'class-validator' + +export class CreateDedicatedDatabaseDto { + @ApiProperty({ example: 200 }) + @IsNotEmpty() + @IsInt() + cpu: number + + @ApiProperty({ example: 256 }) + @IsNotEmpty() + @IsInt() + memory: number + + @ApiProperty({ example: 1024 }) + @IsNotEmpty() + @IsInt() + capacity: number + + @ApiProperty({ example: 3 }) + @IsNotEmpty() + @IsInt() + @Max(9) + replicas: number +} diff --git a/server/src/database/dto/update-dedicated-database-state.dto.ts b/server/src/database/dto/update-dedicated-database-state.dto.ts new file mode 100644 index 0000000000..e9a88e5318 --- /dev/null +++ b/server/src/database/dto/update-dedicated-database-state.dto.ts @@ -0,0 +1,10 @@ +import { IsEnum, IsNotEmpty } from 'class-validator' +import { DedicatedDatabaseState } from '../entities/dedicated-database' +import { ApiProperty } from '@nestjs/swagger' + +export class UpdateDedicatedDatabaseStateDto { + @ApiProperty({ enum: DedicatedDatabaseState }) + @IsEnum(DedicatedDatabaseState) + @IsNotEmpty() + state: DedicatedDatabaseState +} diff --git a/server/src/database/dto/update-dedicated-database.dto.ts b/server/src/database/dto/update-dedicated-database.dto.ts new file mode 100644 index 0000000000..b483d150ef --- /dev/null +++ b/server/src/database/dto/update-dedicated-database.dto.ts @@ -0,0 +1,3 @@ +import { CreateDedicatedDatabaseDto } from './create-dedicated-database.dto' + +export class UpdateDedicatedDatabaseDto extends CreateDedicatedDatabaseDto {} diff --git a/server/src/database/entities/dedicated-database.ts b/server/src/database/entities/dedicated-database.ts new file mode 100644 index 0000000000..6989601b78 --- /dev/null +++ b/server/src/database/entities/dedicated-database.ts @@ -0,0 +1,61 @@ +import { ApiProperty } from '@nestjs/swagger' +import { ObjectId } from 'mongodb' + +export enum DedicatedDatabasePhase { + Starting = 'Starting', + Started = 'Started', + Stopping = 'Stopping', + Stopped = 'Stopped', + Deleting = 'Deleting', + Deleted = 'Deleted', +} + +export enum DedicatedDatabaseState { + Running = 'Running', + Stopped = 'Stopped', + Restarting = 'Restarting', + Deleted = 'Deleted', +} + +export class DedicatedDatabaseSpec { + @ApiProperty({ example: 500 }) + limitCPU: number + + @ApiProperty({ example: 1024 }) + limitMemory: number + + requestCPU: number + + requestMemory: number + + @ApiProperty({ example: 1024 }) + capacity: number + + @ApiProperty({ example: 1 }) + replicas: number +} + +export class DedicatedDatabase { + @ApiProperty({ type: String }) + _id?: ObjectId + + @ApiProperty() + appid: string + + @ApiProperty() + name: string + + @ApiProperty({ enum: DedicatedDatabaseState }) + state: DedicatedDatabaseState + + @ApiProperty({ enum: DedicatedDatabasePhase }) + phase: DedicatedDatabasePhase + + lockedAt: Date + + @ApiProperty() + createdAt: Date + + @ApiProperty() + updatedAt: Date +} diff --git a/server/src/database/listeners/application.listener.ts b/server/src/database/listeners/application.listener.ts new file mode 100644 index 0000000000..efbd40bf1d --- /dev/null +++ b/server/src/database/listeners/application.listener.ts @@ -0,0 +1,21 @@ +import { Injectable } from '@nestjs/common' +import { OnEvent } from '@nestjs/event-emitter' +import { DedicatedDatabaseService } from '../dedicated-database/dedicated-database.service' +import { ApplicationCreatingEvent } from 'src/application/events/application-creating.event' + +@Injectable() +export class ApplicationListener { + constructor( + private readonly dedicatedDatabaseService: DedicatedDatabaseService, + ) {} + + @OnEvent(ApplicationCreatingEvent.eventName, { + promisify: true, + async: true, + }) + handleApplicationCreatedEvent(event: ApplicationCreatingEvent) { + if (event.dto.dedicatedDatabase) { + return this.dedicatedDatabaseService.create(event.appid, event.session) + } + } +} diff --git a/server/src/database/monitor/monitor.controller.ts b/server/src/database/monitor/monitor.controller.ts new file mode 100644 index 0000000000..b25b0e5bec --- /dev/null +++ b/server/src/database/monitor/monitor.controller.ts @@ -0,0 +1,56 @@ +import { Controller, Get, UseGuards } from '@nestjs/common' +import { DedicatedDatabaseMonitorService } from './monitor.service' +import { InjectApplication } from 'src/utils/decorator' +import { ApplicationWithRelations } from 'src/application/entities/application' +import { RegionService } from 'src/region/region.service' +import { JwtAuthGuard } from 'src/authentication/jwt.auth.guard' +import { ApplicationAuthGuard } from 'src/authentication/application.auth.guard' +import { + ApiBearerAuth, + ApiOperation, + ApiResponse, + ApiTags, +} from '@nestjs/swagger' +import { ResponseUtil } from 'src/utils/response' + +@ApiTags('Database') +@ApiBearerAuth('Authorization') +@Controller('apps/:appid/dedicated-database/monitor') +export class DedicatedDatabaseMonitorController { + constructor( + private readonly region: RegionService, + private readonly monitor: DedicatedDatabaseMonitorService, + ) {} + + @ApiOperation({ + summary: 'Get dedicated database resources metrics data', + }) + @ApiResponse({ type: ResponseUtil }) + @UseGuards(JwtAuthGuard, ApplicationAuthGuard) + @Get('resource') + async getResource(@InjectApplication() app: ApplicationWithRelations) { + const region = await this.region.findOne(app.regionId) + const res = await this.monitor.getResource(app.appid, region) + return ResponseUtil.ok(res) + } + + @ApiOperation({ summary: 'Get dedicated database connections metrics data' }) + @ApiResponse({ type: ResponseUtil }) + @UseGuards(JwtAuthGuard, ApplicationAuthGuard) + @Get('connection') + async getConnection(@InjectApplication() app: ApplicationWithRelations) { + const region = await this.region.findOne(app.regionId) + const res = await this.monitor.getConnection(app.appid, region) + return ResponseUtil.ok(res) + } + + @ApiOperation({ summary: 'Get dedicated database performance metrics data' }) + @ApiResponse({ type: ResponseUtil }) + @UseGuards(JwtAuthGuard, ApplicationAuthGuard) + @Get('performance') + async getPerformance(@InjectApplication() app: ApplicationWithRelations) { + const region = await this.region.findOne(app.regionId) + const res = await this.monitor.getPerformance(app.appid, region) + return ResponseUtil.ok(res) + } +} diff --git a/server/src/database/monitor/monitor.service.ts b/server/src/database/monitor/monitor.service.ts new file mode 100644 index 0000000000..f1948dc1ab --- /dev/null +++ b/server/src/database/monitor/monitor.service.ts @@ -0,0 +1,168 @@ +import { HttpService } from '@nestjs/axios' +import { Injectable, Logger } from '@nestjs/common' +import { Region } from 'src/region/entities/region' + +const requestConfig = { + retryAttempts: 5, + retryDelayBase: 300, + rateAccuracy: '1m', +} + +@Injectable() +export class DedicatedDatabaseMonitorService { + private readonly logger = new Logger(DedicatedDatabaseMonitorService.name) + + constructor(private readonly httpService: HttpService) {} + + async getResource(appid: string, region: Region) { + const dbName = this.getDBName(appid) + + const cpu = await this.queryRange( + region, + `laf_mongo_cpu{appid="${appid}"}`, + { + labels: ['pod'], + }, + ) + const memory = await this.queryRange( + region, + `laf_mongo_memory{appid="${appid}"}`, + { + labels: ['pod'], + }, + ) + const dataSize = await this.query( + region, + `sum(mongodb_dbstats_dataSize{pod=~"${dbName}-mongo.+"}) by (database)`, + { + labels: ['database'], + }, + ) + + return { + cpu, + memory, + dataSize, + } + } + + async getConnection(appid: string, region: Region) { + const dbName = this.getDBName(appid) + const query = `mongodb_connections{pod=~"${dbName}-mongo.+",state="current"}` + const connections = await this.queryRange(region, query, { + labels: ['pod'], + }) + return { + connections, + } + } + async getPerformance(appid: string, region: Region) { + const dbName = this.getDBName(appid) + const queries = { + documentOperations: `rate(mongodb_mongod_metrics_document_total{pod=~"${dbName}-mongo.+"}[1m])`, + queryOperations: `rate(mongodb_op_counters_total{pod=~"${dbName}-mongo.+"}[5m]) or irate(mongodb_op_counters_total{pod=~"${dbName}-mongo.+"}[5m])`, + pageFaults: `rate(mongodb_extra_info_page_faults_total{pod=~"${dbName}-mongo.+"}[5m]) or irate(mongodb_extra_info_page_faults_total{pod=~"${dbName}-mongo.+"}[5m])`, + } + + const res = await Promise.all( + Object.keys(queries).map(async (key) => { + const query = queries[key] + const data = await this.queryRange(region, query, { + labels: ['pod', 'type', 'state'], + }) + return data + }), + ) + + const keys = Object.keys(queries) + return res.reduce((acc, cur, idx) => { + acc[keys[idx]] = cur + return acc + }, {}) + } + + getDBName(appid: string) { + return `${appid}` + } + + private async query( + region: Region, + query: string, + queryParams?: Record, + ) { + const host = region.prometheusConf?.apiUrl + if (!host) return [] + const endpoint = `${host}/api/v1/query` + + return await this.queryInternal(endpoint, { query, ...queryParams }) + } + + private async queryRange( + region: Region, + query: string, + queryParams?: Record, + ) { + const host = region.prometheusConf?.apiUrl + if (!host) return [] + + const range = 3600 // 1 hour + const now = Math.floor(Date.now() / 1000) + const start = now - range + const end = now + + queryParams = { + range, + step: 60, + start, + end, + ...queryParams, + } + + const endpoint = `${host}/api/v1/query_range` + + return await this.queryInternal(endpoint, { + query, + ...queryParams, + }) + } + + private async queryInternal( + endpoint: string, + query: Record, + ) { + const labels = query.labels + delete query['labels'] + for (let attempt = 1; attempt <= requestConfig.retryAttempts; attempt++) { + try { + const res = await this.httpService + .post(endpoint, query, { + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + }, + }) + .toPromise() + + if (!labels || !Array.isArray(labels)) return res.data.data.result + + return res.data.data.result.map((v) => { + const metric = v.metric + for (const item in metric) { + if (!labels.includes(item)) { + delete metric[item] + } + } + return v + }) + } catch (error) { + if (attempt >= requestConfig.retryAttempts) { + this.logger.error('Metrics not available', error.message) + return [] + } + + await new Promise((resolve) => + setTimeout(resolve, attempt * requestConfig.retryDelayBase), + ) + } + } + } +} diff --git a/server/src/database/policy/policy.service.ts b/server/src/database/policy/policy.service.ts index d09a059e67..83c7d805fa 100644 --- a/server/src/database/policy/policy.service.ts +++ b/server/src/database/policy/policy.service.ts @@ -10,13 +10,17 @@ import { DatabasePolicyWithRules, } from '../entities/database-policy' import * as assert from 'assert' +import { DedicatedDatabaseService } from '../dedicated-database/dedicated-database.service' @Injectable() export class PolicyService { private readonly logger = new Logger(PolicyService.name) private readonly db = SystemDatabase.db - constructor(private readonly databaseService: DatabaseService) {} + constructor( + private readonly databaseService: DatabaseService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, + ) {} async create(appid: string, dto: CreatePolicyDto) { await this.db.collection('DatabasePolicy').insertOne({ @@ -141,9 +145,10 @@ export class PolicyService { const policy = await this.findOne(appid, name) assert(policy, `policy ${name} not found`) - const { db, client } = await this.databaseService.findAndConnect( - policy.appid, - ) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(appid)) || + (await this.databaseService.findAndConnect(appid)) + const session = client.startSession() const rules = {} @@ -166,7 +171,9 @@ export class PolicyService { } async unpublish(appid: string, name: string) { - const { db, client } = await this.databaseService.findAndConnect(appid) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(appid)) || + (await this.databaseService.findAndConnect(appid)) try { const coll = db.collection(CN_PUBLISHED_POLICIES) await coll.deleteOne({ name }) diff --git a/server/src/function/function.service.ts b/server/src/function/function.service.ts index f504420fd9..5213416bbc 100644 --- a/server/src/function/function.service.ts +++ b/server/src/function/function.service.ts @@ -24,6 +24,7 @@ import { HttpService } from '@nestjs/axios' import { RegionService } from 'src/region/region.service' import { GetApplicationNamespace } from 'src/utils/getter' import { Region } from 'src/region/entities/region' +import { DedicatedDatabaseService } from 'src/database/dedicated-database/dedicated-database.service' @Injectable() export class FunctionService { @@ -32,6 +33,7 @@ export class FunctionService { constructor( private readonly databaseService: DatabaseService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, private readonly jwtService: JwtService, private readonly triggerService: TriggerService, private readonly functionRecycleBinService: FunctionRecycleBinService, @@ -234,7 +236,10 @@ export class FunctionService { } async publish(func: CloudFunction, oldFuncName?: string) { - const { db, client } = await this.databaseService.findAndConnect(func.appid) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(func.appid)) || + (await this.databaseService.findAndConnect(func.appid)) + const session = client.startSession() try { await session.withTransaction(async () => { @@ -252,9 +257,10 @@ export class FunctionService { } async publishMany(funcs: CloudFunction[]) { - const { db, client } = await this.databaseService.findAndConnect( - funcs[0].appid, - ) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(funcs[0].appid)) || + (await this.databaseService.findAndConnect(funcs[0].appid)) + const session = client.startSession() try { await session.withTransaction(async () => { @@ -270,9 +276,10 @@ export class FunctionService { } async publishFunctionTemplateItems(funcs: CloudFunction[]) { - const { db, client } = await this.databaseService.findAndConnect( - funcs[0].appid, - ) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(funcs[0].appid)) || + (await this.databaseService.findAndConnect(funcs[0].appid)) + const session = client.startSession() try { await session.withTransaction(async () => { @@ -287,7 +294,9 @@ export class FunctionService { } async unpublish(appid: string, name: string) { - const { db, client } = await this.databaseService.findAndConnect(appid) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(appid)) || + (await this.databaseService.findAndConnect(appid)) try { const coll = db.collection(CN_PUBLISHED_FUNCTIONS) await coll.deleteOne({ name }) diff --git a/server/src/gateway/website-task.service.ts b/server/src/gateway/website-task.service.ts index fdf10f05d7..6eae366865 100644 --- a/server/src/gateway/website-task.service.ts +++ b/server/src/gateway/website-task.service.ts @@ -16,6 +16,7 @@ import { DomainPhase, DomainState } from './entities/runtime-domain' import { BucketDomain } from './entities/bucket-domain' import { WebsiteHostingGatewayService } from './ingress/website-ingress.service' import { DatabaseService } from 'src/database/database.service' +import { DedicatedDatabaseService } from 'src/database/dedicated-database/dedicated-database.service' @Injectable() export class WebsiteTaskService { @@ -27,6 +28,7 @@ export class WebsiteTaskService { private readonly websiteGateway: WebsiteHostingGatewayService, private readonly certService: CertificateService, private readonly databaseService: DatabaseService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, ) {} @Cron(CronExpression.EVERY_SECOND) @@ -304,9 +306,9 @@ export class WebsiteTaskService { } async publish(website: WebsiteHosting) { - const { db, client } = await this.databaseService.findAndConnect( - website.appid, - ) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(website.appid)) || + (await this.databaseService.findAndConnect(website.appid)) const session = client.startSession() try { @@ -323,9 +325,9 @@ export class WebsiteTaskService { } async unpublish(website: WebsiteHosting) { - const { db, client } = await this.databaseService.findAndConnect( - website.appid, - ) + const { db, client } = + (await this.dedicatedDatabaseService.findAndConnect(website.appid)) || + (await this.databaseService.findAndConnect(website.appid)) try { const coll = db.collection(CN_PUBLISHED_WEBSITE_HOSTING) diff --git a/server/src/initializer/initializer.service.ts b/server/src/initializer/initializer.service.ts index 4c237e94cc..a5350d0851 100644 --- a/server/src/initializer/initializer.service.ts +++ b/server/src/initializer/initializer.service.ts @@ -94,6 +94,9 @@ export class InitializerService { prometheusConf: { apiUrl: ServerConfig.DEFAULT_REGION_PROMETHEUS_URL, }, + deployManifest: JSON.parse( + ServerConfig.DEFAULT_REGION_DEPLOY_MANIFEST || '{}', + ), updatedAt: new Date(), createdAt: new Date(), state: 'Active', @@ -256,6 +259,61 @@ export class InitializerService { createdAt: new Date(), updatedAt: new Date(), }, + { + regionId: region._id, + type: ResourceType.DedicatedDatabaseCPU, + price: 0.0, + specs: [ + { label: '0.2 Core', value: 200 }, + { label: '0.5 Core', value: 500 }, + { label: '1 Core', value: 1000 }, + { label: '2 Core', value: 2000 }, + ], + createdAt: new Date(), + updatedAt: new Date(), + }, + { + regionId: region._id, + type: ResourceType.DedicatedDatabaseMemory, + price: 0.0, + specs: [ + { label: '256 MB', value: 256 }, + { label: '512 MB', value: 512 }, + { label: '1 GB', value: 1024 }, + { label: '2 GB', value: 2048 }, + { label: '4 GB', value: 4096 }, + ], + createdAt: new Date(), + updatedAt: new Date(), + }, + { + regionId: region._id, + type: ResourceType.DedicatedDatabaseCapacity, + price: 0.0, + specs: [ + { label: '1 GB', value: 1024 }, + { label: '4 GB', value: 4096 }, + { label: '16 GB', value: 16384 }, + { label: '64 GB', value: 65536 }, + { label: '256 GB', value: 262144 }, + ], + createdAt: new Date(), + updatedAt: new Date(), + }, + { + regionId: region._id, + type: ResourceType.DedicatedDatabaseReplicas, + price: 0.0, + specs: [ + { label: '1', value: 1 }, + { label: '3', value: 3 }, + { label: '5', value: 5 }, + { label: '7', value: 7 }, + { label: '9', value: 9 }, + ], + createdAt: new Date(), + updatedAt: new Date(), + }, ]) this.logger.verbose('Created default resource options') @@ -287,6 +345,10 @@ export class InitializerService { [ResourceType.DatabaseCapacity]: { value: 1024 }, [ResourceType.StorageCapacity]: { value: 1024 }, [ResourceType.NetworkTraffic]: { value: 0 }, + [ResourceType.DedicatedDatabaseCPU]: { value: 200 }, + [ResourceType.DedicatedDatabaseMemory]: { value: 256 }, + [ResourceType.DedicatedDatabaseCapacity]: { value: 1024 }, + [ResourceType.DedicatedDatabaseReplicas]: { value: 1 }, }, enableFreeTier: false, limitCountOfFreeTierPerUser: 20, @@ -303,6 +365,10 @@ export class InitializerService { [ResourceType.DatabaseCapacity]: { value: 4096 }, [ResourceType.StorageCapacity]: { value: 4096 }, [ResourceType.NetworkTraffic]: { value: 0 }, + [ResourceType.DedicatedDatabaseCPU]: { value: 500 }, + [ResourceType.DedicatedDatabaseMemory]: { value: 512 }, + [ResourceType.DedicatedDatabaseCapacity]: { value: 4096 }, + [ResourceType.DedicatedDatabaseReplicas]: { value: 3 }, }, enableFreeTier: false, createdAt: new Date(), @@ -318,6 +384,10 @@ export class InitializerService { [ResourceType.DatabaseCapacity]: { value: 16384 }, [ResourceType.StorageCapacity]: { value: 65536 }, [ResourceType.NetworkTraffic]: { value: 0 }, + [ResourceType.DedicatedDatabaseCPU]: { value: 1000 }, + [ResourceType.DedicatedDatabaseMemory]: { value: 2048 }, + [ResourceType.DedicatedDatabaseCapacity]: { value: 16384 }, + [ResourceType.DedicatedDatabaseReplicas]: { value: 5 }, }, enableFreeTier: false, createdAt: new Date(), @@ -333,6 +403,10 @@ export class InitializerService { [ResourceType.DatabaseCapacity]: { value: 65536 }, [ResourceType.StorageCapacity]: { value: 262144 }, [ResourceType.NetworkTraffic]: { value: 0 }, + [ResourceType.DedicatedDatabaseCPU]: { value: 2000 }, + [ResourceType.DedicatedDatabaseMemory]: { value: 4096 }, + [ResourceType.DedicatedDatabaseCapacity]: { value: 65536 }, + [ResourceType.DedicatedDatabaseReplicas]: { value: 7 }, }, enableFreeTier: false, createdAt: new Date(), diff --git a/server/src/instance/instance-task.service.ts b/server/src/instance/instance-task.service.ts index 1cf1aca852..2d2820887a 100644 --- a/server/src/instance/instance-task.service.ts +++ b/server/src/instance/instance-task.service.ts @@ -13,13 +13,21 @@ import { DomainState, RuntimeDomain } from 'src/gateway/entities/runtime-domain' import { BucketDomain } from 'src/gateway/entities/bucket-domain' import { WebsiteHosting } from 'src/website/entities/website' import { CronTrigger, TriggerState } from 'src/trigger/entities/cron-trigger' +import { DedicatedDatabaseService } from 'src/database/dedicated-database/dedicated-database.service' +import { + DedicatedDatabasePhase, + DedicatedDatabaseState, +} from 'src/database/entities/dedicated-database' @Injectable() export class InstanceTaskService { readonly lockTimeout = 15 // in second private readonly logger = new Logger(InstanceTaskService.name) - constructor(private readonly instanceService: InstanceService) {} + constructor( + private readonly instanceService: InstanceService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, + ) {} @Cron(CronExpression.EVERY_SECOND) async tick() { @@ -103,9 +111,6 @@ export class InstanceTaskService { if (!res.value) return const app = res.value - // create instance - await this.instanceService.create(app.appid) - // if waiting time is more than 5 minutes, stop the application const waitingTime = Date.now() - app.updatedAt.getTime() if (waitingTime > 1000 * 60 * 5) { @@ -125,7 +130,28 @@ export class InstanceTaskService { return } + // create instance + await this.instanceService.create(app.appid) + const appid = app.appid + + const ddb = await this.dedicatedDatabaseService.findOne(appid) + if (ddb) { + if (ddb.state === DedicatedDatabaseState.Stopped) { + await this.dedicatedDatabaseService.updateState( + appid, + DedicatedDatabaseState.Running, + ) + await this.relock(appid, waitingTime) + return + } + + if (ddb.phase !== DedicatedDatabasePhase.Started) { + await this.relock(appid, waitingTime) + return + } + } + const instance = await this.instanceService.get(appid) const unavailable = instance.deployment?.status?.unavailableReplicas || false @@ -283,6 +309,16 @@ export class InstanceTaskService { { $set: { state: TriggerState.Inactive, updatedAt: new Date() } }, ) + const ddb = await this.dedicatedDatabaseService.findOne(appid) + if (ddb && ddb.state !== DedicatedDatabaseState.Stopped) { + await this.dedicatedDatabaseService.updateState( + appid, + DedicatedDatabaseState.Stopped, + ) + await this.relock(appid, waitingTime) + return + } + // check if the instance is removed const instance = await this.instanceService.get(app.appid) if (instance.deployment) { @@ -337,6 +373,11 @@ export class InstanceTaskService { if (!res.value) return const app = res.value + await this.dedicatedDatabaseService.updateState( + app.appid, + DedicatedDatabaseState.Restarting, + ) + await this.instanceService.restart(app.appid) // update application phase to `Starting` diff --git a/server/src/instance/instance.service.ts b/server/src/instance/instance.service.ts index 3a1b137ae5..c03024ef5c 100644 --- a/server/src/instance/instance.service.ts +++ b/server/src/instance/instance.service.ts @@ -15,6 +15,7 @@ import { ApplicationWithRelations } from 'src/application/entities/application' import { ApplicationService } from 'src/application/application.service' import * as assert from 'assert' import { CloudBinBucketService } from 'src/storage/cloud-bin-bucket.service' +import { DedicatedDatabaseService } from 'src/database/dedicated-database/dedicated-database.service' @Injectable() export class InstanceService { @@ -24,6 +25,7 @@ export class InstanceService { private readonly cluster: ClusterService, private readonly storageService: StorageService, private readonly databaseService: DatabaseService, + private readonly dedicatedDatabaseService: DedicatedDatabaseService, private readonly applicationService: ApplicationService, private readonly cloudbin: CloudBinBucketService, ) {} @@ -244,11 +246,20 @@ export class InstanceService { const npm_install_flags = region.clusterConf.npmInstallFlags || '' // db connection uri - const database = await this.databaseService.findOne(appid) - const dbConnectionUri = this.databaseService.getInternalConnectionUri( - region, - database, - ) + let dbConnectionUri: string + const dedicatedDatabase = await this.dedicatedDatabaseService.findOne(appid) + if (dedicatedDatabase) { + dbConnectionUri = await this.dedicatedDatabaseService.getConnectionUri( + region, + dedicatedDatabase, + ) + } else { + const database = await this.databaseService.findOne(appid) + dbConnectionUri = this.databaseService.getInternalConnectionUri( + region, + database, + ) + } const storage = await this.storageService.findOne(appid) const NODE_MODULES_PUSH_URL = diff --git a/server/src/recycle-bin/recycle-bin.module.ts b/server/src/recycle-bin/recycle-bin.module.ts index 236632bcfb..3056925983 100644 --- a/server/src/recycle-bin/recycle-bin.module.ts +++ b/server/src/recycle-bin/recycle-bin.module.ts @@ -9,6 +9,7 @@ import { MongoService } from 'src/database/mongo.service' import { RegionService } from 'src/region/region.service' import { ApplicationService } from 'src/application/application.service' import { HttpModule } from '@nestjs/axios' +import { DedicatedDatabaseService } from 'src/database/dedicated-database/dedicated-database.service' @Module({ imports: [HttpModule], @@ -16,6 +17,7 @@ import { HttpModule } from '@nestjs/axios' providers: [ ApplicationService, DatabaseService, + DedicatedDatabaseService, JwtService, TriggerService, FunctionRecycleBinService, diff --git a/server/src/region/cluster/cluster.service.ts b/server/src/region/cluster/cluster.service.ts index e4433e7e41..09b539aa13 100644 --- a/server/src/region/cluster/cluster.service.ts +++ b/server/src/region/cluster/cluster.service.ts @@ -88,6 +88,75 @@ export class ClusterService { } } + async applyYamlString(region: Region, specString: string) { + const api = this.makeObjectApi(region) + const specs: KubernetesObject[] = k8s.loadAllYaml(specString) + const validSpecs = specs.filter((s) => s && s.kind && s.metadata) + const created: k8s.KubernetesObject[] = [] + + for (const spec of validSpecs) { + spec.metadata = spec.metadata || {} + spec.metadata.annotations = spec.metadata.annotations || {} + delete spec.metadata.annotations[ + 'kubectl.kubernetes.io/last-applied-configuration' + ] + spec.metadata.annotations[ + 'kubectl.kubernetes.io/last-applied-configuration' + ] = JSON.stringify(spec) + + try { + // try to get the resource, if it does not exist an error will be thrown and we will end up in the catch + // block. + await api.read(spec as any) + // we got the resource, so it exists, so patch it + // + // Note that this could fail if the spec refers to a custom resource. For custom resources you may need + // to specify a different patch merge strategy in the content-type header. + // + // See: https://github.com/kubernetes/kubernetes/issues/97423 + const response = await api.patch( + spec, + undefined, + undefined, + undefined, + undefined, + { + headers: { + 'Content-Type': k8s.PatchUtils.PATCH_FORMAT_JSON_MERGE_PATCH, + }, + }, + ) + created.push(response.body) + } catch (e) { + // not exist, create + const response = await api.create(spec) + created.push(response.body) + } + } + return created + } + + async deleteYamlString(region: Region, specString: string) { + const api = this.makeObjectApi(region) + const specs: k8s.KubernetesObject[] = k8s.loadAllYaml(specString) + const validSpecs = specs.filter((s) => s && s.kind && s.metadata) + const deleted: k8s.KubernetesObject[] = [] + + for (const spec of validSpecs) { + try { + // try to get the resource, if it does not exist an error will be thrown and we will end up in the catch + // block. + await api.read(spec as any) + // we got the resource, so it exists, so delete it + const response = await api.delete(spec) + deleted.push(response.body) + } catch (e) { + // not exist + } + } + return deleted + } + async patchCustomObject(region: Region, spec: KubernetesObject) { const client = this.makeCustomObjectApi(region) const gvk = GroupVersionKind.fromKubernetesObject(spec) diff --git a/server/src/region/entities/region.ts b/server/src/region/entities/region.ts index 231fda532f..45a8ac1e1e 100644 --- a/server/src/region/entities/region.ts +++ b/server/src/region/entities/region.ts @@ -67,6 +67,10 @@ export type PrometheusConf = { apiUrl: string } +export type DeployManifest = { + [key: string]: string +} + export class Region { @ApiProperty({ type: String }) _id?: ObjectId @@ -86,6 +90,8 @@ export class Region { logServerConf: LogServerConf prometheusConf: PrometheusConf + deployManifest: DeployManifest + @ApiProperty() state: 'Active' | 'Inactive' diff --git a/server/src/trigger/trigger.module.ts b/server/src/trigger/trigger.module.ts index 795d8b49ca..0b2e247572 100644 --- a/server/src/trigger/trigger.module.ts +++ b/server/src/trigger/trigger.module.ts @@ -12,6 +12,7 @@ import { DatabaseService } from 'src/database/database.service' import { MongoService } from 'src/database/mongo.service' import { BundleService } from 'src/application/bundle.service' import { FunctionRecycleBinService } from 'src/recycle-bin/cloud-function/function-recycle-bin.service' +import { DedicatedDatabaseService } from 'src/database/dedicated-database/dedicated-database.service' @Module({ imports: [StorageModule, HttpModule], @@ -25,6 +26,7 @@ import { FunctionRecycleBinService } from 'src/recycle-bin/cloud-function/functi TriggerTaskService, FunctionService, DatabaseService, + DedicatedDatabaseService, MongoService, BundleService, ],