From aa19f45388ceecbb03f1966fb8529df11594a246 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 10 Jan 2024 14:24:22 +0100 Subject: [PATCH 1/4] Add nginx generic templates --- deployments/nginx/README.md | 27 +++++++++++ deployments/nginx/ingress.yaml | 68 +++++++++++++++++++++++++++ deployments/nginx/issuer.yaml | 40 ++++++++++++++++ deployments/nginx/values.yaml | 16 +++++++ docker-compose-elasticsearch-huge.yml | 7 ++- docker-compose-elasticsearch.yml | 3 +- docker-compose-frontend-dev.yml | 2 +- 7 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 deployments/nginx/README.md create mode 100644 deployments/nginx/ingress.yaml create mode 100644 deployments/nginx/issuer.yaml create mode 100644 deployments/nginx/values.yaml diff --git a/deployments/nginx/README.md b/deployments/nginx/README.md new file mode 100644 index 00000000..210540bc --- /dev/null +++ b/deployments/nginx/README.md @@ -0,0 +1,27 @@ +# Install helm charts + +## Nginx +```bash +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +``` + +## Cert manager +```bash +helm repo add jetstack https://charts.jetstack.io +helm repo update +``` +From [docs](https://cert-manager.io/docs/tutorials/acme/nginx-ingress/) + +# Deploy +```bash +helm install \ + cert-manager jetstack/cert-manager \ + --namespace nginx \ + --set installCRDs=true + +helm upgrade --install nginx ingress-nginx/ingress-nginx --namespace nginx --create-namespace --values values.yaml + +kubectl apply -f issuer.yaml +kubectl apply -f ingress.yaml +``` diff --git a/deployments/nginx/ingress.yaml b/deployments/nginx/ingress.yaml new file mode 100644 index 00000000..ef047f65 --- /dev/null +++ b/deployments/nginx/ingress.yaml @@ -0,0 +1,68 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: backend + namespace: ridoc # Namespace must be the same as that of target services below. + annotations: + # nginx.ingress.kubernetes.io/ssl-redirect: "false" # Set to true once SSL is set up. + cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$1 +spec: + ingressClassName: nginx + tls: + - hosts: + - kubernetes.ridoc.fr + secretName: letsencrypt-prod + rules: + - host: kubernetes.ridoc.fr + http: + paths: + - path: /backend/(.*) + pathType: ImplementationSpecific + backend: + service: + name: backend + port: + number: 5000 + # - path: / + # pathType: Prefix + # backend: + # service: + # name: frontend + # port: + # number: 3000 + +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: frontend + namespace: ridoc # Namespace must be the same as that of target services below. + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + # nginx.ingress.kubernetes.io/ssl-redirect: "false" # Set to true once SSL is set up. +spec: + ingressClassName: nginx + tls: + - hosts: + - kubernetes.ridoc.fr + secretName: letsencrypt-prod + rules: + - host: kubernetes.ridoc.fr + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: frontend + port: + number: 3000 + - path: /kibana + pathType: Prefix + backend: + service: + name: kibana-kibana + port: + number: 5601 diff --git a/deployments/nginx/issuer.yaml b/deployments/nginx/issuer.yaml new file mode 100644 index 00000000..55dd8d70 --- /dev/null +++ b/deployments/nginx/issuer.yaml @@ -0,0 +1,40 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-staging + # namespace: nginx +spec: + acme: + # The ACME server URL + server: https://acme-staging-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: datalab@interieur.gouv.fr + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-staging + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + ingressClassName: nginx + +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: letsencrypt-prod + # namespace: nginx +spec: + acme: + # The ACME server URL + server: https://acme-v02.api.letsencrypt.org/directory + # Email address used for ACME registration + email: datalab@interieur.gouv.fr + # Name of a secret used to store the ACME account private key + privateKeySecretRef: + name: letsencrypt-prod + # Enable the HTTP-01 challenge provider + solvers: + - http01: + ingress: + ingressClassName: nginx \ No newline at end of file diff --git a/deployments/nginx/values.yaml b/deployments/nginx/values.yaml new file mode 100644 index 00000000..f31432ea --- /dev/null +++ b/deployments/nginx/values.yaml @@ -0,0 +1,16 @@ +# controller: +# service: +# type: NodePort +# publishService: +# enabled: true + +controller: + kind: DaemonSet + hostNetwork: true + dnsPolicy: ClusterFirstWithHostNet + daemonset: + useHostPort: true + service: + type: ClusterIP +rbac: + create: true diff --git a/docker-compose-elasticsearch-huge.yml b/docker-compose-elasticsearch-huge.yml index 2d7661c1..3e84f133 100644 --- a/docker-compose-elasticsearch-huge.yml +++ b/docker-compose-elasticsearch-huge.yml @@ -16,13 +16,13 @@ services: - cluster.name=matchid-cluster - bootstrap.memory_lock=true - TAKE_FILE_OWNERSHIP=true - - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" - discovery.type=single-node ulimits: memlock: soft: -1 hard: -1 - # mem_limit: 1024mM + # mem_limit: 512mM volumes: - ${ES_DATA}/node1:/usr/share/elasticsearch/data - ${ES_DATA}/extra:/usr/share/elasticsearch/data/extra @@ -30,8 +30,7 @@ services: #- ${DATA_PATH}:/data ports: - "9200:9200" - logging: - driver: none + networks: default: diff --git a/docker-compose-elasticsearch.yml b/docker-compose-elasticsearch.yml index c40c753a..63d1b8c3 100644 --- a/docker-compose-elasticsearch.yml +++ b/docker-compose-elasticsearch.yml @@ -30,8 +30,7 @@ services: #- ${DATA_PATH}:/data ports: - "9200:9200" - logging: - driver: none + networks: default: diff --git a/docker-compose-frontend-dev.yml b/docker-compose-frontend-dev.yml index 6c078219..465307f7 100644 --- a/docker-compose-frontend-dev.yml +++ b/docker-compose-frontend-dev.yml @@ -11,7 +11,7 @@ services: no_proxy: ${no_proxy} VIEWERJS_VERSION: ${VIEWERJS_VERSION} stdin_open: true - image: ${FRONTEND_DEV_HOST} + image: ${FRONTEND_DEV_HOST}:${APP_VERSION} container_name: ${FRONTEND_DEV_HOST} volumes: - ${FRONTEND}/src:/app/src From e81bef5c4401c83b0d6dabe3cfa89bfa8f921387 Mon Sep 17 00:00:00 2001 From: victor Date: Wed, 10 Jan 2024 18:14:43 +0100 Subject: [PATCH 2/4] Add json logs nginx --- deployments/nginx/ingress.yaml | 2 ++ deployments/nginx/values.yaml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/deployments/nginx/ingress.yaml b/deployments/nginx/ingress.yaml index ef047f65..21fe6d3a 100644 --- a/deployments/nginx/ingress.yaml +++ b/deployments/nginx/ingress.yaml @@ -8,6 +8,7 @@ metadata: cert-manager.io/cluster-issuer: "letsencrypt-prod" nginx.ingress.kubernetes.io/use-regex: "true" nginx.ingress.kubernetes.io/rewrite-target: /$1 + nginx.ingress.kubernetes.io/enable-access-log: "true" spec: ingressClassName: nginx tls: @@ -41,6 +42,7 @@ metadata: namespace: ridoc # Namespace must be the same as that of target services below. annotations: cert-manager.io/cluster-issuer: "letsencrypt-prod" + nginx.ingress.kubernetes.io/enable-access-log: "false" # nginx.ingress.kubernetes.io/ssl-redirect: "false" # Set to true once SSL is set up. spec: ingressClassName: nginx diff --git a/deployments/nginx/values.yaml b/deployments/nginx/values.yaml index f31432ea..8c6a9f1c 100644 --- a/deployments/nginx/values.yaml +++ b/deployments/nginx/values.yaml @@ -12,5 +12,9 @@ controller: useHostPort: true service: type: ClusterIP + config: + access-log-path: "/var/log/nginx/access.log" + log-format-escape-json: true + log-format-upstream: '{"http_x_forwarded_for":"$http_x_forwarded_for","http_referer":"$http_referer","http_user_agent":"$http_user_agent","remote_addr":"$remote_addr","remote_user":"$remote_user","time_local":"$time_local","request":"$request","request_time":$request_time,"request_method":"$request_method","request_uri":"$request_uri","uri":"$uri","status":$status,"body_bytes_sent":$body_bytes_sent,"request_body": "$request_body"}' rbac: create: true From a8609e8f44ca8923119fdc5c9fee72dcedf3b6e2 Mon Sep 17 00:00:00 2001 From: victorjourne Date: Thu, 11 Jan 2024 12:01:37 +0100 Subject: [PATCH 3/4] WP: nginx values with filePath --- Makefile | 16 ++++++---------- deployments/nginx/values.yaml | 1 + 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index dbf4ccd9..8eacf313 100644 --- a/Makefile +++ b/Makefile @@ -175,27 +175,23 @@ create-namespace: @echo $@ (cat ${KUBE_DIR}/namespace.yaml | envsubst | kubectl apply -f -) && touch $@ -deploy-k8s-traefik: - helm upgrade --install --values ${KUBE_DIR}/traefik/values.yaml traefik traefik/traefik --namespace traefik - @cat ${KUBE_DIR}/traefik/ingress.yaml | envsubst | kubectl apply -f - +deploy-k8s-nginx: + helm upgrade --install --values ${KUBE_DIR}/nginx/values.yaml nginx ingress-nginx/ingress-nginx --namespace nginx + @cat ${KUBE_DIR}/nginx/ingress.yaml | envsubst | kubectl apply -f - deploy-k8s-configmap: create-namespace kubectl create configmap env-${INDEX_NAME} --from-env-file=${ENV_FILE} --namespace ridoc -o yaml --dry-run=client | kubectl apply -f - kubectl create configmap static-${INDEX_NAME} --from-file=${FRONTEND_STATIC_USER} --namespace ridoc -o yaml --dry-run=client | kubectl apply -f - - kubectl create configmap logstash-pipeline --from-file=logstash/pipeline/logstash.conf --namespace ridoc -o yaml --dry-run=client | kubectl apply -f - deploy-k8s-volume: create-namespace @cat ${KUBE_DIR}/volume.yaml | envsubst | kubectl apply -f - -deploy-traefik: - helm upgrade --install --values ${KUBE_DIR}/traefik/values.yaml traefik traefik/traefik --namespace traefik - @cat ${KUBE_DIR}/ingress.yaml | envsubst | kubectl apply -f - - deploy-k8s-ekl: create-namespace @echo $@ - @cat ${KUBE_DIR}/ekl/elasticsearch.yaml | envsubst | helm upgrade --install elasticsearch elastic/elasticsearch -n ridoc -f - - @cat ${KUBE_DIR}/ekl/kibana.yaml | envsubst | helm upgrade --install kibana elastic/kibana -n ridoc -f - + #@cat ${KUBE_DIR}/ekl/elasticsearch.yaml | envsubst | helm upgrade --install elasticsearch elastic/elasticsearch -n ridoc -f - + #@cat ${KUBE_DIR}/ekl/kibana.yaml | envsubst | helm upgrade --install kibana elastic/kibana -n ridoc -f - #@cat ${KUBE_DIR}/ekl/logstash.yaml | envsubst | helm upgrade --install logstash elastic/logstash -n ridoc -f - + @cat ${KUBE_DIR}/ekl/filebeat.yaml | envsubst | helm upgrade --install filebeat elastic/filebeat -n ridoc -f - deploy-k8s-frontend: deploy-k8s-configmap @echo $@ diff --git a/deployments/nginx/values.yaml b/deployments/nginx/values.yaml index 8c6a9f1c..78a3c0f3 100644 --- a/deployments/nginx/values.yaml +++ b/deployments/nginx/values.yaml @@ -16,5 +16,6 @@ controller: access-log-path: "/var/log/nginx/access.log" log-format-escape-json: true log-format-upstream: '{"http_x_forwarded_for":"$http_x_forwarded_for","http_referer":"$http_referer","http_user_agent":"$http_user_agent","remote_addr":"$remote_addr","remote_user":"$remote_user","time_local":"$time_local","request":"$request","request_time":$request_time,"request_method":"$request_method","request_uri":"$request_uri","uri":"$uri","status":$status,"body_bytes_sent":$body_bytes_sent,"request_body": "$request_body"}' + rbac: create: true From 6f1e03e049fc470553580742c7db4657bbe1bc94 Mon Sep 17 00:00:00 2001 From: victorjourne Date: Tue, 6 Feb 2024 23:44:05 +0100 Subject: [PATCH 4/4] =?UTF-8?q?=F0=9F=94=92=EF=B8=8F=20=20Add=20auth=20hea?= =?UTF-8?q?der=20to=20pdf=20js?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- backend/requirements.txt | 2 +- backend/tools/elastic.py | 2 +- deployments/nginx/values.yaml | 11 +- docker-compose-elasticsearch-huge.yml | 4 +- frontend/package.json | 3 +- frontend/src/routes/ResultPage.svelte | 208 +++++++++++++++++++------- 6 files changed, 171 insertions(+), 59 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 1fa2f834..9bd2b26e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,4 +1,4 @@ -Flask>=1.1.1 +Flask>=2.2.2 #flask-httpauth>=4.2.0 flask-jwt-extended>=3.25.0 #flask-cors==3.0.8 diff --git a/backend/tools/elastic.py b/backend/tools/elastic.py index bd5ab221..250931c9 100644 --- a/backend/tools/elastic.py +++ b/backend/tools/elastic.py @@ -46,7 +46,7 @@ ssl_context = None es = Elasticsearch([{ - "scheme": "https", + "scheme": scheme, 'host': getenv('ES_HOST', 'elasticsearch'), 'port': getenv('ES_PORT', '9200'), 'timeout': 240, 'max_retries': 10, diff --git a/deployments/nginx/values.yaml b/deployments/nginx/values.yaml index 78a3c0f3..bdbc13c1 100644 --- a/deployments/nginx/values.yaml +++ b/deployments/nginx/values.yaml @@ -13,9 +13,18 @@ controller: service: type: ClusterIP config: - access-log-path: "/var/log/nginx/access.log" + access-log-path: "/tmp/nginx/access.log" log-format-escape-json: true log-format-upstream: '{"http_x_forwarded_for":"$http_x_forwarded_for","http_referer":"$http_referer","http_user_agent":"$http_user_agent","remote_addr":"$remote_addr","remote_user":"$remote_user","time_local":"$time_local","request":"$request","request_time":$request_time,"request_method":"$request_method","request_uri":"$request_uri","uri":"$uri","status":$status,"body_bytes_sent":$body_bytes_sent,"request_body": "$request_body"}' + extraVolumes: + - name: nginx-logs + hostPath: + path: /tmp + + extraVolumeMounts: + - name: nginx-logs + mountPath: /tmp/nginx rbac: create: true + diff --git a/docker-compose-elasticsearch-huge.yml b/docker-compose-elasticsearch-huge.yml index 3e84f133..663dd52e 100644 --- a/docker-compose-elasticsearch-huge.yml +++ b/docker-compose-elasticsearch-huge.yml @@ -16,13 +16,13 @@ services: - cluster.name=matchid-cluster - bootstrap.memory_lock=true - TAKE_FILE_OWNERSHIP=true - - "ES_JAVA_OPTS=-Xms512m -Xmx512m" + - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m" - discovery.type=single-node ulimits: memlock: soft: -1 hard: -1 - # mem_limit: 512mM + # mem_limit: 1024mM volumes: - ${ES_DATA}/node1:/usr/share/elasticsearch/data - ${ES_DATA}/extra:/usr/share/elasticsearch/data/extra diff --git a/frontend/package.json b/frontend/package.json index d68505c8..ec8a82de 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -19,7 +19,8 @@ "polka": "next", "simple-svelte-autocomplete": "1.2.4", "sirv": "^0.4.0", - "svelte-tags-input": "^2.6.5" + "svelte-tags-input": "^2.6.5", + "pdfjs-dist": "^4.0.3" }, "devDependencies": { "npm-run-all": "^4.1.5", diff --git a/frontend/src/routes/ResultPage.svelte b/frontend/src/routes/ResultPage.svelte index 57a35ca0..9307da92 100644 --- a/frontend/src/routes/ResultPage.svelte +++ b/frontend/src/routes/ResultPage.svelte @@ -1,78 +1,180 @@ - - {filename} - - -{#if meta !=undefined && $itemJson!=undefined && filename !=undefined} - -
-
+ + {filename} + -
- +{#if meta != undefined && $itemJson != undefined && filename != undefined} +
+
+
+ - {#each meta as { value, key, type, placeholder, innerHtml, highlight, metadata, rows, color} (key)} - {#if !isEmpty(value) || (!readonly && metadata) } - - {/if} - {/each} + {#each meta as { value, key, type, placeholder, innerHtml, highlight, metadata, rows, color } (key)} + {#if !isEmpty(value) || (!readonly && metadata)} + + {/if} + {/each} +
+ + +
- -
-
{/if} +