From 1dd3735cd832856d3ba40f3ab0861de1d778a51a Mon Sep 17 00:00:00 2001 From: Mathias Leimgruber Date: Mon, 9 Oct 2023 17:06:44 -0400 Subject: [PATCH] Add option to start a elasticsearch 7 or 8 with your dev environment. --- .github/workflows/tests.yml | 33 ++++++++-- CHANGELOG.md | 4 ++ Makefile | 60 +++++++++++++++---- README.md | 8 +++ docker-compose.dev.yaml | 4 +- docker/elasticsearch.Dockerfile | 2 +- docker/plone.Dockerfile | 4 +- docker/worker.Dockerfile | 4 +- setup.py | 2 +- .../elasticsearch/browser/controlpanel.py | 7 ++- src/collective/elasticsearch/interfaces.py | 3 +- src/collective/elasticsearch/manager.py | 17 ++++-- src/collective/elasticsearch/redis/tasks.py | 5 +- .../elasticsearch/tests/__init__.py | 14 ++++- src/collective/elasticsearch/utils.py | 5 +- 15 files changed, 138 insertions(+), 34 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 91bbabc..5cc004a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -31,13 +31,13 @@ jobs: # git checkout - uses: actions/checkout@v2 - - name: Setup elasticsearch docker container with ingest attachment plugin + - name: Setup elasticsearch 7.17.7 docker container with ingest attachment plugin run: | docker container create --name elastictest \ -e "discovery.type=single-node" \ -e "cluster.name=docker-cluster" \ -e "http.cors.enabled=true" \ - -e "http.cors.allow-origin=*" \ + -e "http.cors.allow-origin='*'" \ -e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \ -e "http.cors.allow-credentials=true" \ -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ @@ -57,9 +57,32 @@ jobs: - name: Install package run: | - pip install -e ".[test, redis]" + pip install -e ".[test, redis]" elasticsearch==7.17.7 - # test - - name: test + # test elasticsearch 7.17.7 + - name: test elasticsearch 7.17.7 run: | + zope-testrunner --auto-color --auto-progress --test-path src; \ + docker stop elastictest; \ + docker rm elastictest + + # test elasticsearch 8.10.2 + - name: Setup elasticsearch 8.10.2 docker container + run: | + docker container create --name elastictest8 \ + -e "discovery.type=single-node" \ + -e "cluster.name=docker-cluster" \ + -e "http.cors.enabled=true" \ + -e "http.cors.allow-origin='*'" \ + -e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \ + -e "http.cors.allow-credentials=true" \ + -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + -p 9200:9200 \ + -p 9300:9300 \ + elasticsearch:8.10.2; \ + docker start elastictest8; \ + + - name: test elasticsearch 8.10.2 + run: | + pip install elasticsearch==8.10.0; / zope-testrunner --auto-color --auto-progress --test-path src diff --git a/CHANGELOG.md b/CHANGELOG.md index 559131c..efdc740 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 5.1.0 (unreleased) + +- Add suport of elasticsearch to 8.10.2 @maethu + ## 5.0.1 (unreleased) - Update elasticsearch to 7.17.7 (Ready for 8.x and apple silicon images are available) @maethu diff --git a/Makefile b/Makefile index 58a6987..6dd1368 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,8 @@ PLONE6=6.0-latest INSTANCE_YAML=instance.yaml -ELASTIC_SEARCH_IMAGE=elasticsearch:7.17.7 +ELASTIC_SEARCH_IMAGE_7=elasticsearch:7.17.7 +ELASTIC_SEARCH_IMAGE_8=elasticsearch:8.10.2 ELASTIC_SEARCH_CONTAINER=elastictest REDIS_IMAGE=redis:7.0.5 @@ -138,29 +139,57 @@ lint-pyroma: ## validate using pyroma lint-zpretty: ## validate ZCML/XML using zpretty $(LINT) zpretty ${CODEPATH} -.PHONY: elastic -elastic: ## Create Elastic Search container +.PHONY: elastic-7 +elastic-7: ## Create Elastic Search container + @echo "$(GREEN)==> Create Elastic Search Version 7 Container $(RESET)" @if [ $(ELASTIC_SEARCH_CONTAINERS) -eq 0 ]; then \ docker container create --name $(ELASTIC_SEARCH_CONTAINER) \ -e "discovery.type=single-node" \ -e "cluster.name=docker-cluster" \ -e "http.cors.enabled=true" \ - -e "http.cors.allow-origin=*" \ + -e "http.cors.allow-origin='*'" \ -e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \ -e "http.cors.allow-credentials=true" \ + -e "xpack.security.enabled=false" \ -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ -p 9200:9200 \ -p 9300:9300 \ - $(ELASTIC_SEARCH_IMAGE); \ + $(ELASTIC_SEARCH_IMAGE_7); \ docker start $(ELASTIC_SEARCH_CONTAINER); \ docker exec $(ELASTIC_SEARCH_CONTAINER) /bin/sh -c "bin/elasticsearch-plugin install ingest-attachment -b"; \ - docker stop $(ELASTIC_SEARCH_CONTAINER);fi + docker stop $(ELASTIC_SEARCH_CONTAINER); else \ + echo "$(RED)==> Could not create container: Container already exists $(RESET)";fi -.PHONY: start-elastic -start-elastic: elastic ## Start Elastic Search +.PHONY: elastic-8 +elastic-8: ## Create Elastic Search container + @echo "$(GREEN)==> Create Elastic Search Version 8 Container $(RESET)" + @if [ $(ELASTIC_SEARCH_CONTAINERS) -eq 0 ]; then \ + pip install elasticsearch==8.10.0; \ + docker container create --name $(ELASTIC_SEARCH_CONTAINER) \ + -e "discovery.type=single-node" \ + -e "cluster.name=docker-cluster" \ + -e "http.cors.enabled=true" \ + -e "http.cors.allow-origin='*'" \ + -e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \ + -e "http.cors.allow-credentials=true" \ + -e "xpack.security.enabled=false" \ + -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \ + -p 9200:9200 \ + -p 9300:9300 \ + $(ELASTIC_SEARCH_IMAGE_8); else \ + echo "$(RED)==> Could not create container: Container already exists $(RESET)";fi + +.PHONY: start-elastic-7 +start-elastic-7: elastic-7 ## Start Elastic Search @echo "$(GREEN)==> Start Elastic Search$(RESET)" @docker start $(ELASTIC_SEARCH_CONTAINER) +.PHONY: start-elastic-8 +start-elastic-8: elastic-8 ## Start Elastic Search + @echo "$(GREEN)==> Start Elastic Search$(RESET)" + @docker start $(ELASTIC_SEARCH_CONTAINER) + + .PHONY: stop-elastic stop-elastic: ## Stop Elastic Search @echo "$(GREEN)==> Stop Elastic Search$(RESET)" @@ -187,12 +216,23 @@ stop-redis: ## Stop redis .PHONY: test test: ## run tests - make start-elastic + bin/pip install elasticsearch==8.10.0 + make start-elastic-8 + make start-redis + PYTHONWARNINGS=ignore ./bin/zope-testrunner --auto-color --auto-progress --test-path src/ + make stop-elastic + make stop-redis + +.PHONY: test-elastic-7 +test-elastic-7: ## run tests with elasticsearch 7 + bin/pip install elasticsearch==7.17.7 + make start-elastic-7 make start-redis PYTHONWARNINGS=ignore ./bin/zope-testrunner --auto-color --auto-progress --test-path src/ make stop-elastic make stop-redis + .PHONY: start start: ## Start a Plone instance on localhost:8080 PYTHONWARNINGS=ignore ./bin/runwsgi instance/etc/zope.ini @@ -202,7 +242,7 @@ populate: ## Populate site with wikipedia content PYTHONWARNINGS=ignore ./bin/zconsole run etc/zope.conf scripts/populate.py .PHONY: start-redis-support -start-redis-support: ## Start a Plone instance on localhost:8080 +start-redis-support: ## Start a Plone instance on localhost:8080 with redis support @echo "$(GREEN)==> Set env variables, PLONE_REDIS_DSN, PLONE_BACKEND, PLONE_USERNAME and PLONE_PASSWORD before start instance$(RESET)" PYTHONWARNINGS=ignore \ $(DEFAULT_ENV_ES_REDIS) \ diff --git a/README.md b/README.md index be51ed2..82a9be9 100644 --- a/README.md +++ b/README.md @@ -263,6 +263,14 @@ make format make lint ``` +# Upgrade from elasticsearch 7.17.7 to 8.10.x + +I you were already on elasticsearch 7.17.7 there is usually not mutch you need to do with elasticsearch itself. I will update itself automatically, once you update to elasticsearch 8 and restart the service. + +- If there is a configuration issue creating the new elasticsearch 8 container, make sure you have single or douple quotes around certain values. Like change http.cors.allow-origin=* to http.cors.allow-origin="*". +- Add port to all hosts in the control panel hosts section. If there is no port defined we add port 9200 by default if you are running elasticsearch 8. +- Please check the elasticsearch upgrade guide https://www.elastic.co/guide/en/elastic-stack/8.10/upgrading-elastic-stack.html#prepare-to-upgrade + ## License The project is licensed under the GPLv2. diff --git a/docker-compose.dev.yaml b/docker-compose.dev.yaml index 1156aa0..024c844 100644 --- a/docker-compose.dev.yaml +++ b/docker-compose.dev.yaml @@ -20,9 +20,10 @@ services: - discovery.type=single-node - cluster.name=docker-cluster - http.cors.enabled=true - - http.cors.allow-origin=* + - http.cors.allow-origin="*" - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization - http.cors.allow-credentials=true + - xpack.security.enabled=false - ES_JAVA_OPTS=-Xms512m -Xmx512m volumes: - elasticsearch_data:/usr/share/elasticsearch/data @@ -36,6 +37,7 @@ services: - PLONE_BACKEND=http://plone:8080/Plone - PLONE_USERNAME=admin - PLONE_PASSWORD=admin + - PLONE_ELASTICSEARCH_HOST=host.docker.internal plone: build: diff --git a/docker/elasticsearch.Dockerfile b/docker/elasticsearch.Dockerfile index dabe8d3..fe7a537 100644 --- a/docker/elasticsearch.Dockerfile +++ b/docker/elasticsearch.Dockerfile @@ -1,3 +1,3 @@ -FROM elasticsearch:7.17.7 +FROM elasticsearch:8.10.2 RUN bin/elasticsearch-plugin install ingest-attachment -b diff --git a/docker/plone.Dockerfile b/docker/plone.Dockerfile index 6fccf36..aae039b 100644 --- a/docker/plone.Dockerfile +++ b/docker/plone.Dockerfile @@ -1,8 +1,8 @@ -FROM plone/plone-backend:6.0.0b3 +FROM plone/plone-backend:6.0.7 WORKDIR /app -RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@mle-redis-rq#egg=collective.elasticsearch[redis] +RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@main#egg=collective.elasticsearch[redis] ENV PROFILES="collective.elasticsearch:default collective.elasticsearch:docker-dev" ENV TYPE="classic" diff --git a/docker/worker.Dockerfile b/docker/worker.Dockerfile index 9af93bf..be9b2bd 100644 --- a/docker/worker.Dockerfile +++ b/docker/worker.Dockerfile @@ -1,7 +1,7 @@ -FROM plone/plone-backend:6.0.0b3 +FROM plone/plone-backend:6.0.7 WORKDIR /app -RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@mle-redis-rq#egg=collective.elasticsearch[redis] +RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@main#egg=collective.elasticsearch[redis] CMD /app/bin/rq worker normal low --with-scheduler --url=$PLONE_REDIS_DSN diff --git a/setup.py b/setup.py index 61fe2aa..12e042c 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,7 @@ python_requires=">=3.7", install_requires=[ "setuptools", - "elasticsearch==7.17.7, 8.10.0", + "elasticsearch>=7.17.7, <=8.10.0", "plone.app.registry", "plone.api", "setuptools", diff --git a/src/collective/elasticsearch/browser/controlpanel.py b/src/collective/elasticsearch/browser/controlpanel.py index ab6e8de..ffc93d7 100644 --- a/src/collective/elasticsearch/browser/controlpanel.py +++ b/src/collective/elasticsearch/browser/controlpanel.py @@ -1,6 +1,7 @@ from collective.elasticsearch.interfaces import IElasticSettings from collective.elasticsearch.manager import ElasticSearchManager from collective.elasticsearch.utils import is_redis_available +from elastic_transport import ConnectionTimeout from elasticsearch.exceptions import ConnectionError as conerror from plone import api from plone.app.registry.browser.controlpanel import ControlPanelFormWrapper @@ -52,6 +53,7 @@ def connection_status(self): except ( conerror, ConnectionError, + ConnectionTimeout, NewConnectionError, ConnectionRefusedError, AttributeError, @@ -60,7 +62,10 @@ def connection_status(self): @property def es_info(self): - return self.es.info + try: + return self.es.info + except ConnectionTimeout: + return None @property def enabled(self): diff --git a/src/collective/elasticsearch/interfaces.py b/src/collective/elasticsearch/interfaces.py index 34917b4..84ed242 100644 --- a/src/collective/elasticsearch/interfaces.py +++ b/src/collective/elasticsearch/interfaces.py @@ -1,3 +1,4 @@ +from collective.elasticsearch.utils import ELASTIC_SEARCH_VERSION from dataclasses import dataclass from Products.CMFCore.interfaces import IIndexQueueProcessor from typing import Dict @@ -57,7 +58,7 @@ class IElasticSettings(Interface): hosts = schema.List( title="Hosts", - default=["http://127.0.0.1:9200"], + default=ELASTIC_SEARCH_VERSION == 8 and ["http://127.0.0.1:9200"] or ["127.0.0.1"], unique=True, value_type=schema.TextLine(title="Host"), ) diff --git a/src/collective/elasticsearch/manager.py b/src/collective/elasticsearch/manager.py index 714de5e..39a9714 100644 --- a/src/collective/elasticsearch/manager.py +++ b/src/collective/elasticsearch/manager.py @@ -9,7 +9,6 @@ from DateTime import DateTime from elasticsearch import Elasticsearch from elasticsearch import exceptions -from elasticsearch.exceptions import NotFoundError from plone import api from Products.CMFCore.indexing import processQueue from Products.CMFCore.permissions import AccessInactivePortalContent @@ -28,6 +27,13 @@ INDEX_VERSION_ATTR = "_elasticindexversion" +try: + from elasticsearch.exceptions import BadRequestError +except ImportError: + # Backwards compatibility with ES 7 + BadRequestError = Exception + + @implementer(interfaces.IElasticSearchManager) class ElasticSearchManager: @@ -156,7 +162,7 @@ def info(self) -> list: ("Elastic Search Version", cluster_version), ("Number of docs (Catalog)", catalog_docs), ] - except NotFoundError: + except exceptions.NotFoundError: logger.warning("Error getting stats", exc_info=True) return [] @@ -195,7 +201,7 @@ def _recreate_catalog(self): conn.indices.delete(index=self.real_index_name) except exceptions.NotFoundError: pass - except (exceptions.BadRequestError, exceptions.TransportError) as exc: + except (BadRequestError, exceptions.TransportError) as exc: if exc.error != "illegal_argument_exception": raise conn.indices.delete_alias(index="_all", name=self.real_index_name) @@ -249,7 +255,6 @@ def _setup_attachment_pipeline_and_default_index(self): "attachment": { "target_field": "_ingest._value.attachment", "field": "_ingest._value.data", - "remove_binary": True, # version 8 "properties": ["content"], } }, @@ -284,6 +289,10 @@ def _setup_attachment_pipeline_and_default_index(self): # } # ] } + + if ELASTIC_SEARCH_VERSION == 8: + body['processors'][0]['foreach']['processor']['attachment']['remove_binary'] = True + self.connection.ingest.put_pipeline(id="cbor-attachments", body=body) settings = {"index": {"default_pipeline": "cbor-attachments"}} diff --git a/src/collective/elasticsearch/redis/tasks.py b/src/collective/elasticsearch/redis/tasks.py index b22a6c2..470e8f4 100644 --- a/src/collective/elasticsearch/redis/tasks.py +++ b/src/collective/elasticsearch/redis/tasks.py @@ -74,7 +74,10 @@ def bulk_update(hosts, params, index_name, body): item[1]["doc"] = data es_data = [item for sublist in body for item in sublist] - connection.bulk(index=index_name, operations=es_data) + if ELASTIC_SEARCH_VERSION == 8: + connection.bulk(index=index_name, operations=es_data) + else: + connection.bulk(es_data, index=index_name) return "Done" diff --git a/src/collective/elasticsearch/tests/__init__.py b/src/collective/elasticsearch/tests/__init__.py index 49767ed..7b08ff0 100644 --- a/src/collective/elasticsearch/tests/__init__.py +++ b/src/collective/elasticsearch/tests/__init__.py @@ -35,10 +35,18 @@ def setUp(self): os.environ["PLONE_BACKEND"] = self.portal.absolute_url() settings = utils.get_settings() - # disable sniffing hosts in tests because docker... - settings.sniffer_timeout = None settings.enabled = True - settings.sniffer_timeout = 0.0 + + # Elasticsearch 8 requires a sniffing timeout, otherwise the python + # ES client tries to sniff first even though the new ES instance with + # index is not yet initialized. + + # This means, basically never sniff during tests + # Since it's a single instance installation in tests sniffing does not + # make sense anyway. + + # With ES 7 setting 0.0 was enough + settings.sniffer_timeout = 10000.0 # Raise elastic search exceptions settings.raise_search_exception = True diff --git a/src/collective/elasticsearch/utils.py b/src/collective/elasticsearch/utils.py index fff883c..1886a35 100644 --- a/src/collective/elasticsearch/utils.py +++ b/src/collective/elasticsearch/utils.py @@ -1,5 +1,4 @@ from collective.elasticsearch import logger -from collective.elasticsearch.interfaces import IElasticSettings from plone.registry.interfaces import IRegistry from plone.uuid.interfaces import IUUID from Products.ZCatalog import ZCatalog @@ -46,6 +45,8 @@ def get_brain_from_path(zcatalog: ZCatalog, path: str) -> AbstractCatalogBrain: def get_settings(): """Return IElasticSettings values.""" + from collective.elasticsearch.interfaces import IElasticSettings + registry = getUtility(IRegistry) try: settings = registry.forInterface(IElasticSettings, check=False) @@ -62,7 +63,7 @@ def get_connection_settings(): # Make sure the is a port defined for all hosts for index, host in enumerate(settings.hosts): if ":" not in host: - hosts[index] = f"{host}:9200" + hosts[index] = f"http://{host}:9200" return hosts, { "retry_on_timeout": settings.retry_on_timeout,