Add option to start a elasticsearch 7 or 8 with your dev environment.

.github/workflows/tests.yml

@@ -31,13 +31,13 @@ jobs:
       # git checkout
       - uses: actions/checkout@v2
 
-      - name: Setup elasticsearch docker container with ingest attachment plugin
+      - name: Setup elasticsearch 7.17.7 docker container with ingest attachment plugin
         run: |
           docker container create --name elastictest \
           -e "discovery.type=single-node" \
           -e "cluster.name=docker-cluster" \
           -e "http.cors.enabled=true" \
-          -e "http.cors.allow-origin=*" \
+          -e "http.cors.allow-origin='*'" \
           -e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \
           -e "http.cors.allow-credentials=true" \
           -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
@@ -57,9 +57,32 @@ jobs:
 
       - name: Install package
         run: |
-          pip install -e ".[test, redis]"
+          pip install -e ".[test, redis]" elasticsearch==7.17.7
 
-      # test
-      - name: test
+      # test elasticsearch 7.17.7
+      - name: test elasticsearch 7.17.7
         run: |
+          zope-testrunner --auto-color --auto-progress --test-path src; \
+          docker stop elastictest; \
+          docker rm elastictest
+
+      # test elasticsearch 8.10.2
+      - name: Setup elasticsearch 8.10.2 docker container
+        run: |
+          docker container create --name elastictest8 \
+          -e "discovery.type=single-node" \
+          -e "cluster.name=docker-cluster" \
+          -e "http.cors.enabled=true" \
+          -e "http.cors.allow-origin='*'" \
+          -e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \
+          -e "http.cors.allow-credentials=true" \
+          -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
+          -p 9200:9200 \
+          -p 9300:9300 \
+          elasticsearch:8.10.2; \
+          docker start elastictest8; \
+
+      - name: test elasticsearch 8.10.2
+        run: |
+          pip install elasticsearch==8.10.0; /
           zope-testrunner --auto-color --auto-progress --test-path src

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 5.1.0 (unreleased)
+
+- Add suport of elasticsearch to 8.10.2 @maethu
+
 ## 5.0.1 (unreleased)
 
 - Update elasticsearch to 7.17.7 (Ready for 8.x and apple silicon images are available) @maethu

Makefile

@@ -20,7 +20,8 @@ PLONE6=6.0-latest
 
 INSTANCE_YAML=instance.yaml
 
-ELASTIC_SEARCH_IMAGE=elasticsearch:7.17.7
+ELASTIC_SEARCH_IMAGE_7=elasticsearch:7.17.7
+ELASTIC_SEARCH_IMAGE_8=elasticsearch:8.10.2
 ELASTIC_SEARCH_CONTAINER=elastictest
 
 REDIS_IMAGE=redis:7.0.5
@@ -138,29 +139,57 @@ lint-pyroma: ## validate using pyroma
 lint-zpretty: ## validate ZCML/XML using zpretty
 	$(LINT) zpretty ${CODEPATH}
 
-.PHONY: elastic
-elastic: ## Create Elastic Search container
+.PHONY: elastic-7
+elastic-7: ## Create Elastic Search container
+	@echo "$(GREEN)==> Create Elastic Search Version 7 Container $(RESET)"
 	@if [ $(ELASTIC_SEARCH_CONTAINERS) -eq 0 ]; then \
 		docker container create --name $(ELASTIC_SEARCH_CONTAINER) \
 		-e "discovery.type=single-node" \
 		-e "cluster.name=docker-cluster" \
 		-e "http.cors.enabled=true" \
-		-e "http.cors.allow-origin=*" \
+		-e "http.cors.allow-origin='*'" \
 		-e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \
 		-e "http.cors.allow-credentials=true" \
+		-e "xpack.security.enabled=false" \
 		-e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
 		-p 9200:9200 \
 		-p 9300:9300 \
-		$(ELASTIC_SEARCH_IMAGE); \
+		$(ELASTIC_SEARCH_IMAGE_7); \
 		docker start $(ELASTIC_SEARCH_CONTAINER); \
 		docker exec $(ELASTIC_SEARCH_CONTAINER) /bin/sh -c "bin/elasticsearch-plugin install ingest-attachment -b"; \
-		docker stop $(ELASTIC_SEARCH_CONTAINER);fi
+		docker stop $(ELASTIC_SEARCH_CONTAINER); else \
+		echo "$(RED)==> Could not create container: Container already exists $(RESET)";fi
 
-.PHONY: start-elastic
-start-elastic: elastic ## Start Elastic Search
+.PHONY: elastic-8
+elastic-8: ## Create Elastic Search container
+	@echo "$(GREEN)==> Create Elastic Search Version 8 Container $(RESET)"
+	@if [ $(ELASTIC_SEARCH_CONTAINERS) -eq 0 ]; then \
+		pip install elasticsearch==8.10.0; \
+		docker container create --name $(ELASTIC_SEARCH_CONTAINER) \
+		-e "discovery.type=single-node" \
+		-e "cluster.name=docker-cluster" \
+		-e "http.cors.enabled=true" \
+		-e "http.cors.allow-origin='*'" \
+		-e "http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization" \
+		-e "http.cors.allow-credentials=true" \
+		-e "xpack.security.enabled=false" \
+		-e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
+		-p 9200:9200 \
+		-p 9300:9300 \
+		$(ELASTIC_SEARCH_IMAGE_8); else \
+		echo "$(RED)==> Could not create container: Container already exists $(RESET)";fi
+
+.PHONY: start-elastic-7
+start-elastic-7: elastic-7 ## Start Elastic Search
 	@echo "$(GREEN)==> Start Elastic Search$(RESET)"
 	@docker start $(ELASTIC_SEARCH_CONTAINER)
 
+.PHONY: start-elastic-8
+start-elastic-8: elastic-8 ## Start Elastic Search
+	@echo "$(GREEN)==> Start Elastic Search$(RESET)"
+	@docker start $(ELASTIC_SEARCH_CONTAINER)
+
+
 .PHONY: stop-elastic
 stop-elastic: ## Stop Elastic Search
 	@echo "$(GREEN)==> Stop Elastic Search$(RESET)"
@@ -187,12 +216,23 @@ stop-redis: ## Stop redis
 
 .PHONY: test
 test: ## run tests
-	make start-elastic
+	bin/pip install elasticsearch==8.10.0
+	make start-elastic-8
+	make start-redis
+	PYTHONWARNINGS=ignore ./bin/zope-testrunner --auto-color --auto-progress --test-path src/
+	make stop-elastic
+	make stop-redis
+
+.PHONY: test-elastic-7
+test-elastic-7: ## run tests with elasticsearch 7
+	bin/pip install elasticsearch==7.17.7
+	make start-elastic-7
 	make start-redis
 	PYTHONWARNINGS=ignore ./bin/zope-testrunner --auto-color --auto-progress --test-path src/
 	make stop-elastic
 	make stop-redis
 
+
 .PHONY: start
 start: ## Start a Plone instance on localhost:8080
 	PYTHONWARNINGS=ignore ./bin/runwsgi instance/etc/zope.ini
@@ -202,7 +242,7 @@ populate: ## Populate site with wikipedia content
 	PYTHONWARNINGS=ignore ./bin/zconsole run etc/zope.conf scripts/populate.py
 
 .PHONY: start-redis-support
-start-redis-support: ## Start a Plone instance on localhost:8080
+start-redis-support: ## Start a Plone instance on localhost:8080 with redis support
 	@echo "$(GREEN)==> Set env variables, PLONE_REDIS_DSN, PLONE_BACKEND, PLONE_USERNAME and PLONE_PASSWORD before start instance$(RESET)"
 	PYTHONWARNINGS=ignore \
 	$(DEFAULT_ENV_ES_REDIS) \

README.md

@@ -263,6 +263,14 @@ make format
 make lint
 ```
 
+# Upgrade from elasticsearch 7.17.7 to 8.10.x
+
+I you were already on elasticsearch 7.17.7 there is usually not mutch you need to do with elasticsearch itself. I will update itself automatically, once you update to elasticsearch 8 and restart the service.
+
+- If there is a configuration issue creating the new elasticsearch 8 container, make sure you have single or douple quotes around certain values. Like change http.cors.allow-origin=* to http.cors.allow-origin="*".
+- Add port to all hosts in the control panel hosts section. If there is no port defined we add port 9200 by default if you are running elasticsearch 8.
+- Please check the elasticsearch upgrade guide https://www.elastic.co/guide/en/elastic-stack/8.10/upgrading-elastic-stack.html#prepare-to-upgrade
+
 ## License
 
 The project is licensed under the GPLv2.

docker-compose.dev.yaml

@@ -20,9 +20,10 @@ services:
       - discovery.type=single-node
       - cluster.name=docker-cluster
       - http.cors.enabled=true
-      - http.cors.allow-origin=*
+      - http.cors.allow-origin="*"
       - http.cors.allow-headers=X-Requested-With,X-Auth-Token,Content-Type,Content-Length,Authorization
       - http.cors.allow-credentials=true
+      - xpack.security.enabled=false
       - ES_JAVA_OPTS=-Xms512m -Xmx512m
     volumes:
       - elasticsearch_data:/usr/share/elasticsearch/data
@@ -36,6 +37,7 @@ services:
       - PLONE_BACKEND=http://plone:8080/Plone
       - PLONE_USERNAME=admin
       - PLONE_PASSWORD=admin
+      - PLONE_ELASTICSEARCH_HOST=host.docker.internal
 
   plone:
     build:

docker/elasticsearch.Dockerfile

@@ -1,3 +1,3 @@
-FROM elasticsearch:7.17.7
+FROM elasticsearch:8.10.2
 
 RUN bin/elasticsearch-plugin install ingest-attachment -b

docker/plone.Dockerfile

@@ -1,8 +1,8 @@
-FROM plone/plone-backend:6.0.0b3
+FROM plone/plone-backend:6.0.7
 
 WORKDIR /app
 
-RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@mle-redis-rq#egg=collective.elasticsearch[redis]
+RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@main#egg=collective.elasticsearch[redis]
 
 ENV PROFILES="collective.elasticsearch:default collective.elasticsearch:docker-dev"
 ENV TYPE="classic"

docker/worker.Dockerfile

@@ -1,7 +1,7 @@
-FROM plone/plone-backend:6.0.0b3
+FROM plone/plone-backend:6.0.7
 
 WORKDIR /app
 
-RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@mle-redis-rq#egg=collective.elasticsearch[redis]
+RUN /app/bin/pip install git+https://github.com/collective/collective.elasticsearch.git@main#egg=collective.elasticsearch[redis]
 
 CMD /app/bin/rq worker normal low --with-scheduler --url=$PLONE_REDIS_DSN

setup.py

@@ -56,7 +56,7 @@
     python_requires=">=3.7",
     install_requires=[
         "setuptools",
-        "elasticsearch==7.17.7, 8.10.0",
+        "elasticsearch>=7.17.7, <=8.10.0",
         "plone.app.registry",
         "plone.api",
         "setuptools",

src/collective/elasticsearch/browser/controlpanel.py

@@ -1,6 +1,7 @@
 from collective.elasticsearch.interfaces import IElasticSettings
 from collective.elasticsearch.manager import ElasticSearchManager
 from collective.elasticsearch.utils import is_redis_available
+from elastic_transport import ConnectionTimeout
 from elasticsearch.exceptions import ConnectionError as conerror
 from plone import api
 from plone.app.registry.browser.controlpanel import ControlPanelFormWrapper
@@ -52,6 +53,7 @@ def connection_status(self):
             except (
                 conerror,
                 ConnectionError,
+                ConnectionTimeout,
                 NewConnectionError,
                 ConnectionRefusedError,
                 AttributeError,
@@ -60,7 +62,10 @@ def connection_status(self):
 
     @property
     def es_info(self):
-        return self.es.info
+        try:
+            return self.es.info
+        except ConnectionTimeout:
+            return None
 
     @property
     def enabled(self):

src/collective/elasticsearch/interfaces.py

@@ -1,3 +1,4 @@
+from collective.elasticsearch.utils import ELASTIC_SEARCH_VERSION
 from dataclasses import dataclass
 from Products.CMFCore.interfaces import IIndexQueueProcessor
 from typing import Dict
@@ -57,7 +58,7 @@ class IElasticSettings(Interface):
 
     hosts = schema.List(
         title="Hosts",
-        default=["http://127.0.0.1:9200"],
+        default=ELASTIC_SEARCH_VERSION == 8 and ["http://127.0.0.1:9200"] or ["127.0.0.1"],
         unique=True,
         value_type=schema.TextLine(title="Host"),
     )

src/collective/elasticsearch/manager.py

@@ -9,7 +9,6 @@
 from DateTime import DateTime
 from elasticsearch import Elasticsearch
 from elasticsearch import exceptions
-from elasticsearch.exceptions import NotFoundError
 from plone import api
 from Products.CMFCore.indexing import processQueue
 from Products.CMFCore.permissions import AccessInactivePortalContent
@@ -28,6 +27,13 @@
 INDEX_VERSION_ATTR = "_elasticindexversion"
 
 
+try:
+    from elasticsearch.exceptions import BadRequestError
+except ImportError:
+    # Backwards compatibility with ES 7
+    BadRequestError = Exception
+
+
 @implementer(interfaces.IElasticSearchManager)
 class ElasticSearchManager:
 
@@ -156,7 +162,7 @@ def info(self) -> list:
                     ("Elastic Search Version", cluster_version),
                     ("Number of docs (Catalog)", catalog_docs),
                 ]
-        except NotFoundError:
+        except exceptions.NotFoundError:
             logger.warning("Error getting stats", exc_info=True)
             return []
 
@@ -195,7 +201,7 @@ def _recreate_catalog(self):
             conn.indices.delete(index=self.real_index_name)
         except exceptions.NotFoundError:
             pass
-        except (exceptions.BadRequestError, exceptions.TransportError) as exc:
+        except (BadRequestError, exceptions.TransportError) as exc:
             if exc.error != "illegal_argument_exception":
                 raise
             conn.indices.delete_alias(index="_all", name=self.real_index_name)
@@ -249,7 +255,6 @@ def _setup_attachment_pipeline_and_default_index(self):
                             "attachment": {
                                 "target_field": "_ingest._value.attachment",
                                 "field": "_ingest._value.data",
-                                "remove_binary": True,  # version 8
                                 "properties": ["content"],
                             }
                         },
@@ -284,6 +289,10 @@ def _setup_attachment_pipeline_and_default_index(self):
             #     }
             # ]
         }
+
+        if ELASTIC_SEARCH_VERSION == 8:
+            body['processors'][0]['foreach']['processor']['attachment']['remove_binary'] = True
+
         self.connection.ingest.put_pipeline(id="cbor-attachments", body=body)
 
         settings = {"index": {"default_pipeline": "cbor-attachments"}}

src/collective/elasticsearch/redis/tasks.py

@@ -74,7 +74,10 @@ def bulk_update(hosts, params, index_name, body):
             item[1]["doc"] = data
 
     es_data = [item for sublist in body for item in sublist]
-    connection.bulk(index=index_name, operations=es_data)
+    if ELASTIC_SEARCH_VERSION == 8:
+        connection.bulk(index=index_name, operations=es_data)
+    else:
+        connection.bulk(es_data, index=index_name)
     return "Done"
 
 

src/collective/elasticsearch/tests/__init__.py

@@ -35,10 +35,18 @@ def setUp(self):
         os.environ["PLONE_BACKEND"] = self.portal.absolute_url()
 
         settings = utils.get_settings()
-        # disable sniffing hosts in tests because docker...
-        settings.sniffer_timeout = None
         settings.enabled = True
-        settings.sniffer_timeout = 0.0
+
+        # Elasticsearch 8 requires a sniffing timeout, otherwise the python
+        # ES client tries to sniff first even though the new ES instance with
+        # index is not yet initialized.
+
+        # This means, basically never sniff during tests
+        # Since it's a single instance installation in tests sniffing does not
+        # make sense anyway.
+
+        # With ES 7 setting 0.0 was enough
+        settings.sniffer_timeout = 10000.0
 
         # Raise elastic search exceptions
         settings.raise_search_exception = True

src/collective/elasticsearch/utils.py

@@ -1,5 +1,4 @@
 from collective.elasticsearch import logger
-from collective.elasticsearch.interfaces import IElasticSettings
 from plone.registry.interfaces import IRegistry
 from plone.uuid.interfaces import IUUID
 from Products.ZCatalog import ZCatalog
@@ -46,6 +45,8 @@ def get_brain_from_path(zcatalog: ZCatalog, path: str) -> AbstractCatalogBrain:
 
 def get_settings():
     """Return IElasticSettings values."""
+    from collective.elasticsearch.interfaces import IElasticSettings
+
     registry = getUtility(IRegistry)
     try:
         settings = registry.forInterface(IElasticSettings, check=False)
@@ -62,7 +63,7 @@ def get_connection_settings():
         # Make sure the is a port defined for all hosts
         for index, host in enumerate(settings.hosts):
             if ":" not in host:
-                hosts[index] = f"{host}:9200"
+                hosts[index] = f"http://{host}:9200"
 
     return hosts, {
         "retry_on_timeout": settings.retry_on_timeout,