Added elasticsearch middleware

package.json

@@ -16,7 +16,24 @@
     "type": "git",
     "url": "git@github.com:eea/volto-searchlib.git"
   },
-  "dependencies": {},
+  "dependencies": {
+    "@elastic/react-search-ui": "1.7.0",
+    "@elastic/search-ui": "1.7.0",
+    "@visx/group": "2.1.0",
+    "@visx/responsive": "2.1.1",
+    "@visx/scale": "2.1.0",
+    "@visx/shape": "2.1.1",
+    "@visx/tooltip": "2.1.0",
+    "csv-stringify": "5.6.5",
+    "elasticsearch": "16.7.2",
+    "jotai": "1.3.4",
+    "lodash.clonedeep": "4.5.0",
+    "lodash.isfunction": "3.0.9",
+    "lodash.uniq": "4.5.0",
+    "re-resizable": "6.9.1",
+    "react-masonry-component": "6.3.0",
+    "react-speech-recognition": "3.8.2"
+  },
   "devDependencies": {
     "@cypress/code-coverage": "^3.9.5",
     "babel-plugin-transform-class-properties": "^6.24.1"

razzle.extend.js

@@ -0,0 +1,26 @@
+const path = require('path');
+const makeLoaderFinder = require('razzle-dev-utils/makeLoaderFinder');
+
+const pkgs = ['@eeacms/search', '@eeacms/globalsearch'];
+
+const modify = (config, { target, dev }, webpack) => {
+  const projectRootPath = path.resolve('.');
+  const jsConfig = require(`${projectRootPath}/jsconfig.json`);
+  const searchlibConf = jsConfig.compilerOptions.paths.searchlib;
+  if (!searchlibConf) return config;
+
+  const babelLoaderFinder = makeLoaderFinder('babel-loader');
+  const babelLoader = config.module.rules.find(babelLoaderFinder);
+  const { include } = babelLoader;
+
+  pkgs.forEach((name) => {
+    include.push(config.resolve.alias[name]);
+  });
+
+  return config;
+};
+
+module.exports = {
+  plugins: (plugs) => plugs,
+  modify,
+};

src/SearchBlock/SearchBlockEdit.jsx

@@ -2,9 +2,17 @@ import React from 'react';
 import { SearchBlockSchema } from './schema';
 import { BlockDataForm, SidebarPortal } from '@plone/volto/components';
 import SearchBlockView from './SearchBlockView';
+import config from '@plone/volto/registry';
 
 const SearchBlockEdit = (props) => {
   const schema = SearchBlockSchema(props);
+  const conf = config.settings.searchlib.searchui;
+  schema.properties.appName.choices = Object.keys(conf).map((k) => [
+    k,
+    k,
+    // conf[k].title || k,
+  ]);
+
   return (
     <div>
       <SearchBlockView {...props} />

src/SearchBlock/SearchBlockView.jsx

@@ -1,5 +1,14 @@
 import React from 'react';
+import config from '@plone/volto/registry';
+import { SearchApp } from '@eeacms/search';
 
-export default function SearchBlockView(props) {
-  return <div>Search</div>;
+export default function SearchBlockView({ data = {} }) {
+  const { appName = 'default' } = data;
+  const registry = config.settings.searchlib;
+  console.log(config.settings.searchlib);
+  return (
+    <div className="searchlib-block">
+      <SearchApp registry={registry} appName={appName} />
+    </div>
+  );
 }

src/SearchBlock/schema.js

@@ -6,7 +6,7 @@ export const SearchBlockSchema = () => ({
       id: 'default',
       title: 'Default',
       fields: [
-        'registry',
+        'appName',
         // 'es_index'
       ],
     },
@@ -18,10 +18,9 @@ export const SearchBlockSchema = () => ({
   ],
 
   properties: {
-    registry: {
-      title: 'Configuration registry',
-      // choices: [
-      // ],
+    appName: {
+      title: 'Searchlib app',
+      choices: [],
     },
     // url: {
     //   title: 'ES URL',
@@ -48,5 +47,5 @@ export const SearchBlockSchema = () => ({
     // },
   },
 
-  required: ['url'],
+  required: ['appName'],
 });

src/index.js

@@ -7,6 +7,8 @@ import SearchBlockView from './SearchBlock/SearchBlockView';
 import SearchBlockEdit from './SearchBlock/SearchBlockEdit';
 
 const applyConfig = (config) => {
+  config.settings.searchlib = registry;
+
   config.blocks.blocksConfig.searchlib = {
     id: 'searchlib',
     title: 'Searchlib',
@@ -24,13 +26,35 @@ const applyConfig = (config) => {
     },
   };
 
-  config.settings.searchlib = registry;
+  if (__SERVER__) {
+    const express = require('express');
+    const { createHandler } = require('./middleware/elasticsearch');
+
+    const handler = createHandler({
+      urlES: process.env.RAZZLE_PROXY_ES_DSN || 'http://localhost:9200/_all',
+      urlQA:
+        process.env.RAZZLE_PROXY_QA_DSN || 'http://localhost:8000/api/query',
+    });
+
+    const middleware = express.Router();
+    middleware.use(express.json());
+    middleware.use(express.urlencoded({ extended: true }));
+    middleware.all('**/_es/*', handler);
+    middleware.id = 'esProxyMiddleware';
+
+    config.settings.expressMiddleware = [
+      middleware,
+      ...config.settings.expressMiddleware,
+    ];
+  }
   return config;
 };
 
 export const installGlobalSearch = (config) => {
+  // config.settings.devProxyToApiPath = false;
   config.settings.searchlib = installConfig(config.settings.searchlib);
-  console.log(config.settings.searchlib);
+  config.settings.searchlib.searchui.globalsearch.elastic_index =
+    '_es/globalsearch';
   return config;
 };
 

src/middleware/download.js

@@ -0,0 +1,172 @@
+import { buildRequest } from '@eeacms/search';
+import stringify from 'csv-stringify';
+import es from 'elasticsearch';
+
+const SCROLL_TIME = '5m';
+const SCROLL_SIZE = 2000;
+
+const download = (es_config, appConfig, req, res) => {
+  const es_url = new URL(es_config);
+  const es_path_parts = es_url.pathname.split('/');
+
+  const es_index = es_path_parts.pop();
+  es_url.pathname = es_path_parts.join();
+
+  const es_host = es_url.href;
+
+  const download_mapping = appConfig.download_fields;
+
+  const { filters, searchTerm } = JSON.parse(req.body.query);
+
+  const dataQuery = buildRequest(
+    {
+      current: 1,
+      filters: filters,
+      resultsPerPage: SCROLL_SIZE,
+      searchTerm: searchTerm,
+      sortDirection: '',
+      sortField: '',
+    },
+    appConfig,
+  );
+  delete dataQuery.highlight;
+  delete dataQuery.aggs;
+  const linebreak = '\n';
+  const delimiter = ',';
+
+  const stringifier = stringify({ delimiter: delimiter });
+
+  res.setHeader('Content-Encoding', 'UTF-8');
+  res.setHeader('Content-Type', 'text/csv;charset=UTF-8');
+  res.setHeader('Content-disposition', 'attachment; filename=data.csv');
+  res.write('\uFEFF');
+  res.once('close', function () {
+    res.end();
+  });
+
+  var csv_header = [];
+  for (var i = 0; i < download_mapping.length; i++) {
+    csv_header.push(download_mapping[i].name);
+  }
+
+  res.write(stringifier.stringify(csv_header));
+  res.write(linebreak);
+
+  const client = new es.Client({
+    host: es_host,
+    type: 'stdio',
+    levels: ['error'],
+  });
+
+  var offset = 0;
+  dataQuery.size = SCROLL_SIZE;
+  // console.log('here');
+  // console.log(es_host);
+  // console.log(es_index);
+  // console.log(SCROLL_TIME);
+  // console.log(JSON.stringify(dataQuery));
+  client.search(
+    {
+      index: es_index,
+      scroll: SCROLL_TIME, // keep the search results "scrollable" for the time specified in SCROLL_TIME constant
+      body: dataQuery,
+    },
+
+    function getMoreUntilDone(error, data) {
+      if (error || data === undefined || data.hits === undefined) {
+        if (error) {
+          // eslint-disable-next-line
+          console.log('Error while downloading:', error);
+        } else {
+          // eslint-disable-next-line
+          console.log('Error in data while downloading:', data);
+        }
+        res.destroy();
+        return;
+      }
+      if (res.finished) {
+        return;
+      }
+      let total = 0;
+      total = data.hits.total.value;
+
+      if (total > offset) {
+        var chunk = '';
+
+        for (var i = 0; i < data.hits.hits.length; i++) {
+          const row = data.hits.hits[i];
+          let csv_row = [];
+          for (var j = 0; j < download_mapping.length; j++) {
+            let value = row['_source'][download_mapping[j].field];
+            if (value === undefined) {
+              value = '';
+            }
+            if (!Array.isArray(value)) {
+              value = [value];
+            }
+            const field_whitelist = download_mapping[j].values_whitelist;
+            if (field_whitelist !== undefined) {
+              var new_value = [];
+              for (
+                let value_count = 0;
+                value_count < value.length;
+                value_count++
+              ) {
+                if (field_whitelist.indexOf(value[value_count]) !== -1) {
+                  new_value.push(value[value_count]);
+                }
+              }
+              value = new_value;
+            }
+
+            var field_blacklist = download_mapping[j].values_blacklist;
+            if (field_blacklist !== undefined) {
+              let new_value = [];
+              for (
+                let value_count = 0;
+                value_count < value.length;
+                value_count++
+              ) {
+                if (field_blacklist.indexOf(value[value_count]) === -1) {
+                  new_value.push(value[value_count]);
+                }
+              }
+              value = new_value;
+            }
+
+            csv_row.push(value.toString());
+          }
+          chunk += stringifier.stringify(csv_row);
+          chunk += linebreak;
+        }
+        let write_res = res.write(chunk);
+
+        // ask elasticsearch for the next set of hits from this search
+        offset += SCROLL_SIZE;
+        if (!write_res) {
+          res.once('drain', function () {
+            client.scroll(
+              {
+                scroll: SCROLL_TIME,
+                scrollId: data._scroll_id,
+              },
+              getMoreUntilDone,
+            );
+          });
+        } else {
+          client.scroll(
+            {
+              scroll: SCROLL_TIME,
+              scrollId: data._scroll_id,
+            },
+            getMoreUntilDone,
+          );
+        }
+      } else {
+        res.end();
+      }
+    },
+  );
+};
+
+export default download;