Merge pull request #100 from DivanteLtd/release/v1.11

Release/v1.11

CHANGELOG.md

@@ -4,7 +4,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
-# [1.10] - 2019.07.10
+## [1.11] - 2020.04.15
+### Added
+ - Elastic7 support - @pkarw (#96) 
+ - Add product attributes_metadata - @andrzejewsky (#99)
+
+## [1.10] - 2019.07.10
 ### Added
  - Added optional Redis Auth functionality. - @rain2o (#42)
  - MSI support - @dimasch (#86)

README.md

@@ -54,6 +54,28 @@ npm run dev
 
 The key command is `docker-compose up -d` which runs the ElasticSearch and Redis instances - both required by `mage2vuestorefront`
 
+### Elastic 7 Support
+
+By default, Vue Storefront API docker files and config are based on Elastic 5.6. We plan to change the default Elastic version to 7 with the 1.11 stable release. As for now, the [Elastic 7 support](https://github.com/DivanteLtd/vue-storefront-api/pull/342) is marked as **experimental**. 
+
+In order to index data to Elastic 7 please make sure you set the proper `apiVersion` in the `config.js`:
+
+```js
+  elasticsearch: {
+    apiVersion: process.env.ELASTICSEARCH_API_VERSION || '7.1'
+  },
+```
+
+or just use the env variable:
+
+```bash
+export ELASTICSEARCH_API_VERSION=7.1
+```
+
+Starting from [Elasitc 6 and 7](https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html) we can have **just single** document type per single index. Vue Storefront used to have `product`, `category` ... types defined in the `vue_storefront_catalog`.
+
+From now on, we're using the separate indexes per each entity type. The convention is: `${indexName}_${entityType}`. If your' **logical index name** is `vue_storefront_catalog` then it will be mapped to the **physical indexes** of: `vue_storefront_catalog_product`, `vue_storefront_catalog_category` ...
+
 ### Initial Vue Storefront import
 
 Now, You're ready to run the importer. Please check the [config file](https://github.com/DivanteLtd/mage2vuestorefront/blob/master/src/config.js). You may setup the Magento access data and URLs by config values or ENV variables.

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '2'
 services:
   esm1:
-    image: elasticsearch:2.4.6
+    image: elasticsearch:7.3.2
     container_name: esm1
     environment:
       - cluster.name=docker-cluster
@@ -19,7 +19,7 @@ services:
     networks:
       - esnet
   esm2:
-    image: elasticsearch:5.5
+    image: elasticsearch:7.3.2
     container_name: esm2
     environment:
       - cluster.name=docker-cluster

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mage2vuestorefront",
   "private": true,
-  "version": "1.7.1",
+  "version": "1.11.0",
   "description": "Magento sync for products, categories, users and orders",
   "author": "Piotr Karwatka",
   "license": "MIT",
@@ -10,10 +10,10 @@
     "test": "echo \"Error: no test specified\" && exit 1"
   },
   "dependencies": {
+    "@elastic/elasticsearch": "^7.3.0",
     "agentkeepalive": "^3.3.0",
     "body-parser": "^1.17.1",
     "commander": "^2.18.0",
-    "elasticsearch": "^15.2.0",
     "elasticsearch-deletebyquery": "^1.6.0",
     "express": "^4.15.2",
     "jsonfile": "^4.0.0",

src/adapters/magento/product.js

@@ -48,6 +48,7 @@ class ProductAdapter extends AbstractMagentoAdapter {
     this.links_sync = true;
     this.configurable_sync = true;
     this.is_federated = true; // by default use federated behaviour
+
   }
 
   getEntityType() {
@@ -226,6 +227,73 @@ class ProductAdapter extends AbstractMagentoAdapter {
   isNumeric(value) {
     return /^\d+$/.test(value);
   }
+
+  processAttributes(customAttributes, configurableOptions) {
+    const loadFromCache = (key) => new Promise((resolve) =>
+      this.cache.get(key, (err, serializedAtr) => resolve(JSON.parse(serializedAtr)))
+    )
+    const findConfigurableOptionsValues = attributeId => {
+      const attribute = configurableOptions.find(
+        opt => parseInt(opt.attribute_id) === parseInt(attributeId)
+      )
+
+      if (attribute) {
+        return attribute.values.map(val => parseInt(val.value_index))
+      }
+
+      return []
+    }
+
+    const findCustomAttributesValues = (attributeCode) => {
+      const attribute = customAttributes.find(
+        opt => opt.attribute_code === attributeCode
+      )
+
+      return attribute ? [parseInt(attribute.value)] : []
+    }
+
+    const findOptionValues = option => ([
+      ...findConfigurableOptionsValues(option.attribute_id),
+      ...findCustomAttributesValues(option.attribute_code)
+    ])
+
+    const selectFields = (res) => res.map(o => {
+      const attributeOptionValues = findOptionValues(o)
+      const options = o.options.filter(opt => attributeOptionValues.includes(parseInt(opt.value)))
+
+      return {
+        is_visible_on_front: o.is_visible_on_front,
+        is_visible: o.is_visible,
+        default_frontend_label: o.default_frontend_label,
+        attribute_id: o.attribute_id,
+        entity_type_id: o.entity_type_id,
+        id: o.id,
+        frontend_input: o.frontend_input,
+        is_user_defined: o.is_user_defined,
+        is_comparable: o.is_comparable,
+        attribute_code: o.attribute_code,
+        slug: o.slug,
+        options
+      }
+    })
+
+    const attributeCodes = customAttributes.map(obj => new Promise((resolve) => {
+      const key = util.format(CacheKeys.CACHE_KEY_ATTRIBUTE, obj.attribute_code);
+      loadFromCache(key).then(resolve)
+    }))
+
+    const attributeIds = configurableOptions.map(obj => new Promise((resolve) => {
+      const key = util.format(CacheKeys.CACHE_KEY_ATTRIBUTE, obj.attribute_id);
+      loadFromCache(key).then(resolve)
+    }))
+
+    return Promise.all([
+      ...attributeCodes,
+      ...attributeIds
+    ])
+      .then(selectFields)
+  }
+
   /**
    *
    * @param {Object} item
@@ -247,7 +315,6 @@ class ProductAdapter extends AbstractMagentoAdapter {
       item[customAttribute.attribute_code] = attrValue;
     }
     item.slug = _slugify(item.name + '-' + item.id);
-    item.custom_attributes = null;
 
     return new Promise((done, reject) => {
       // TODO: add denormalization of productcategories into product categories
@@ -417,8 +484,7 @@ class ProductAdapter extends AbstractMagentoAdapter {
                   visibility: prOption.visibility,
                   name: prOption.name,
                   price: prOption.price,
-                  tier_prices: prOption.tier_prices
-                  // custom_attributes: prOption.custom_attributes
+                  tier_prices: prOption.tier_prices,
                 };
 
                 if (prOption.custom_attributes) {
@@ -511,6 +577,16 @@ class ProductAdapter extends AbstractMagentoAdapter {
         })
       }
 
+      subSyncPromises.push(() => {
+        return new Promise((resolve) => {
+          this.processAttributes(item.custom_attributes, item.configurable_options || []).then(res => {
+            item.attributes_metadata = res
+            item.custom_attributes = null
+            resolve(item)
+          })
+        })
+      });
+
       // CATEGORIES SYNC
       subSyncPromises.push(() => {
         return new Promise((resolve, reject) => {

src/adapters/nosql/elasticsearch.js

@@ -1,6 +1,6 @@
 'use strict';
 const AbstractNosqlAdapter = require('./abstract');
-const elasticsearch = require('elasticsearch');
+const elasticsearch = require('@elastic/elasticsearch');
 const AgentKeepAlive = require('agentkeepalive');
 const AgentKeepAliveHttps = require('agentkeepalive').HttpsAgent;
 
@@ -28,6 +28,32 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
     this.updateDocument.bind(this);
   }
 
+  /**
+   * Get physical Elastic index name; since 7.x we're adding an entity name to get real index: vue_storefront_catalog_product, vue_storefront_catalog_category and so on
+   * @param {*} baseIndexName 
+   * @param {*} config 
+   */
+  getPhysicalIndexName(collectionName, config) {
+    if (parseInt(config.elasticsearch.apiVersion) >= 6) {
+      return `${config.db.indexName}_${collectionName}`
+    } else {
+      return config.db.indexName
+    }
+  }
+
+  /**
+   * Get physical Elastic type name; since 7.x index can have one type _doc
+   * @param {*} baseIndexName 
+   * @param {*} config 
+   */
+  getPhysicalTypeName(collectionName, config) {
+    if (parseInt(config.elasticsearch.apiVersion) >= 6) {
+      return `_doc`
+    } else {
+      return collectionName
+    }
+  }  
+
   /**
    * Close the nosql database connection - abstract to the driver
    */
@@ -42,11 +68,14 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
   */  
   getDocuments(collectionName, queryBody) {
     return new Promise((resolve, reject) => {
-      this.db.search({ // requires ES 5.5
-        index: this.config.db.indexName,
-        type: collectionName,
-          body: queryBody
-      }, function (error, response) {
+      const searchQueryBody = {
+        index: this.getPhysicalIndexName(collectionName, this.config),
+        body: queryBody
+      }
+      if (parseInt(this.config.elasticsearch.apiVersion) < 6)
+       searchQueryBody.type  = this.getPhysicalTypeName(collectionName, this.config)
+
+      this.db.search(searchQueryBody, function (error, { body: response }) {
         if (error) reject(error);
         if (response.hits && response.hits.hits) {
           resolve(response.hits.hits.map(h => h._source))
@@ -62,20 +91,21 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
    * @param {object} item document to be updated in database
    */
   updateDocument(collectionName, item) {
-
     const itemtbu = item;
-
-    this.db.update({
-      index: this.config.db.indexName,
+    const updateRequestBody = {
+      index: this.getPhysicalIndexName(collectionName, this.config),
       id: item.id,
-      type: collectionName,
       body: {
         // put the partial document under the `doc` key
         upsert: itemtbu,
         doc: itemtbu
 
       }
-    }, function (error, response) {
+    }
+    if (parseInt(this.config.elasticsearch.apiVersion) < 6)
+      updateRequestBody.type = this.getPhysicalTypeName(collectionName, this.config)
+
+    this.db.update(updateRequestBody, function (error, response) {
       if (error)
         throw new Error(error);
     });
@@ -89,11 +119,10 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
   cleanupByTransactionkey(collectionName, transactionKey) {
 
     if (transactionKey) {
-      this.db.deleteByQuery({ // requires ES 5.5
-        index: this.config.db.indexName,
+      const query = {
+        index: this.getPhysicalIndexName(collectionName, this.config),
         conflicts: 'proceed',
-        type: collectionName,
-         body: {
+        body: {
           query: {
             bool: {
               must_not: {
@@ -102,9 +131,12 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
             }
           }
         }
-      }, function (error, response) {
+      };
+      if (parseInt(this.config.elasticsearch.apiVersion) < 6)
+        query.type = this.getPhysicalTypeName(collectionName, this.config)
+
+      this.db.deleteByQuery(query, function (error, response) {
         if (error) throw new Error(error);
-        logger.info(response);
       });
     }
   }
@@ -120,12 +152,15 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
     let bulkSize = 500;
 
     for (let doc of items) {
+      const query = {
+        _index: this.getPhysicalIndexName(collectionName, this.config),
+        _id: doc.id,
+      };
+      if (parseInt(this.config.elasticsearch.apiVersion) < 6)
+        query.type = this.getPhysicalTypeName(collectionName, this.config)
+
       requests.push({
-        update: {
-          _index: this.config.db.indexName,
-          _id: doc.id,
-          _type: collectionName,
-        }
+        update: query
       });
 
       requests.push({
@@ -161,15 +196,15 @@ class ElasticsearchAdapter extends AbstractNosqlAdapter {
 
     if (!global.es) {
       this.db = new elasticsearch.Client({
-        host: this.config.db.url,
-        log: 'error',
+        node: this.config.db.url,
+        log: 'debug',
         apiVersion: this.config.elasticsearch.apiVersion,
 
         maxRetries: 10,
         keepAlive: true,
         maxSockets: 10,
         minSockets: 10,
-        requestTimeout: 1800000,
+        requestTimeout: 1800000,        
 
         createNodeAgent: function (connection, config) {
           if (connection.useSsl) {

src/config.js

@@ -47,7 +47,7 @@ module.exports = {
 
   product: {
     expandConfigurableFilters: ['manufacturer'],
-    synchronizeCatalogSpecialPrices: process.env.PRODUCTS_SPECIAL_PRICES || false,
+    synchronizeCatalogSpecialPrices: process.env.PRODUCTS_SPECIAL_PRICES || true,
     renderCatalogRegularPrices: process.env.PRODUCTS_RENDER_PRICES || false,
     excludeDisabledProducts: process.env.PRODUCTS_EXCLUDE_DISABLED || false
   },