Retrieve related WFS features from service metadata

__tests__/lib/jobs/consolidate-record/feature-types.js → __tests__/lib/utils/feature-types.js

@@ -1,6 +1,6 @@
-const {normalizeTypeName} = require('../../../../lib/jobs/consolidate-record/feature-types')
+const {normalizeTypeName} = require('../../../lib/utils/feature-types')
 
-describe('lib.jobs.consolidate-records.feature-types', () => {
+describe('lib.utils.feature-types', () => {
   describe('normalizeTypeName', () => {
     it('should normalize the typeNames', () => {
       const testCases = [

lib/jobs/consolidate-record/feature-types.js

@@ -1,16 +1,13 @@
-'use strict'
-
 const mongoose = require('mongoose')
-const {strRight} = require('underscore.string')
 const _ = require('lodash')
 
-function normalizeTypeName(typeName) {
-  return strRight(typeName, ':').toLowerCase()
-}
+const {normalizeTypeName} = require('../../utils/feature-types')
 
-async function getFeatureType(featureType) {
-  const FeatureType = mongoose.model('FeatureType')
+const ConsolidatedRecord = mongoose.model('ConsolidatedRecord')
+const Service = mongoose.model('Service')
+const FeatureType = mongoose.model('FeatureType')
 
+async function getFeatureType(featureType) {
   const typeName = normalizeTypeName(featureType.typeName)
   const featureTypes = await FeatureType.findByService(featureType.serviceId)
   const matchingFeatureType = featureTypes.find(ft => {
@@ -20,10 +17,30 @@ async function getFeatureType(featureType) {
   return matchingFeatureType
 }
 
-function resolveFeatureTypes(featureTypes = []) {
-  const Service = mongoose.model('Service')
+async function getRelatedFeatureTypes(recordId) {
+  const related = await ConsolidatedRecord.collection.find({
+    'metadata.type': 'service',
+    'metadata.serviceType': 'download',
+    'metadata.serviceProtocol': 'wfs',
+    'metadata.featureTypes.relatedTo': recordId
+  }, {
+    projection: {
+      'metadata.featureTypes': 1
+    }
+  }).toArray()
+
+  return _(related)
+    .map(record => record.metadata.featureTypes)
+    .flatten()
+    .filter(ft => ft.relatedTo === recordId)
+    .value()
+}
 
-  featureTypes = _.uniqBy(featureTypes, featureType => {
+async function resolveFeatureTypes(recordId, featureTypes = []) {
+  featureTypes = _.uniqBy([
+    ...await getRelatedFeatureTypes(recordId),
+    ...featureTypes
+  ], featureType => {
     return `${featureType.serviceURL}@${featureType.typeName}`.toLowerCase()
   })
 
@@ -80,4 +97,8 @@ async function retrieveResourcesFromFeatureTypes(featureTypes = []) {
     .value()
 }
 
-module.exports = {resolveFeatureTypes, normalizeTypeName, retrieveResourcesFromFeatureTypes}
+module.exports = {
+  resolveFeatureTypes,
+  normalizeTypeName,
+  retrieveResourcesFromFeatureTypes
+}

lib/jobs/consolidate-record/index.js

@@ -52,8 +52,7 @@ async function consolidateRecord({data: {recordId, freshness}}) {
     const links = await resolveLinks(metadata.links, record._links)
     record.set('_links', links)
 
-    const featureTypes = await resolveFeatureTypes(metadata.featureTypes)
-    record.set('_featureTypes', featureTypes)
+    const featureTypes = await resolveFeatureTypes(recordId, metadata.featureTypes)
 
     record.set('resources', [
       ...await retrieveResourcesFromLinks(links),

lib/jobs/lookup-wfs/index.js

@@ -5,7 +5,7 @@ const mongoose = require('mongoose')
 const ServiceSyncJob = require('../../sync-job')
 
 const FeatureType = mongoose.model('FeatureType')
-const Record = mongoose.model('ConsolidatedRecord')
+const ConsolidatedRecord = mongoose.model('ConsolidatedRecord')
 
 function filterValidFeatureTypes(featureTypes = []) {
   return featureTypes.filter(ft => ft.name && ft.name.length > 0)
@@ -21,15 +21,15 @@ class WfsLookupJob extends ServiceSyncJob {
   }
 
   async propagateChanges() {
-    const records = await Record.distinct('recordId', {
-      '_featureTypes.serviceId': this.service._id
+    const records = await ConsolidatedRecord.distinct('recordId', {
+      'resources.serviceId': this.service._id
     }).exec()
 
     debug(`Propagating changes to ${records.length} related records`)
 
     return Promise.all(
       records.map(recordId => {
-        return Record.triggerUpdated(recordId, 'feature types updated')
+        return ConsolidatedRecord.triggerUpdated(recordId, 'feature types updated')
       })
     )
   }
@@ -60,8 +60,8 @@ class WfsLookupJob extends ServiceSyncJob {
   }
 
   async _sync() {
-    const count = await Record.collection.countDocuments({
-      '_featureTypes.serviceId': this.service._id
+    const count = await ConsolidatedRecord.collection.countDocuments({
+      'resources.serviceId': this.service._id
     })
 
     if (count === 0) {

lib/models/ConsolidatedRecord.js

@@ -94,8 +94,20 @@ const schema = new Schema({
 
 /* Indexes */
 schema.index({createdAt: -1})
-schema.index({'_featureTypes.serviceId': 1})
 schema.index({'facets.name': 1, 'facets.value': 1})
+schema.index({'resources.serviceId': 1})
+
+schema.index({
+  'metadata.type': 1,
+  'metadata.serviceType': 1,
+  'metadata.serviceProtocol': 1,
+  'metadata.featureTypes.relatedTo': 1
+}, {
+  partialFilterExpression: {
+    'metadata.type': 'service',
+    'metadata.serviceType': 'download'
+  }
+})
 
 const textIndexOptions = {
   default_language: 'french',

lib/utils/feature-types.js

@@ -0,0 +1,9 @@
+const {strRight} = require('underscore.string')
+
+function normalizeTypeName(typeName) {
+  return strRight(typeName, ':').toLowerCase()
+}
+
+module.exports = {
+  normalizeTypeName
+}