Skip to content

Commit

Permalink
Update to support ES7
Browse files Browse the repository at this point in the history
  • Loading branch information
@AlexeyRaga
AlexeyRaga committed Oct 31, 2019
1 parent fc310b2 commit 176ca14
Show file tree
Hide file tree
Showing 20 changed files with 326 additions and 449 deletions.
14 changes: 10 additions & 4 deletions docker-compose.yml
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,15 @@
version: "2"
services:
elasticsearch1:
build: .
image: docker.elastic.co/elasticsearch/elasticsearch:7.4.1
container_name: elasticsearch1
environment:
- node.name=elasticsearch1
- cluster.name=docker-cluster
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- discovery.seed_hosts=elasticsearch2
- cluster.initial_master_nodes=elasticsearch1,elasticsearch2
ulimits:
memlock:
soft: -1
Expand All @@ -16,13 +19,16 @@ services:
- 9200:9200
networks:
- esnet

elasticsearch2:
build: .
image: docker.elastic.co/elasticsearch/elasticsearch:7.4.1
environment:
- node.name=elasticsearch2
- cluster.name=docker-cluster
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m"
- "discovery.zen.ping.unicast.hosts=elasticsearch1"
- discovery.seed_hosts=elasticsearch1
- cluster.initial_master_nodes=elasticsearch1,elasticsearch2
ulimits:
memlock:
soft: -1
Expand All @@ -32,4 +38,4 @@ services:
- esnet

networks:
esnet: {}
esnet:
3 changes: 0 additions & 3 deletions elasticsearch/Dockerfile
View file

This file was deleted.

3 changes: 0 additions & 3 deletions elasticsearch/config/elasticsearch.yml
View file

This file was deleted.

9 changes: 4 additions & 5 deletions examples/Tweet.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,15 +60,15 @@ main = runBH' $ do
-- set up index
_ <- createIndex indexSettings testIndex
True <- indexExists testIndex
_ <- putMapping testIndex testMapping TweetMapping
_ <- putMapping testIndex TweetMapping

-- create a tweet
resp <- indexDocument testIndex testMapping defaultIndexDocumentSettings exampleTweet (DocId "1")
resp <- indexDocument testIndex defaultIndexDocumentSettings exampleTweet (DocId "1")
liftIO (print resp)
-- Response {responseStatus = Status {statusCode = 201, statusMessage = "Created"}, responseVersion = HTTP/1.1, responseHeaders = [("Content-Type","application/json; charset=UTF-8"),("Content-Length","74")], responseBody = "{\"_index\":\"twitter\",\"_type\":\"tweet\",\"_id\":\"1\",\"_version\":1,\"created\":true}", responseCookieJar = CJ {expose = []}, responseClose' = ResponseClose}

-- bulk load
let stream = V.fromList [BulkIndex testIndex testMapping (DocId "2") (toJSON exampleTweet)]
let stream = V.fromList [BulkIndex testIndex (DocId "2") (toJSON exampleTweet)]
_ <- bulk stream
-- Bulk loads require an index refresh before new data is loaded.
_ <- refreshIndex testIndex
Expand All @@ -92,7 +92,7 @@ main = runBH' $ do
let boost = Nothing
let query = TermQuery (Term "user" "bitemyapp") boost
let search = mkSearch (Just query) boost
_ <- searchByType testIndex testMapping search
_ <- searchByType testIndex search

-- clean up
_ <- deleteTemplate templateName
Expand All @@ -104,5 +104,4 @@ main = runBH' $ do
testServer = Server "http://localhost:9200"
runBH' = withBH defaultManagerSettings testServer
testIndex = IndexName "twitter"
testMapping = MappingName "tweet"
indexSettings = IndexSettings (ShardCount 1) (ReplicaCount 0)
229 changes: 106 additions & 123 deletions src/Database/Bloodhound/Client.hs
View file

Large diffs are not rendered by default.

8 changes: 1 addition & 7 deletions src/Database/Bloodhound/Internal/Aggregation.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,15 +255,11 @@ instance ToJSON CollectionMode where

data ExecutionHint = Ordinals
| GlobalOrdinals
| GlobalOrdinalsHash
| GlobalOrdinalsLowCardinality
| Map deriving (Eq, Show)

instance ToJSON ExecutionHint where
toJSON Ordinals = "ordinals"
toJSON GlobalOrdinals = "global_ordinals"
toJSON GlobalOrdinalsHash = "global_ordinals_hash"
toJSON GlobalOrdinalsLowCardinality = "global_ordinals_low_cardinality"
toJSON Map = "map"

-- | See <https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#date-math> for more information.
Expand Down Expand Up @@ -433,7 +429,7 @@ data SearchHits a =

instance (FromJSON a) => FromJSON (SearchHits a) where
parseJSON (Object v) = SearchHits <$>
v .: "total" <*>
((v .: "total") >>= (.: "value")) <*>
v .: "max_score" <*>
v .: "hits"
parseJSON _ = empty
Expand All @@ -450,7 +446,6 @@ type SearchAfterKey = [Aeson.Value]

data Hit a =
Hit { hitIndex :: IndexName
, hitType :: MappingName
, hitDocId :: DocId
, hitScore :: Score
, hitSource :: Maybe a
Expand All @@ -461,7 +456,6 @@ data Hit a =
instance (FromJSON a) => FromJSON (Hit a) where
parseJSON (Object v) = Hit <$>
v .: "_index" <*>
v .: "_type" <*>
v .: "_id" <*>
v .: "_score" <*>
v .:? "_source" <*>
Expand Down
124 changes: 25 additions & 99 deletions src/Database/Bloodhound/Internal/Client.hs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import GHC.Enum
import Network.HTTP.Client
import Text.Read (Read (..))
import qualified Text.Read as TR
import Data.Map.Strict (Map)

import Database.Bloodhound.Internal.Analysis
import Database.Bloodhound.Internal.Newtypes
Expand Down Expand Up @@ -593,8 +594,7 @@ data MappingField =
if possible.
-}
data Mapping =
Mapping { typeName :: TypeName
, mappingFields :: [MappingField] }
Mapping { mappingFields :: [MappingField] }
deriving (Eq, Show)

data UpsertActionMetadata
Expand Down Expand Up @@ -649,21 +649,21 @@ instance FromJSON AllocationPolicy where
discussed further on github.
-}
data BulkOperation =
BulkIndex IndexName MappingName DocId Value
BulkIndex IndexName DocId Value
-- ^ Create the document, replacing it if it already exists.
| BulkIndexAuto IndexName MappingName Value
| BulkIndexAuto IndexName Value
-- ^ Create a document with an autogenerated id.
| BulkIndexEncodingAuto IndexName MappingName Encoding
| BulkIndexEncodingAuto IndexName Encoding
-- ^ Create a document with an autogenerated id. Use fast JSON encoding.
| BulkCreate IndexName MappingName DocId Value
| BulkCreate IndexName DocId Value
-- ^ Create a document, failing if it already exists.
| BulkCreateEncoding IndexName MappingName DocId Encoding
| BulkCreateEncoding IndexName DocId Encoding
-- ^ Create a document, failing if it already exists. Use fast JSON encoding.
| BulkDelete IndexName MappingName DocId
| BulkDelete IndexName DocId
-- ^ Delete the document
| BulkUpdate IndexName MappingName DocId Value
| BulkUpdate IndexName DocId Value
-- ^ Update the document, merging the new value with the existing one.
| BulkUpsert IndexName MappingName DocId UpsertPayload [UpsertActionMetadata]
| BulkUpsert IndexName DocId UpsertPayload [UpsertActionMetadata]
-- ^ Update the document if it already exists, otherwise insert it.
deriving (Eq, Show)

Expand Down Expand Up @@ -904,18 +904,18 @@ data VersionControl = NoVersionControl
-- care, as this could result in data loss.
deriving (Eq, Show, Ord)

{-| 'DocumentParent' is used to specify a parent document.
-}
newtype DocumentParent = DocumentParent DocId
deriving (Eq, Show)
data JoinRelation
= ParentDocument FieldName RelationName
| ChildDocument FieldName RelationName DocId
deriving (Show, Eq)

{-| 'IndexDocumentSettings' are special settings supplied when indexing
a document. For the best backwards compatiblity when new fields are
added, you should probably prefer to start with 'defaultIndexDocumentSettings'
-}
data IndexDocumentSettings =
IndexDocumentSettings { idsVersionControl :: VersionControl
, idsParent :: Maybe DocumentParent
, idsJoinRelation :: Maybe JoinRelation
} deriving (Eq, Show)

{-| Reasonable default settings. Chooses no version control and no parent.
Expand Down Expand Up @@ -1080,7 +1080,7 @@ data NodeStats = NodeStats {
, nodeStatsTransport :: NodeTransportStats
, nodeStatsFS :: NodeFSStats
, nodeStatsNetwork :: Maybe NodeNetworkStats
, nodeStatsThreadPool :: NodeThreadPoolsStats
, nodeStatsThreadPool :: Map Text NodeThreadPoolStats
, nodeStatsJVM :: NodeJVMStats
, nodeStatsProcess :: NodeProcessStats
, nodeStatsOS :: NodeOSStats
Expand Down Expand Up @@ -1164,26 +1164,6 @@ data NodeNetworkStats = NodeNetworkStats {
, nodeNetTCPActiveOpens :: Int
} deriving (Eq, Show)

data NodeThreadPoolsStats = NodeThreadPoolsStats {
nodeThreadPoolsStatsSnapshot :: NodeThreadPoolStats
, nodeThreadPoolsStatsBulk :: NodeThreadPoolStats
, nodeThreadPoolsStatsMerge :: NodeThreadPoolStats
, nodeThreadPoolsStatsGet :: NodeThreadPoolStats
, nodeThreadPoolsStatsManagement :: NodeThreadPoolStats
, nodeThreadPoolsStatsFetchShardStore :: Maybe NodeThreadPoolStats
, nodeThreadPoolsStatsOptimize :: Maybe NodeThreadPoolStats
, nodeThreadPoolsStatsFlush :: NodeThreadPoolStats
, nodeThreadPoolsStatsSearch :: NodeThreadPoolStats
, nodeThreadPoolsStatsWarmer :: NodeThreadPoolStats
, nodeThreadPoolsStatsGeneric :: NodeThreadPoolStats
, nodeThreadPoolsStatsSuggest :: Maybe NodeThreadPoolStats
, nodeThreadPoolsStatsRefresh :: NodeThreadPoolStats
, nodeThreadPoolsStatsIndex :: NodeThreadPoolStats
, nodeThreadPoolsStatsListener :: Maybe NodeThreadPoolStats
, nodeThreadPoolsStatsFetchShardStarted :: Maybe NodeThreadPoolStats
, nodeThreadPoolsStatsPercolate :: Maybe NodeThreadPoolStats
} deriving (Eq, Show)

data NodeThreadPoolStats = NodeThreadPoolStats {
nodeThreadPoolCompleted :: Int
, nodeThreadPoolLargest :: Int
Expand Down Expand Up @@ -1324,7 +1304,7 @@ data NodeIndicesStats = NodeIndicesStats {
, nodeIndicesStatsIndexingIndexCurrent :: Int
, nodeIndicesStatsIndexingIndexTime :: NominalDiffTime
, nodeIndicesStatsIndexingTotal :: Int
, nodeIndicesStatsStoreThrottleTime :: NominalDiffTime
, nodeIndicesStatsStoreThrottleTime :: Maybe NominalDiffTime
, nodeIndicesStatsStoreSize :: Bytes
, nodeIndicesStatsDocsDeleted :: Int
, nodeIndicesStatsDocsCount :: Int
Expand Down Expand Up @@ -1356,7 +1336,7 @@ data NodeInfo = NodeInfo {
, nodeInfoHTTP :: NodeHTTPInfo
, nodeInfoTransport :: NodeTransportInfo
, nodeInfoNetwork :: Maybe NodeNetworkInfo
, nodeInfoThreadPool :: NodeThreadPoolsInfo
, nodeInfoThreadPool :: Map Text NodeThreadPoolInfo
, nodeInfoJVM :: NodeJVMInfo
, nodeInfoProcess :: NodeProcessInfo
, nodeInfoOS :: NodeOSInfo
Expand Down Expand Up @@ -1407,25 +1387,10 @@ data NodeNetworkInterface = NodeNetworkInterface {
, nodeNetIfaceAddress :: Server
} deriving (Eq, Show)

data NodeThreadPoolsInfo = NodeThreadPoolsInfo {
nodeThreadPoolsRefresh :: NodeThreadPoolInfo
, nodeThreadPoolsManagement :: NodeThreadPoolInfo
, nodeThreadPoolsPercolate :: Maybe NodeThreadPoolInfo
, nodeThreadPoolsListener :: Maybe NodeThreadPoolInfo
, nodeThreadPoolsFetchShardStarted :: Maybe NodeThreadPoolInfo
, nodeThreadPoolsSearch :: NodeThreadPoolInfo
, nodeThreadPoolsFlush :: NodeThreadPoolInfo
, nodeThreadPoolsWarmer :: NodeThreadPoolInfo
, nodeThreadPoolsOptimize :: Maybe NodeThreadPoolInfo
, nodeThreadPoolsBulk :: NodeThreadPoolInfo
, nodeThreadPoolsSuggest :: Maybe NodeThreadPoolInfo
, nodeThreadPoolsMerge :: NodeThreadPoolInfo
, nodeThreadPoolsSnapshot :: NodeThreadPoolInfo
, nodeThreadPoolsGet :: NodeThreadPoolInfo
, nodeThreadPoolsFetchShardStore :: Maybe NodeThreadPoolInfo
, nodeThreadPoolsIndex :: NodeThreadPoolInfo
, nodeThreadPoolsGeneric :: NodeThreadPoolInfo
} deriving (Eq, Show)
data ThreadPool = ThreadPool {
nodeThreadPoolName :: Text
, nodeThreadPoolInfo :: NodeThreadPoolInfo
} deriving (Eq, Show)

data NodeThreadPoolInfo = NodeThreadPoolInfo {
nodeThreadPoolQueueSize :: ThreadPoolSize
Expand All @@ -1442,6 +1407,7 @@ data ThreadPoolSize = ThreadPoolBounded Int
data ThreadPoolType = ThreadPoolScaling
| ThreadPoolFixed
| ThreadPoolCached
| ThreadPoolFixedAutoQueueSize
deriving (Eq, Show)

data NodeJVMInfo = NodeJVMInfo {
Expand Down Expand Up @@ -1953,26 +1919,6 @@ instance FromJSON NodeNetworkStats where
<*> tcp .: "passive_opens"
<*> tcp .: "active_opens"

instance FromJSON NodeThreadPoolsStats where
parseJSON = withObject "NodeThreadPoolsStats" parse
where
parse o = NodeThreadPoolsStats <$> o .: "snapshot"
<*> o .: "bulk"
<*> o .: "force_merge"
<*> o .: "get"
<*> o .: "management"
<*> o .:? "fetch_shard_store"
<*> o .:? "optimize"
<*> o .: "flush"
<*> o .: "search"
<*> o .: "warmer"
<*> o .: "generic"
<*> o .:? "suggest"
<*> o .: "refresh"
<*> o .: "index"
<*> o .:? "listener"
<*> o .:? "fetch_shard_started"
<*> o .:? "percolate"
instance FromJSON NodeThreadPoolStats where
parseJSON = withObject "NodeThreadPoolStats" parse
where
Expand Down Expand Up @@ -2168,7 +2114,7 @@ instance FromJSON NodeIndicesStats where
<*> indexing .: "index_current"
<*> (unMS <$> indexing .: "index_time_in_millis")
<*> indexing .: "index_total"
<*> (unMS <$> store .: "throttle_time_in_millis")
<*> (fmap unMS <$> store .:? "throttle_time_in_millis")
<*> store .: "size_in_bytes"
<*> docs .: "deleted"
<*> docs .: "count"
Expand Down Expand Up @@ -2289,27 +2235,6 @@ instance FromJSON JVMMemoryInfo where
<*> o .: "heap_max_in_bytes"
<*> o .: "heap_init_in_bytes"

instance FromJSON NodeThreadPoolsInfo where
parseJSON = withObject "NodeThreadPoolsInfo" parse
where
parse o = NodeThreadPoolsInfo <$> o .: "refresh"
<*> o .: "management"
<*> o .:? "percolate"
<*> o .:? "listener"
<*> o .:? "fetch_shard_started"
<*> o .: "search"
<*> o .: "flush"
<*> o .: "warmer"
<*> o .:? "optimize"
<*> o .: "bulk"
<*> o .:? "suggest"
<*> o .: "force_merge"
<*> o .: "snapshot"
<*> o .: "get"
<*> o .:? "fetch_shard_store"
<*> o .: "index"
<*> o .: "generic"

instance FromJSON NodeThreadPoolInfo where
parseJSON = withObject "NodeThreadPoolInfo" parse
where
Expand Down Expand Up @@ -2397,6 +2322,7 @@ instance FromJSON ThreadPoolType where
parse "scaling" = return ThreadPoolScaling
parse "fixed" = return ThreadPoolFixed
parse "cached" = return ThreadPoolCached
parse "fixed_auto_queue_size" = return ThreadPoolFixedAutoQueueSize
parse e = fail ("Unexpected thread pool type" <> T.unpack e)

instance FromJSON NodeTransportInfo where
Expand Down
Loading

0 comments on commit 176ca14

Please sign in to comment.