log-elasticsearch 0.9.1.0 → 0.10.0.0

raw patch · 8 files changed

+190/−76 lines, 8 filesdep ~bloodhoundPVP ok

version bump matches the API change (PVP)

Dependency ranges changed: bloodhound

API changes (from Hackage documentation)

Files

• CHANGELOG.md +12/−5
• log-elasticsearch.cabal +3/−3
• src/Log/Backend/ElasticSearch/V1.hs +51/−34
• src/Log/Backend/ElasticSearch/V1/Internal.hs +8/−0
• src/Log/Backend/ElasticSearch/V1/Lens.hs +14/−0
• src/Log/Backend/ElasticSearch/V5.hs +80/−34
• src/Log/Backend/ElasticSearch/V5/Internal.hs +8/−0
• src/Log/Backend/ElasticSearch/V5/Lens.hs +14/−0

@@ -1,18 +1,25 @@+# log-elasticsearch-0.10.0.0 (2018-03-28)+* Expose `checkElasticSearchConnection` and `checkElasticSearchLogin`.+* Add config parameters for number of shards and replicas to+  `ElasticSearchConfig` (and make default of 4 shards and 1 replica+  explicit in 'defaultElasticSearchConfig').+ # log-elasticsearch-0.9.1.0 (2017-08-10)-* Add 'toEncoding' to 'ToJSON LogsMapping' instance+* Add `toEncoding` to `ToJSON LogsMapping` instance  # log-elasticsearch-0.9.0.1 (2017-06-19)-* 'withElasticSearchLogger' no longer fails when the Elasticsearch server is down.+* `withElasticSearchLogger` no longer fails when the Elasticsearch+  server is down.  # log-elasticsearch-0.9.0.0 (2017-05-04) * Now works with bloodhound-0.14.0.0 (#30).  # log-elasticsearch-0.8.1 (2017-03-27)-* Log.Backend.ElasticSearch.Internal now exports 'EsUsername' and-  'EsPassword'.+* `Log.Backend.ElasticSearch.Internal` now exports `EsUsername` and+  `EsPassword`.  # log-elasticsearch-0.8 (2017-03-16)-* Made ElasticSearchConfig an abstract type (#27).+* Made `ElasticSearchConfig` an abstract type (#27). * Added support for HTTPS and basic auth (#26).  # log-elasticsearch-0.7 (2016-11-25)

@@ -1,5 +1,5 @@ name:                log-elasticsearch-version:             0.9.1.0+version:             0.10.0.0 synopsis:            Structured logging solution (Elasticsearch back end)  description:         Elasticsearch back end for the 'log' library suite.@@ -34,13 +34,13 @@                        Log.Backend.ElasticSearch                        Log.Backend.ElasticSearch.Lens                        Log.Backend.ElasticSearch.Internal-  build-depends:       base <5,+  build-depends:       base >= 4.8 && <5,                        log-base >= 0.7,                        aeson >=1.0.0.0,                        aeson-pretty >=0.8.2,                        bytestring,                        base64-bytestring,-                       bloodhound >= 0.13 && < 0.15,+                       bloodhound >= 0.13 && < 0.16,                        deepseq,                        http-client,                        http-client-tls,

@@ -3,9 +3,13 @@     ElasticSearchConfig   , esServer   , esIndex+  , esShardCount+  , esReplicaCount   , esMapping   , esLogin   , esLoginInsecure+  , checkElasticSearchLogin+  , checkElasticSearchConnection   , defaultElasticSearchConfig   , withElasticSearchLogger   , elasticSearchLogger@@ -68,9 +72,20 @@   -> IO Word32        -- ^ Generate a random 32-bit word for use in                       -- document IDs.   -> IO Logger-elasticSearchLogger ElasticSearchConfig{..} genRandomWord = do-  checkElasticSearchLogin-  checkElasticSearchConnection+elasticSearchLogger esConf@ElasticSearchConfig{..} genRandomWord = do++  checkElasticSearchLogin esConf >>= \case+    Left (ex :: IOException) -> error . show $ ex+    Right _ -> return ()++  checkElasticSearchConnection esConf >>= \case+      Left (ex :: HttpException) ->+        hPutStrLn stderr $+                  "ElasticSearch: unexpected error: " <>+                  show ex <>+                  " (is ElasticSearch server running?)"+      Right () -> return ()+   indexRef <- newIORef $ IndexName T.empty   mkBulkLogger "ElasticSearch" (\msgs -> do     now <- getCurrentTime@@ -82,7 +97,7 @@     baseID <- (<>)       <$> (littleEndianRep <$> liftIO genRandomWord)       <*> pure (littleEndianRep . floor $ timeToDouble now)-    retryOnException . runBH_ $ do+    retryOnException . runBH_ esConf $ do       -- Elasticsearch index names are additionally indexed by date so that each       -- day is logged to a separate index to make log management easier.       let index = IndexName $ T.concat [@@ -96,11 +111,11 @@         -- index that already exists is harmless.         indexExists' <- indexExists index         unless indexExists' $ do-          -- Bloodhound is weird and won't let us create index using default-          -- settings, so pass these as the default ones.+          -- Note that Bloodhound won't let us create index using+          -- default settings.           let indexSettings = IndexSettings {-                  indexShards   = ShardCount 4-                , indexReplicas = ReplicaCount 1+                  indexShards   = ShardCount esShardCount+                , indexReplicas = ReplicaCount esReplicaCount                 }           void $ createIndex indexSettings index           reply <- putMapping index mapping LogsMapping@@ -144,29 +159,12 @@           void $ bulk (V.map (toBulk index baseID) dummyMsgs))     (elasticSearchSync indexRef)   where-    server  = Server esServer     mapping = MappingName esMapping      elasticSearchSync :: IORef IndexName -> IO ()     elasticSearchSync indexRef = do       indexName <- readIORef indexRef-      void . runBH_ $ refreshIndex indexName--    checkElasticSearchLogin :: IO ()-    checkElasticSearchLogin =-      when (isJust esLogin-            && not esLoginInsecure-            && not ("https:" `T.isPrefixOf` esServer)) $-        error $ "ElasticSearch: insecure login: "-          <> "Attempting to send login credentials over an insecure connection. "-          <> "Set esLoginInsecure = True to disable this check."--    checkElasticSearchConnection :: IO ()-    checkElasticSearchConnection = try (void $ runBH_ listIndices) >>= \case-      Left (ex::HttpException) ->-        hPutStrLn stderr $ "ElasticSearch: unexpected error: " <> show ex-          <> " (is ElasticSearch server running?)"-      Right () -> return ()+      void . runBH_ esConf $ refreshIndex indexName      retryOnException :: forall r. IO r -> IO r     retryOnException m = try m >>= \case@@ -180,14 +178,6 @@     timeToDouble :: UTCTime -> Double     timeToDouble = realToFrac . utcTimeToPOSIXSeconds -    runBH_ :: forall r. BH IO r -> IO r-    runBH_ f = do-      mgr <- newManager tlsManagerSettings-      let hook = maybe return (uncurry basicAuthHook) esLogin-      let env = (mkBHEnv server mgr) { bhRequestHook = hook }-      runBH env f--     jsonToBSL :: Value -> BSL.ByteString     jsonToBSL = encodePretty' defConfig { confIndent = Spaces 2 } @@ -281,3 +271,30 @@ decodeReply reply = case eitherDecode' $ responseBody reply of   Right body -> body   Left  err  -> object ["decoding_error" .= err]++-- | Check that login credentials are specified properly.+--+-- @since 0.10.1.0+checkElasticSearchLogin :: ElasticSearchConfig -> IO (Either IOException ())+checkElasticSearchLogin ElasticSearchConfig{..} =+  try $+    when (isJust esLogin+          && not esLoginInsecure+          && not ("https:" `T.isPrefixOf` esServer)) $+      error $ "ElasticSearch: insecure login: "+        <> "Attempting to send login credentials over an insecure connection. "+        <> "Set esLoginInsecure = True to disable this check."++-- | Check that we can connect to the ES server.+--+-- @since 0.10.1.0+checkElasticSearchConnection :: ElasticSearchConfig -> IO (Either HttpException ())+checkElasticSearchConnection esConf =+    try (void $ runBH_ esConf listIndices)++runBH_ :: forall r. ElasticSearchConfig -> BH IO r -> IO r+runBH_ ElasticSearchConfig{..} f = do+  mgr <- newManager tlsManagerSettings+  let hook = maybe return (uncurry basicAuthHook) esLogin+  let env = (mkBHEnv (Server esServer) mgr) { bhRequestHook = hook }+  runBH env f

@@ -15,6 +15,12 @@ data ElasticSearchConfig = ElasticSearchConfig {     esServer        :: !T.Text -- ^ Elasticsearch server address.   , esIndex         :: !T.Text -- ^ Elasticsearch index name.+  , esShardCount    :: !Int    -- ^ Elasticsearch shard count for the named index.+                               --+                               -- @since 0.10.0.0+  , esReplicaCount  :: !Int    -- ^ Elasticsearch replica count for the named index.+                               --+                               -- @since 0.10.0.0   , esMapping       :: !T.Text -- ^ Elasticsearch mapping name.   , esLogin         :: Maybe (EsUsername, EsPassword) -- ^ Elasticsearch basic authentication username and password.   , esLoginInsecure :: !Bool   -- ^ Allow basic authentication over non-TLS connections.@@ -25,6 +31,8 @@ defaultElasticSearchConfig = ElasticSearchConfig {   esServer        = "http://localhost:9200",   esIndex         = "logs",+  esShardCount    = 4,+  esReplicaCount  = 1,   esMapping       = "log",   esLogin         = Nothing,   esLoginInsecure = False

@@ -4,6 +4,8 @@     I.ElasticSearchConfig   , esServer   , esIndex+  , esShardCount+  , esReplicaCount   , esMapping   , esLogin   , esLoginInsecure@@ -26,6 +28,18 @@ -- | Elasticsearch index name. esIndex :: Lens' I.ElasticSearchConfig T.Text esIndex f esc = fmap (\x -> esc { I.esIndex = x }) $ f (I.esIndex esc)++-- | Elasticsearch shard count for the named index.+--+-- @since 0.10.0.0+esShardCount :: Lens' I.ElasticSearchConfig Int+esShardCount f esc = fmap (\x -> esc { I.esShardCount = x }) $ f (I.esShardCount esc)++-- | Elasticsearch replica count for the named index.+--+-- @since 0.10.0.0+esReplicaCount :: Lens' I.ElasticSearchConfig Int+esReplicaCount f esc = fmap (\x -> esc { I.esReplicaCount = x }) $ f (I.esReplicaCount esc)  -- | Elasticsearch mapping name. esMapping :: Lens' I.ElasticSearchConfig T.Text

@@ -3,9 +3,13 @@     ElasticSearchConfig   , esServer   , esIndex+  , esShardCount+  , esReplicaCount   , esMapping   , esLogin   , esLoginInsecure+  , checkElasticSearchLogin+  , checkElasticSearchConnection   , defaultElasticSearchConfig   , withElasticSearchLogger   , elasticSearchLogger@@ -34,6 +38,7 @@ import Prelude import System.IO import TextShow+import qualified Data.Aeson.Encoding as Aeson import qualified Data.ByteString as BS import qualified Data.ByteString.Base64 as B64 import qualified Data.ByteString.Lazy.Char8 as BSL@@ -67,9 +72,20 @@   -> IO Word32        -- ^ Generate a random 32-bit word for use in                       -- document IDs.   -> IO Logger-elasticSearchLogger ElasticSearchConfig{..} genRandomWord = do-  checkElasticSearchLogin-  checkElasticSearchConnection+elasticSearchLogger esConf@ElasticSearchConfig{..} genRandomWord = do++  checkElasticSearchLogin esConf >>= \case+    Left (ex :: IOException) -> error . show $ ex+    Right _ -> return ()++  checkElasticSearchConnection esConf >>= \case+      Left (ex :: HttpException) ->+        hPutStrLn stderr $+                  "ElasticSearch: unexpected error: " <>+                  show ex <>+                  " (is ElasticSearch server running?)"+      Right () -> return ()+   indexRef <- newIORef $ IndexName T.empty   mkBulkLogger "ElasticSearch" (\msgs -> do     now <- getCurrentTime@@ -81,7 +97,7 @@     baseID <- (<>)       <$> (littleEndianRep <$> liftIO genRandomWord)       <*> pure (littleEndianRep . floor $ timeToDouble now)-    retryOnException . runBH_ $ do+    retryOnException . runBH_ esConf $ do       -- Elasticsearch index names are additionally indexed by date so that each       -- day is logged to a separate index to make log management easier.       let index = IndexName $ T.concat [@@ -95,11 +111,11 @@         -- index that already exists is harmless.         indexExists' <- indexExists index         unless indexExists' $ do-          -- Bloodhound is weird and won't let us create index using default-          -- settings, so pass these as the default ones.+          -- Note that Bloodhound won't let us create index using+          -- default settings.           let indexSettings = IndexSettings {-                  indexShards   = ShardCount 4-                , indexReplicas = ReplicaCount 1+                  indexShards   = ShardCount esShardCount+                , indexReplicas = ReplicaCount esReplicaCount                 }           void $ createIndex indexSettings index           reply <- putMapping index mapping LogsMapping@@ -143,29 +159,12 @@           void $ bulk (V.map (toBulk index baseID) dummyMsgs))     (elasticSearchSync indexRef)   where-    server  = Server esServer     mapping = MappingName esMapping      elasticSearchSync :: IORef IndexName -> IO ()     elasticSearchSync indexRef = do       indexName <- readIORef indexRef-      void . runBH_ $ refreshIndex indexName--    checkElasticSearchLogin :: IO ()-    checkElasticSearchLogin =-      when (isJust esLogin-            && not esLoginInsecure-            && not ("https:" `T.isPrefixOf` esServer)) $-        error $ "ElasticSearch: insecure login: "-          <> "Attempting to send login credentials over an insecure connection. "-          <> "Set esLoginInsecure = True to disable this check."--    checkElasticSearchConnection :: IO ()-    checkElasticSearchConnection = try (void $ runBH_ listIndices) >>= \case-      Left (ex::HttpException) ->-        hPutStrLn stderr $ "ElasticSearch: unexpected error: " <> show ex-          <> " (is ElasticSearch server running?)"-      Right () -> return ()+      void . runBH_ esConf $ refreshIndex indexName      retryOnException :: forall r. IO r -> IO r     retryOnException m = try m >>= \case@@ -179,14 +178,6 @@     timeToDouble :: UTCTime -> Double     timeToDouble = realToFrac . utcTimeToPOSIXSeconds -    runBH_ :: forall r. BH IO r -> IO r-    runBH_ f = do-      mgr <- newManager tlsManagerSettings-      let hook = maybe return (uncurry basicAuthHook) esLogin-      let env = (mkBHEnv server mgr) { bhRequestHook = hook }-      runBH env f--     jsonToBSL :: Value -> BSL.ByteString     jsonToBSL = encodePretty' defConfig { confIndent = Spaces 2 } @@ -241,6 +232,34 @@         ]     ] +  toEncoding LogsMapping = Aeson.pairs $ mconcat+    [ Aeson.pair "properties" $ Aeson.pairs $ mconcat+      [ Aeson.pair "insertion_order"  $ Aeson.pairs $ mconcat+        [ "type" .= ("integer"::T.Text)+        ]+      , Aeson.pair "insertion_time" $ Aeson.pairs $ mconcat+        [ "type"   .= ("date"::T.Text)+        , "format" .= ("date_time"::T.Text)+        ]+      , Aeson.pair "time" $ Aeson.pairs $ mconcat+        [ "type"   .= ("date"::T.Text)+        , "format" .= ("date_time"::T.Text)+        ]+      , Aeson.pair "domain" $ Aeson.pairs $ mconcat+        [ "type" .= ("string"::T.Text)+        ]+      , Aeson.pair "level" $ Aeson.pairs $ mconcat+        [ "type" .= ("string"::T.Text)+        ]+      , Aeson.pair "component" $ Aeson.pairs $ mconcat+        [ "type" .= ("string"::T.Text)+        ]+      , Aeson.pair "message" $ Aeson.pairs $ mconcat+        [ "type" .= ("string"::T.Text)+        ]+      ]+    ]+ ----------------------------------------  littleEndianRep :: Word32 -> BS.ByteString@@ -252,3 +271,30 @@ decodeReply reply = case eitherDecode' $ responseBody reply of   Right body -> body   Left  err  -> object ["decoding_error" .= err]++-- | Check that login credentials are specified properly.+--+-- @since 0.10.1.0+checkElasticSearchLogin :: ElasticSearchConfig -> IO (Either IOException ())+checkElasticSearchLogin ElasticSearchConfig{..} =+  try $+    when (isJust esLogin+          && not esLoginInsecure+          && not ("https:" `T.isPrefixOf` esServer)) $+      error $ "ElasticSearch: insecure login: "+        <> "Attempting to send login credentials over an insecure connection. "+        <> "Set esLoginInsecure = True to disable this check."++-- | Check that we can connect to the ES server.+--+-- @since 0.10.1.0+checkElasticSearchConnection :: ElasticSearchConfig -> IO (Either HttpException ())+checkElasticSearchConnection esConf =+    try (void $ runBH_ esConf listIndices)++runBH_ :: forall r. ElasticSearchConfig -> BH IO r -> IO r+runBH_ ElasticSearchConfig{..} f = do+  mgr <- newManager tlsManagerSettings+  let hook = maybe return (uncurry basicAuthHook) esLogin+  let env = (mkBHEnv (Server esServer) mgr) { bhRequestHook = hook }+  runBH env f

@@ -15,6 +15,12 @@ data ElasticSearchConfig = ElasticSearchConfig {     esServer        :: !T.Text -- ^ Elasticsearch server address.   , esIndex         :: !T.Text -- ^ Elasticsearch index name.+  , esShardCount    :: !Int    -- ^ Elasticsearch shard count for the named index.+                               --+                               -- @since 0.10.0.0+  , esReplicaCount  :: !Int    -- ^ Elasticsearch replica count for the named index.+                               --+                               -- @since 0.10.0.0   , esMapping       :: !T.Text -- ^ Elasticsearch mapping name.   , esLogin         :: Maybe (EsUsername, EsPassword) -- ^ Elasticsearch basic authentication username and password.   , esLoginInsecure :: !Bool   -- ^ Allow basic authentication over non-TLS connections.@@ -25,6 +31,8 @@ defaultElasticSearchConfig = ElasticSearchConfig {   esServer        = "http://localhost:9200",   esIndex         = "logs",+  esShardCount    = 4,+  esReplicaCount  = 1,   esMapping       = "log",   esLogin         = Nothing,   esLoginInsecure = False

@@ -4,6 +4,8 @@     I.ElasticSearchConfig   , esServer   , esIndex+  , esShardCount+  , esReplicaCount   , esMapping   , esLogin   , esLoginInsecure@@ -26,6 +28,18 @@ -- | Elasticsearch index name. esIndex :: Lens' I.ElasticSearchConfig T.Text esIndex f esc = fmap (\x -> esc { I.esIndex = x }) $ f (I.esIndex esc)++-- | Elasticsearch shard count for the named index.+--+-- @since 0.10.0.0+esShardCount :: Lens' I.ElasticSearchConfig Int+esShardCount f esc = fmap (\x -> esc { I.esShardCount = x }) $ f (I.esShardCount esc)++-- | Elasticsearch replica count for the named index.+--+-- @since 0.10.0.0+esReplicaCount :: Lens' I.ElasticSearchConfig Int+esReplicaCount f esc = fmap (\x -> esc { I.esReplicaCount = x }) $ f (I.esReplicaCount esc)  -- | Elasticsearch mapping name. esMapping :: Lens' I.ElasticSearchConfig T.Text