diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,25 @@ +# log-elasticsearch-0.10.0.0 (2018-03-28) +* Expose `checkElasticSearchConnection` and `checkElasticSearchLogin`. +* Add config parameters for number of shards and replicas to + `ElasticSearchConfig` (and make default of 4 shards and 1 replica + explicit in 'defaultElasticSearchConfig'). + # log-elasticsearch-0.9.1.0 (2017-08-10) -* Add 'toEncoding' to 'ToJSON LogsMapping' instance +* Add `toEncoding` to `ToJSON LogsMapping` instance # log-elasticsearch-0.9.0.1 (2017-06-19) -* 'withElasticSearchLogger' no longer fails when the Elasticsearch server is down. +* `withElasticSearchLogger` no longer fails when the Elasticsearch + server is down. # log-elasticsearch-0.9.0.0 (2017-05-04) * Now works with bloodhound-0.14.0.0 (#30). # log-elasticsearch-0.8.1 (2017-03-27) -* Log.Backend.ElasticSearch.Internal now exports 'EsUsername' and - 'EsPassword'. +* `Log.Backend.ElasticSearch.Internal` now exports `EsUsername` and + `EsPassword`. # log-elasticsearch-0.8 (2017-03-16) -* Made ElasticSearchConfig an abstract type (#27). +* Made `ElasticSearchConfig` an abstract type (#27). * Added support for HTTPS and basic auth (#26). # log-elasticsearch-0.7 (2016-11-25) diff --git a/log-elasticsearch.cabal b/log-elasticsearch.cabal --- a/log-elasticsearch.cabal +++ b/log-elasticsearch.cabal @@ -1,5 +1,5 @@ name: log-elasticsearch -version: 0.9.1.0 +version: 0.10.0.0 synopsis: Structured logging solution (Elasticsearch back end) description: Elasticsearch back end for the 'log' library suite. @@ -34,13 +34,13 @@ Log.Backend.ElasticSearch Log.Backend.ElasticSearch.Lens Log.Backend.ElasticSearch.Internal - build-depends: base <5, + build-depends: base >= 4.8 && <5, log-base >= 0.7, aeson >=1.0.0.0, aeson-pretty >=0.8.2, bytestring, base64-bytestring, - bloodhound >= 0.13 && < 0.15, + bloodhound >= 0.13 && < 0.16, deepseq, http-client, http-client-tls, diff --git a/src/Log/Backend/ElasticSearch/V1.hs b/src/Log/Backend/ElasticSearch/V1.hs --- a/src/Log/Backend/ElasticSearch/V1.hs +++ b/src/Log/Backend/ElasticSearch/V1.hs @@ -3,9 +3,13 @@ ElasticSearchConfig , esServer , esIndex + , esShardCount + , esReplicaCount , esMapping , esLogin , esLoginInsecure + , checkElasticSearchLogin + , checkElasticSearchConnection , defaultElasticSearchConfig , withElasticSearchLogger , elasticSearchLogger @@ -68,9 +72,20 @@ -> IO Word32 -- ^ Generate a random 32-bit word for use in -- document IDs. -> IO Logger -elasticSearchLogger ElasticSearchConfig{..} genRandomWord = do - checkElasticSearchLogin - checkElasticSearchConnection +elasticSearchLogger esConf@ElasticSearchConfig{..} genRandomWord = do + + checkElasticSearchLogin esConf >>= \case + Left (ex :: IOException) -> error . show $ ex + Right _ -> return () + + checkElasticSearchConnection esConf >>= \case + Left (ex :: HttpException) -> + hPutStrLn stderr $ + "ElasticSearch: unexpected error: " <> + show ex <> + " (is ElasticSearch server running?)" + Right () -> return () + indexRef <- newIORef $ IndexName T.empty mkBulkLogger "ElasticSearch" (\msgs -> do now <- getCurrentTime @@ -82,7 +97,7 @@ baseID <- (<>) <$> (littleEndianRep <$> liftIO genRandomWord) <*> pure (littleEndianRep . floor $ timeToDouble now) - retryOnException . runBH_ $ do + retryOnException . runBH_ esConf $ do -- Elasticsearch index names are additionally indexed by date so that each -- day is logged to a separate index to make log management easier. let index = IndexName $ T.concat [ @@ -96,11 +111,11 @@ -- index that already exists is harmless. indexExists' <- indexExists index unless indexExists' $ do - -- Bloodhound is weird and won't let us create index using default - -- settings, so pass these as the default ones. + -- Note that Bloodhound won't let us create index using + -- default settings. let indexSettings = IndexSettings { - indexShards = ShardCount 4 - , indexReplicas = ReplicaCount 1 + indexShards = ShardCount esShardCount + , indexReplicas = ReplicaCount esReplicaCount } void $ createIndex indexSettings index reply <- putMapping index mapping LogsMapping @@ -144,29 +159,12 @@ void $ bulk (V.map (toBulk index baseID) dummyMsgs)) (elasticSearchSync indexRef) where - server = Server esServer mapping = MappingName esMapping elasticSearchSync :: IORef IndexName -> IO () elasticSearchSync indexRef = do indexName <- readIORef indexRef - void . runBH_ $ refreshIndex indexName - - checkElasticSearchLogin :: IO () - checkElasticSearchLogin = - when (isJust esLogin - && not esLoginInsecure - && not ("https:" `T.isPrefixOf` esServer)) $ - error $ "ElasticSearch: insecure login: " - <> "Attempting to send login credentials over an insecure connection. " - <> "Set esLoginInsecure = True to disable this check." - - checkElasticSearchConnection :: IO () - checkElasticSearchConnection = try (void $ runBH_ listIndices) >>= \case - Left (ex::HttpException) -> - hPutStrLn stderr $ "ElasticSearch: unexpected error: " <> show ex - <> " (is ElasticSearch server running?)" - Right () -> return () + void . runBH_ esConf $ refreshIndex indexName retryOnException :: forall r. IO r -> IO r retryOnException m = try m >>= \case @@ -180,14 +178,6 @@ timeToDouble :: UTCTime -> Double timeToDouble = realToFrac . utcTimeToPOSIXSeconds - runBH_ :: forall r. BH IO r -> IO r - runBH_ f = do - mgr <- newManager tlsManagerSettings - let hook = maybe return (uncurry basicAuthHook) esLogin - let env = (mkBHEnv server mgr) { bhRequestHook = hook } - runBH env f - - jsonToBSL :: Value -> BSL.ByteString jsonToBSL = encodePretty' defConfig { confIndent = Spaces 2 } @@ -281,3 +271,30 @@ decodeReply reply = case eitherDecode' $ responseBody reply of Right body -> body Left err -> object ["decoding_error" .= err] + +-- | Check that login credentials are specified properly. +-- +-- @since 0.10.1.0 +checkElasticSearchLogin :: ElasticSearchConfig -> IO (Either IOException ()) +checkElasticSearchLogin ElasticSearchConfig{..} = + try $ + when (isJust esLogin + && not esLoginInsecure + && not ("https:" `T.isPrefixOf` esServer)) $ + error $ "ElasticSearch: insecure login: " + <> "Attempting to send login credentials over an insecure connection. " + <> "Set esLoginInsecure = True to disable this check." + +-- | Check that we can connect to the ES server. +-- +-- @since 0.10.1.0 +checkElasticSearchConnection :: ElasticSearchConfig -> IO (Either HttpException ()) +checkElasticSearchConnection esConf = + try (void $ runBH_ esConf listIndices) + +runBH_ :: forall r. ElasticSearchConfig -> BH IO r -> IO r +runBH_ ElasticSearchConfig{..} f = do + mgr <- newManager tlsManagerSettings + let hook = maybe return (uncurry basicAuthHook) esLogin + let env = (mkBHEnv (Server esServer) mgr) { bhRequestHook = hook } + runBH env f diff --git a/src/Log/Backend/ElasticSearch/V1/Internal.hs b/src/Log/Backend/ElasticSearch/V1/Internal.hs --- a/src/Log/Backend/ElasticSearch/V1/Internal.hs +++ b/src/Log/Backend/ElasticSearch/V1/Internal.hs @@ -15,6 +15,12 @@ data ElasticSearchConfig = ElasticSearchConfig { esServer :: !T.Text -- ^ Elasticsearch server address. , esIndex :: !T.Text -- ^ Elasticsearch index name. + , esShardCount :: !Int -- ^ Elasticsearch shard count for the named index. + -- + -- @since 0.10.0.0 + , esReplicaCount :: !Int -- ^ Elasticsearch replica count for the named index. + -- + -- @since 0.10.0.0 , esMapping :: !T.Text -- ^ Elasticsearch mapping name. , esLogin :: Maybe (EsUsername, EsPassword) -- ^ Elasticsearch basic authentication username and password. , esLoginInsecure :: !Bool -- ^ Allow basic authentication over non-TLS connections. @@ -25,6 +31,8 @@ defaultElasticSearchConfig = ElasticSearchConfig { esServer = "http://localhost:9200", esIndex = "logs", + esShardCount = 4, + esReplicaCount = 1, esMapping = "log", esLogin = Nothing, esLoginInsecure = False diff --git a/src/Log/Backend/ElasticSearch/V1/Lens.hs b/src/Log/Backend/ElasticSearch/V1/Lens.hs --- a/src/Log/Backend/ElasticSearch/V1/Lens.hs +++ b/src/Log/Backend/ElasticSearch/V1/Lens.hs @@ -4,6 +4,8 @@ I.ElasticSearchConfig , esServer , esIndex + , esShardCount + , esReplicaCount , esMapping , esLogin , esLoginInsecure @@ -26,6 +28,18 @@ -- | Elasticsearch index name. esIndex :: Lens' I.ElasticSearchConfig T.Text esIndex f esc = fmap (\x -> esc { I.esIndex = x }) $ f (I.esIndex esc) + +-- | Elasticsearch shard count for the named index. +-- +-- @since 0.10.0.0 +esShardCount :: Lens' I.ElasticSearchConfig Int +esShardCount f esc = fmap (\x -> esc { I.esShardCount = x }) $ f (I.esShardCount esc) + +-- | Elasticsearch replica count for the named index. +-- +-- @since 0.10.0.0 +esReplicaCount :: Lens' I.ElasticSearchConfig Int +esReplicaCount f esc = fmap (\x -> esc { I.esReplicaCount = x }) $ f (I.esReplicaCount esc) -- | Elasticsearch mapping name. esMapping :: Lens' I.ElasticSearchConfig T.Text diff --git a/src/Log/Backend/ElasticSearch/V5.hs b/src/Log/Backend/ElasticSearch/V5.hs --- a/src/Log/Backend/ElasticSearch/V5.hs +++ b/src/Log/Backend/ElasticSearch/V5.hs @@ -3,9 +3,13 @@ ElasticSearchConfig , esServer , esIndex + , esShardCount + , esReplicaCount , esMapping , esLogin , esLoginInsecure + , checkElasticSearchLogin + , checkElasticSearchConnection , defaultElasticSearchConfig , withElasticSearchLogger , elasticSearchLogger @@ -34,6 +38,7 @@ import Prelude import System.IO import TextShow +import qualified Data.Aeson.Encoding as Aeson import qualified Data.ByteString as BS import qualified Data.ByteString.Base64 as B64 import qualified Data.ByteString.Lazy.Char8 as BSL @@ -67,9 +72,20 @@ -> IO Word32 -- ^ Generate a random 32-bit word for use in -- document IDs. -> IO Logger -elasticSearchLogger ElasticSearchConfig{..} genRandomWord = do - checkElasticSearchLogin - checkElasticSearchConnection +elasticSearchLogger esConf@ElasticSearchConfig{..} genRandomWord = do + + checkElasticSearchLogin esConf >>= \case + Left (ex :: IOException) -> error . show $ ex + Right _ -> return () + + checkElasticSearchConnection esConf >>= \case + Left (ex :: HttpException) -> + hPutStrLn stderr $ + "ElasticSearch: unexpected error: " <> + show ex <> + " (is ElasticSearch server running?)" + Right () -> return () + indexRef <- newIORef $ IndexName T.empty mkBulkLogger "ElasticSearch" (\msgs -> do now <- getCurrentTime @@ -81,7 +97,7 @@ baseID <- (<>) <$> (littleEndianRep <$> liftIO genRandomWord) <*> pure (littleEndianRep . floor $ timeToDouble now) - retryOnException . runBH_ $ do + retryOnException . runBH_ esConf $ do -- Elasticsearch index names are additionally indexed by date so that each -- day is logged to a separate index to make log management easier. let index = IndexName $ T.concat [ @@ -95,11 +111,11 @@ -- index that already exists is harmless. indexExists' <- indexExists index unless indexExists' $ do - -- Bloodhound is weird and won't let us create index using default - -- settings, so pass these as the default ones. + -- Note that Bloodhound won't let us create index using + -- default settings. let indexSettings = IndexSettings { - indexShards = ShardCount 4 - , indexReplicas = ReplicaCount 1 + indexShards = ShardCount esShardCount + , indexReplicas = ReplicaCount esReplicaCount } void $ createIndex indexSettings index reply <- putMapping index mapping LogsMapping @@ -143,29 +159,12 @@ void $ bulk (V.map (toBulk index baseID) dummyMsgs)) (elasticSearchSync indexRef) where - server = Server esServer mapping = MappingName esMapping elasticSearchSync :: IORef IndexName -> IO () elasticSearchSync indexRef = do indexName <- readIORef indexRef - void . runBH_ $ refreshIndex indexName - - checkElasticSearchLogin :: IO () - checkElasticSearchLogin = - when (isJust esLogin - && not esLoginInsecure - && not ("https:" `T.isPrefixOf` esServer)) $ - error $ "ElasticSearch: insecure login: " - <> "Attempting to send login credentials over an insecure connection. " - <> "Set esLoginInsecure = True to disable this check." - - checkElasticSearchConnection :: IO () - checkElasticSearchConnection = try (void $ runBH_ listIndices) >>= \case - Left (ex::HttpException) -> - hPutStrLn stderr $ "ElasticSearch: unexpected error: " <> show ex - <> " (is ElasticSearch server running?)" - Right () -> return () + void . runBH_ esConf $ refreshIndex indexName retryOnException :: forall r. IO r -> IO r retryOnException m = try m >>= \case @@ -179,14 +178,6 @@ timeToDouble :: UTCTime -> Double timeToDouble = realToFrac . utcTimeToPOSIXSeconds - runBH_ :: forall r. BH IO r -> IO r - runBH_ f = do - mgr <- newManager tlsManagerSettings - let hook = maybe return (uncurry basicAuthHook) esLogin - let env = (mkBHEnv server mgr) { bhRequestHook = hook } - runBH env f - - jsonToBSL :: Value -> BSL.ByteString jsonToBSL = encodePretty' defConfig { confIndent = Spaces 2 } @@ -241,6 +232,34 @@ ] ] + toEncoding LogsMapping = Aeson.pairs $ mconcat + [ Aeson.pair "properties" $ Aeson.pairs $ mconcat + [ Aeson.pair "insertion_order" $ Aeson.pairs $ mconcat + [ "type" .= ("integer"::T.Text) + ] + , Aeson.pair "insertion_time" $ Aeson.pairs $ mconcat + [ "type" .= ("date"::T.Text) + , "format" .= ("date_time"::T.Text) + ] + , Aeson.pair "time" $ Aeson.pairs $ mconcat + [ "type" .= ("date"::T.Text) + , "format" .= ("date_time"::T.Text) + ] + , Aeson.pair "domain" $ Aeson.pairs $ mconcat + [ "type" .= ("string"::T.Text) + ] + , Aeson.pair "level" $ Aeson.pairs $ mconcat + [ "type" .= ("string"::T.Text) + ] + , Aeson.pair "component" $ Aeson.pairs $ mconcat + [ "type" .= ("string"::T.Text) + ] + , Aeson.pair "message" $ Aeson.pairs $ mconcat + [ "type" .= ("string"::T.Text) + ] + ] + ] + ---------------------------------------- littleEndianRep :: Word32 -> BS.ByteString @@ -252,3 +271,30 @@ decodeReply reply = case eitherDecode' $ responseBody reply of Right body -> body Left err -> object ["decoding_error" .= err] + +-- | Check that login credentials are specified properly. +-- +-- @since 0.10.1.0 +checkElasticSearchLogin :: ElasticSearchConfig -> IO (Either IOException ()) +checkElasticSearchLogin ElasticSearchConfig{..} = + try $ + when (isJust esLogin + && not esLoginInsecure + && not ("https:" `T.isPrefixOf` esServer)) $ + error $ "ElasticSearch: insecure login: " + <> "Attempting to send login credentials over an insecure connection. " + <> "Set esLoginInsecure = True to disable this check." + +-- | Check that we can connect to the ES server. +-- +-- @since 0.10.1.0 +checkElasticSearchConnection :: ElasticSearchConfig -> IO (Either HttpException ()) +checkElasticSearchConnection esConf = + try (void $ runBH_ esConf listIndices) + +runBH_ :: forall r. ElasticSearchConfig -> BH IO r -> IO r +runBH_ ElasticSearchConfig{..} f = do + mgr <- newManager tlsManagerSettings + let hook = maybe return (uncurry basicAuthHook) esLogin + let env = (mkBHEnv (Server esServer) mgr) { bhRequestHook = hook } + runBH env f diff --git a/src/Log/Backend/ElasticSearch/V5/Internal.hs b/src/Log/Backend/ElasticSearch/V5/Internal.hs --- a/src/Log/Backend/ElasticSearch/V5/Internal.hs +++ b/src/Log/Backend/ElasticSearch/V5/Internal.hs @@ -15,6 +15,12 @@ data ElasticSearchConfig = ElasticSearchConfig { esServer :: !T.Text -- ^ Elasticsearch server address. , esIndex :: !T.Text -- ^ Elasticsearch index name. + , esShardCount :: !Int -- ^ Elasticsearch shard count for the named index. + -- + -- @since 0.10.0.0 + , esReplicaCount :: !Int -- ^ Elasticsearch replica count for the named index. + -- + -- @since 0.10.0.0 , esMapping :: !T.Text -- ^ Elasticsearch mapping name. , esLogin :: Maybe (EsUsername, EsPassword) -- ^ Elasticsearch basic authentication username and password. , esLoginInsecure :: !Bool -- ^ Allow basic authentication over non-TLS connections. @@ -25,6 +31,8 @@ defaultElasticSearchConfig = ElasticSearchConfig { esServer = "http://localhost:9200", esIndex = "logs", + esShardCount = 4, + esReplicaCount = 1, esMapping = "log", esLogin = Nothing, esLoginInsecure = False diff --git a/src/Log/Backend/ElasticSearch/V5/Lens.hs b/src/Log/Backend/ElasticSearch/V5/Lens.hs --- a/src/Log/Backend/ElasticSearch/V5/Lens.hs +++ b/src/Log/Backend/ElasticSearch/V5/Lens.hs @@ -4,6 +4,8 @@ I.ElasticSearchConfig , esServer , esIndex + , esShardCount + , esReplicaCount , esMapping , esLogin , esLoginInsecure @@ -26,6 +28,18 @@ -- | Elasticsearch index name. esIndex :: Lens' I.ElasticSearchConfig T.Text esIndex f esc = fmap (\x -> esc { I.esIndex = x }) $ f (I.esIndex esc) + +-- | Elasticsearch shard count for the named index. +-- +-- @since 0.10.0.0 +esShardCount :: Lens' I.ElasticSearchConfig Int +esShardCount f esc = fmap (\x -> esc { I.esShardCount = x }) $ f (I.esShardCount esc) + +-- | Elasticsearch replica count for the named index. +-- +-- @since 0.10.0.0 +esReplicaCount :: Lens' I.ElasticSearchConfig Int +esReplicaCount f esc = fmap (\x -> esc { I.esReplicaCount = x }) $ f (I.esReplicaCount esc) -- | Elasticsearch mapping name. esMapping :: Lens' I.ElasticSearchConfig T.Text