diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,9 @@ +0.4.1.0 +======= +* Fix bug where index was created in `mkEsScribe` when it would not be used due to index sharding. +* Update some index settings if index already exists and sharding is not used. +* For ES V5 and higher, stop using the deprecated (and in 6.x, removed) `string` type for index templates, instead using `text` and `keyword` as appropriate. This makes `katip-elasticsearch` compatible with ES 6.x. + 0.4.0.4 ======= * Allow http-types 0.12 diff --git a/examples/example.hs b/examples/example.hs --- a/examples/example.hs +++ b/examples/example.hs @@ -17,7 +17,7 @@ main :: IO () main = do mgr <- newManager defaultManagerSettings - let bhe = mkBHEnv (Server "localhost") mgr + let bhe = mkBHEnv (Server "http://localhost:9200") mgr esScribe <- mkEsScribe -- Reasonable for production defaultEsScribeCfgV5 diff --git a/katip-elasticsearch.cabal b/katip-elasticsearch.cabal --- a/katip-elasticsearch.cabal +++ b/katip-elasticsearch.cabal @@ -1,7 +1,7 @@ name: katip-elasticsearch synopsis: ElasticSearch scribe for the Katip logging framework. description: See README.md for more details. -version: 0.4.0.4 +version: 0.4.1.0 license: BSD3 license-file: LICENSE author: Ozgun Ataman, Michael Xavier @@ -53,6 +53,7 @@ , http-types >= 0.8 && < 0.13 , time >= 1 && < 1.9 , bytestring + , semigroups hs-source-dirs: src default-language: Haskell2010 hs-source-dirs: src diff --git a/src/Katip/Scribes/ElasticSearch.hs b/src/Katip/Scribes/ElasticSearch.hs --- a/src/Katip/Scribes/ElasticSearch.hs +++ b/src/Katip/Scribes/ElasticSearch.hs @@ -17,7 +17,7 @@ -- main :: IO () -- main = do -- mgr <- newManager defaultManagerSettings --- let bhe = mkBHEnv (Server "localhost") mgr +-- let bhe = mkBHEnv (Server "http://localhost:9200") mgr -- esScribe <- mkEsScribe -- -- Reasonable for production -- defaultEsScribeCfgV5 diff --git a/src/Katip/Scribes/ElasticSearch/Internal.hs b/src/Katip/Scribes/ElasticSearch/Internal.hs --- a/src/Katip/Scribes/ElasticSearch/Internal.hs +++ b/src/Katip/Scribes/ElasticSearch/Internal.hs @@ -28,6 +28,7 @@ recovering) import Data.Aeson import Data.ByteString.Lazy (ByteString) +import Data.List.NonEmpty (NonEmpty(..)) import Data.Monoid ((<>)) import Data.Text (Text) import qualified Data.Text as T @@ -240,7 +241,10 @@ ------------------------------------------------------------------------------- data EsScribeSetupError = CouldNotCreateIndex !(Response ByteString) - | CouldNotCreateMapping !(Response ByteString) deriving (Typeable, Show) + | CouldNotUpdateIndexSettings !(Response ByteString) + | CouldNotCreateMapping !(Response ByteString) + | CouldNotPutTemplate !(Response ByteString) + deriving (Typeable, Show) instance Exception EsScribeSetupError @@ -273,18 +277,26 @@ endSig <- newEmptyMVar runBH prx env $ do - chk <- indexExists prx ix - -- note that this doesn't update settings. That's not available - -- through the Bloodhound API yet - unless chk $ void $ do - r1 <- createIndex prx essIndexSettings ix - unless (statusIsSuccessful (responseStatus r1)) $ - liftIO $ throwIO (CouldNotCreateIndex r1) - r2 <- if shardingEnabled - then putTemplate prx tpl tplName - else putMapping prx ix mapping base - unless (statusIsSuccessful (responseStatus r2)) $ - liftIO $ throwIO (CouldNotCreateMapping r2) + if shardingEnabled + then do + -- create or update + res <- putTemplate prx tpl tplName + unless (statusIsSuccessful (responseStatus res)) $ + liftIO $ throwIO (CouldNotPutTemplate res) + else do + ixExists <- indexExists prx ix + if ixExists + then do + res <- updateIndexSettings prx (toUpdatabaleIndexSettings prx essIndexSettings) ix + unless (statusIsSuccessful (responseStatus res)) $ + liftIO $ throwIO (CouldNotUpdateIndexSettings res) + else do + r1 <- createIndex prx essIndexSettings ix + unless (statusIsSuccessful (responseStatus r1)) $ + liftIO $ throwIO (CouldNotCreateIndex r1) + r2 <- putMapping prx ix mapping base + unless (statusIsSuccessful (responseStatus r2)) $ + liftIO $ throwIO (CouldNotCreateMapping r2) workers <- replicateM (unEsPoolSize essPoolSize) $ async $ startWorker cfg env mapping q @@ -320,24 +332,27 @@ baseMapping :: ESVersion v => proxy v -> MappingName v -> Value baseMapping prx mn = object [ fromMappingName prx mn .= object ["properties" .= object prs] ] - where prs = [ str "thread" - , str "sev" - , str "pid" - , str "ns" - , str "msg" + where prs = [ unanalyzedString "thread" + , unanalyzedString "sev" + , unanalyzedString "pid" + -- ns is frequently fulltext searched + , analyzedString "ns" + -- we want message to be fulltext searchable + , analyzedString "msg" , "loc" .= locType - , str "host" - , str "env" + , unanalyzedString "host" + , unanalyzedString "env" , "at" .= dateType - , str "app" + , unanalyzedString "app" ] - str k = k .= object ["type" .= String "string"] + unanalyzedString k = k .= object ["type" .= String (unanalyzedStringType prx)] + analyzedString k = k .= object ["type" .= String (analyzedStringType prx)] locType = object ["properties" .= object locPairs] - locPairs = [ str "loc_pkg" - , str "loc_mod" - , str "loc_ln" - , str "loc_fn" - , str "loc_col" + locPairs = [ unanalyzedString "loc_pkg" + , unanalyzedString "loc_mod" + , unanalyzedString "loc_ln" + , unanalyzedString "loc_fn" + , unanalyzedString "loc_col" ] dateType = object [ "format" .= esDateFormat , "type" .= String "date" @@ -433,6 +448,7 @@ type BHEnv v type IndexSettings v defaultIndexSettings :: proxy v -> IndexSettings v + type UpdatableIndexSetting v type IndexName v toIndexName :: proxy v -> Text -> IndexName v fromIndexName :: proxy v -> IndexName v -> Text @@ -451,15 +467,24 @@ type IndexDocumentSettings v defaultIndexDocumentSettings :: proxy v -> IndexDocumentSettings v + toUpdatabaleIndexSettings :: proxy v -> IndexSettings v -> NonEmpty (UpdatableIndexSetting v) + -- Operations -- We're deciding on IO here, but it isn't necessary indexExists :: proxy v -> IndexName v -> BH v IO Bool indexDocument :: ToJSON doc => proxy v -> IndexName v -> MappingName v -> IndexDocumentSettings v -> doc -> DocId v -> BH v IO (Response ByteString) createIndex :: proxy v -> IndexSettings v -> IndexName v -> BH v IO (Response ByteString) + updateIndexSettings :: proxy v -> NonEmpty (UpdatableIndexSetting v) -> IndexName v -> BH v IO (Response ByteString) putTemplate :: proxy v -> IndexTemplate v -> TemplateName v -> BH v IO (Response ByteString) putMapping :: (ToJSON a) => proxy v -> IndexName v -> MappingName v -> a -> BH v IO (Response ByteString) + -- In ES 5 and beyond, "string" was deprecated in favor of text for + -- fulltext and keyword for unanalyzed tokens + unanalyzedStringType :: proxy v -> Text + analyzedStringType :: proxy v -> Text + + data ESV1 = ESV1 instance ESVersion ESV1 where @@ -467,6 +492,7 @@ type IndexSettings ESV1 = V1.IndexSettings defaultIndexSettings _ = V1.defaultIndexSettings type IndexName ESV1 = V1.IndexName + type UpdatableIndexSetting ESV1 = V1.UpdatableIndexSetting toIndexName _ = V1.IndexName fromIndexName _ (V1.IndexName x) = x type MappingName ESV1 = V1.MappingName @@ -482,12 +508,17 @@ type IndexTemplate ESV1 = V1.IndexTemplate toIndexTemplate _ = V1.IndexTemplate type IndexDocumentSettings ESV1 = V1.IndexDocumentSettings + toUpdatabaleIndexSettings _ s = + (V1.NumberOfReplicas (V1.indexReplicas s)) :| [] defaultIndexDocumentSettings _ = V1.defaultIndexDocumentSettings indexExists _ = V1.indexExists indexDocument _ = V1.indexDocument createIndex _ = V1.createIndex + updateIndexSettings _ = V1.updateIndexSettings putTemplate _ = V1.putTemplate putMapping _ = V1.putMapping + unanalyzedStringType _ = "string" + analyzedStringType _ = "string" data ESV5 = ESV5 @@ -495,6 +526,7 @@ instance ESVersion ESV5 where type BHEnv ESV5 = V5.BHEnv type IndexSettings ESV5 = V5.IndexSettings + type UpdatableIndexSetting ESV5 = V5.UpdatableIndexSetting defaultIndexSettings _ = V5.defaultIndexSettings type IndexName ESV5 = V5.IndexName toIndexName _ = V5.IndexName @@ -512,9 +544,14 @@ type IndexTemplate ESV5 = V5.IndexTemplate toIndexTemplate _ = V5.IndexTemplate type IndexDocumentSettings ESV5 = V5.IndexDocumentSettings + toUpdatabaleIndexSettings _ s = + (V5.NumberOfReplicas (V5.indexReplicas s)) :| [] defaultIndexDocumentSettings _ = V5.defaultIndexDocumentSettings indexExists _ = V5.indexExists indexDocument _ = V5.indexDocument createIndex _ = V5.createIndex + updateIndexSettings _ = V5.updateIndexSettings putTemplate _ = V5.putTemplate putMapping _ = V5.putMapping + unanalyzedStringType _ = "keyword" + analyzedStringType _ = "text" diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -17,6 +17,7 @@ ------------------------------------------------------------------------------- import Control.Applicative as A import Control.Concurrent.STM +import Control.Exception import Control.Lens hiding (mapping, (.=)) import Control.Monad import Control.Monad.IO.Class @@ -237,6 +238,14 @@ let l = head logs l ^? key "_source" . key "msg" . _String @?= Just "A test message" l ^? key "_source" . key "data" . key "whatever::b" . _Bool @?= Just True + , testCase "can set up twice with no sharding" $ do + let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = NoIndexSharding })) (teardownSearch prx) + setup $ setup $ + return () + , testCase "can set up twice with sharding" $ do + let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = DailyIndexSharding })) (teardownSearch prx) + setup $ setup $ + return () , withSearch prx $ \setup -> testCase "date-based index sharding" $ do let t1 = mkTime 2016 1 2 3 4 5 fakeClock <- newTVarIO t1