katip-elasticsearch 0.4.0.4 → 0.4.1.0

raw patch · 6 files changed

+83/−30 lines, 6 filesdep +semigroupsPVP ok

version bump matches the API change (PVP)

Dependencies added: semigroups

API changes (from Hackage documentation)

Files

• changelog.md +6/−0
• examples/example.hs +1/−1
• katip-elasticsearch.cabal +2/−1
• src/Katip/Scribes/ElasticSearch.hs +1/−1
• src/Katip/Scribes/ElasticSearch/Internal.hs +64/−27
• test/Main.hs +9/−0

@@ -1,3 +1,9 @@+0.4.1.0+=======+* Fix bug where index was created in `mkEsScribe` when it would not be used due to index sharding.+* Update some index settings if index already exists and sharding is not used.+* For ES V5 and higher, stop using the deprecated (and in 6.x, removed) `string` type for index templates, instead using `text` and `keyword` as appropriate. This makes `katip-elasticsearch` compatible with ES 6.x.+ 0.4.0.4 ======= * Allow http-types 0.12

@@ -17,7 +17,7 @@ main :: IO () main = do   mgr <- newManager defaultManagerSettings-  let bhe = mkBHEnv (Server "localhost") mgr+  let bhe = mkBHEnv (Server "http://localhost:9200") mgr   esScribe <- mkEsScribe     -- Reasonable for production     defaultEsScribeCfgV5

@@ -1,7 +1,7 @@ name:                katip-elasticsearch synopsis:            ElasticSearch scribe for the Katip logging framework. description:         See README.md for more details.-version:             0.4.0.4+version:             0.4.1.0 license:             BSD3 license-file:        LICENSE author:              Ozgun Ataman, Michael Xavier@@ -53,6 +53,7 @@     , http-types >= 0.8 && < 0.13     , time >= 1 && < 1.9     , bytestring+    , semigroups   hs-source-dirs:      src   default-language:    Haskell2010   hs-source-dirs:      src

@@ -17,7 +17,7 @@ -- main :: IO () -- main = do --   mgr <- newManager defaultManagerSettings---   let bhe = mkBHEnv (Server "localhost") mgr+--   let bhe = mkBHEnv (Server "http://localhost:9200") mgr --   esScribe <- mkEsScribe --     -- Reasonable for production --     defaultEsScribeCfgV5

@@ -28,6 +28,7 @@                                                           recovering) import           Data.Aeson import           Data.ByteString.Lazy                    (ByteString)+import           Data.List.NonEmpty                      (NonEmpty(..)) import           Data.Monoid                             ((<>)) import           Data.Text                               (Text) import qualified Data.Text                               as T@@ -240,7 +241,10 @@  ------------------------------------------------------------------------------- data EsScribeSetupError = CouldNotCreateIndex !(Response ByteString)-                        | CouldNotCreateMapping !(Response ByteString) deriving (Typeable, Show)+                        | CouldNotUpdateIndexSettings !(Response ByteString)+                        | CouldNotCreateMapping !(Response ByteString)+                        | CouldNotPutTemplate !(Response ByteString)+                        deriving (Typeable, Show)   instance Exception EsScribeSetupError@@ -273,18 +277,26 @@   endSig <- newEmptyMVar    runBH prx env $ do-    chk <- indexExists prx ix-    -- note that this doesn't update settings. That's not available-    -- through the Bloodhound API yet-    unless chk $ void $ do-      r1 <- createIndex prx essIndexSettings ix-      unless (statusIsSuccessful (responseStatus r1)) $-        liftIO $ throwIO (CouldNotCreateIndex r1)-      r2 <- if shardingEnabled-              then putTemplate prx tpl tplName-              else putMapping prx ix mapping base-      unless (statusIsSuccessful (responseStatus r2)) $-        liftIO $ throwIO (CouldNotCreateMapping r2)+    if shardingEnabled+       then do+         -- create or update+         res <- putTemplate prx tpl tplName+         unless (statusIsSuccessful (responseStatus res)) $+           liftIO $ throwIO (CouldNotPutTemplate res)+       else do+         ixExists <- indexExists prx ix+         if ixExists+            then do+              res <- updateIndexSettings prx (toUpdatabaleIndexSettings prx essIndexSettings) ix+              unless (statusIsSuccessful (responseStatus res)) $+                liftIO $ throwIO (CouldNotUpdateIndexSettings res)+            else do+              r1 <- createIndex prx essIndexSettings ix+              unless (statusIsSuccessful (responseStatus r1)) $+                liftIO $ throwIO (CouldNotCreateIndex r1)+              r2 <- putMapping prx ix mapping base+              unless (statusIsSuccessful (responseStatus r2)) $+                liftIO $ throwIO (CouldNotCreateMapping r2)    workers <- replicateM (unEsPoolSize essPoolSize) $ async $     startWorker cfg env mapping q@@ -320,24 +332,27 @@ baseMapping :: ESVersion v => proxy v -> MappingName v -> Value baseMapping prx mn =   object [ fromMappingName prx mn .= object ["properties" .= object prs] ]-  where prs = [ str "thread"-              , str "sev"-              , str "pid"-              , str "ns"-              , str "msg"+  where prs = [ unanalyzedString "thread"+              , unanalyzedString "sev"+              , unanalyzedString "pid"+              -- ns is frequently fulltext searched+              , analyzedString "ns"+              -- we want message to be fulltext searchable+              , analyzedString "msg"               , "loc" .= locType-              , str "host"-              , str "env"+              , unanalyzedString "host"+              , unanalyzedString "env"               , "at" .= dateType-              , str "app"+              , unanalyzedString "app"               ]-        str k = k .= object ["type" .= String "string"]+        unanalyzedString k = k .= object ["type" .= String (unanalyzedStringType prx)]+        analyzedString k = k .= object ["type" .= String (analyzedStringType prx)]         locType = object ["properties" .= object locPairs]-        locPairs = [ str "loc_pkg"-                   , str "loc_mod"-                   , str "loc_ln"-                   , str "loc_fn"-                   , str "loc_col"+        locPairs = [ unanalyzedString "loc_pkg"+                   , unanalyzedString "loc_mod"+                   , unanalyzedString "loc_ln"+                   , unanalyzedString "loc_fn"+                   , unanalyzedString "loc_col"                    ]         dateType = object [ "format" .= esDateFormat                           , "type" .= String "date"@@ -433,6 +448,7 @@   type BHEnv v   type IndexSettings v   defaultIndexSettings :: proxy v -> IndexSettings v+  type UpdatableIndexSetting v   type IndexName v   toIndexName :: proxy v -> Text -> IndexName v   fromIndexName :: proxy v -> IndexName v -> Text@@ -451,15 +467,24 @@   type IndexDocumentSettings v   defaultIndexDocumentSettings :: proxy v -> IndexDocumentSettings v +  toUpdatabaleIndexSettings :: proxy v -> IndexSettings v -> NonEmpty (UpdatableIndexSetting v)+   -- Operations   -- We're deciding on IO here, but it isn't necessary   indexExists :: proxy v -> IndexName v -> BH v IO Bool   indexDocument :: ToJSON doc => proxy v -> IndexName v -> MappingName v -> IndexDocumentSettings v -> doc -> DocId v -> BH v IO (Response ByteString)   createIndex :: proxy v -> IndexSettings v -> IndexName v -> BH v IO (Response ByteString)+  updateIndexSettings :: proxy v -> NonEmpty (UpdatableIndexSetting v) -> IndexName v -> BH v IO (Response ByteString)   putTemplate :: proxy v -> IndexTemplate v -> TemplateName v -> BH v IO (Response ByteString)   putMapping :: (ToJSON a) => proxy v -> IndexName v -> MappingName v -> a -> BH v IO (Response ByteString) +  -- In ES 5 and beyond, "string" was deprecated in favor of text for+  -- fulltext and keyword for unanalyzed tokens+  unanalyzedStringType :: proxy v -> Text+  analyzedStringType :: proxy v -> Text ++ data ESV1 = ESV1  instance ESVersion ESV1 where@@ -467,6 +492,7 @@   type IndexSettings ESV1 = V1.IndexSettings   defaultIndexSettings _ = V1.defaultIndexSettings   type IndexName ESV1 = V1.IndexName+  type UpdatableIndexSetting ESV1 = V1.UpdatableIndexSetting   toIndexName _ = V1.IndexName   fromIndexName _ (V1.IndexName x) = x   type MappingName ESV1 = V1.MappingName@@ -482,12 +508,17 @@   type IndexTemplate ESV1 = V1.IndexTemplate   toIndexTemplate _ = V1.IndexTemplate   type IndexDocumentSettings ESV1 = V1.IndexDocumentSettings+  toUpdatabaleIndexSettings _ s =+    (V1.NumberOfReplicas (V1.indexReplicas s)) :| []   defaultIndexDocumentSettings _ = V1.defaultIndexDocumentSettings   indexExists _ = V1.indexExists   indexDocument _ = V1.indexDocument   createIndex _ = V1.createIndex+  updateIndexSettings _ = V1.updateIndexSettings   putTemplate _ = V1.putTemplate   putMapping _ = V1.putMapping+  unanalyzedStringType _ = "string"+  analyzedStringType _ = "string"   data ESV5 = ESV5@@ -495,6 +526,7 @@ instance ESVersion ESV5 where   type BHEnv ESV5 = V5.BHEnv   type IndexSettings ESV5 = V5.IndexSettings+  type UpdatableIndexSetting ESV5 = V5.UpdatableIndexSetting   defaultIndexSettings _ = V5.defaultIndexSettings   type IndexName ESV5 = V5.IndexName   toIndexName _ = V5.IndexName@@ -512,9 +544,14 @@   type IndexTemplate ESV5 = V5.IndexTemplate   toIndexTemplate _ = V5.IndexTemplate   type IndexDocumentSettings ESV5 = V5.IndexDocumentSettings+  toUpdatabaleIndexSettings _ s =+    (V5.NumberOfReplicas (V5.indexReplicas s)) :| []   defaultIndexDocumentSettings _ = V5.defaultIndexDocumentSettings   indexExists _ = V5.indexExists   indexDocument _ = V5.indexDocument   createIndex _ = V5.createIndex+  updateIndexSettings _ = V5.updateIndexSettings   putTemplate _ = V5.putTemplate   putMapping _ = V5.putMapping+  unanalyzedStringType _ = "keyword"+  analyzedStringType _ = "text"

@@ -17,6 +17,7 @@ ------------------------------------------------------------------------------- import           Control.Applicative                     as A import           Control.Concurrent.STM+import           Control.Exception import           Control.Lens                            hiding (mapping, (.=)) import           Control.Monad import           Control.Monad.IO.Class@@ -237,6 +238,14 @@          let l = head logs          l ^? key "_source" . key "msg" . _String @?= Just "A test message"          l ^? key "_source" . key "data" . key "whatever::b" . _Bool @?= Just True+  , testCase "can set up twice with no sharding" $ do+      let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = NoIndexSharding })) (teardownSearch prx)+      setup $ setup $+        return ()+  , testCase "can set up twice with sharding" $ do+      let setup = bracket_ (setupSearch prx (\c -> c { essIndexSharding = DailyIndexSharding })) (teardownSearch prx)+      setup $ setup $+        return ()   , withSearch prx $ \setup -> testCase "date-based index sharding" $ do       let t1 = mkTime 2016 1 2 3 4 5       fakeClock <- newTVarIO t1