diff --git a/changelog.md b/changelog.md --- a/changelog.md +++ b/changelog.md @@ -1,3 +1,18 @@ +0.3.0.0 +============== +* Added zero padding to date-based indices. This shouldn't negatively + impact most users but to be safe, this was put behind a breaking + version number. Previously, you may see indices + `log-index-prefix-2016-1-2`. That index will now be + `log-index-prefix-2016-01-02`, so at worst when you deploy, the day + of the change will have 2 indices: one zero-padded and one not. If + you are using custom index sharding, you will not be affected by + this. The reasoning here is that most existing elasticsearch tools + such as `curator` expects date indices to be zero padded. By + switching to the standard, users can easily get log rotation and + other features from `curator` rather than having to roll their own + or add them to Katip. + 0.2.1.0 ============== diff --git a/katip-elasticsearch.cabal b/katip-elasticsearch.cabal --- a/katip-elasticsearch.cabal +++ b/katip-elasticsearch.cabal @@ -1,7 +1,7 @@ name: katip-elasticsearch synopsis: ElasticSearch scribe for the Katip logging framework. description: See README.md for more details. -version: 0.2.1.0 +version: 0.3.0.0 license: BSD3 license-file: LICENSE author: Ozgun Ataman, Michael Xavier @@ -27,7 +27,7 @@ Katip.Scribes.ElasticSearch.Annotations build-depends: base >=4.6 && <5 - , katip >= 0.2.0.0 && < 0.3 + , katip >= 0.2.0.0 && < 0.4 , bloodhound >= 0.11.0.0 && < 0.12 , uuid >= 1.3.12 && < 1.4 , aeson >=0.6 && <0.12 diff --git a/src/Katip/Scribes/ElasticSearch.hs b/src/Katip/Scribes/ElasticSearch.hs --- a/src/Katip/Scribes/ElasticSearch.hs +++ b/src/Katip/Scribes/ElasticSearch.hs @@ -3,6 +3,24 @@ {-# LANGUAGE RankNTypes #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ScopedTypeVariables #-} +-- | Includes a scribe that can be used to log structured, JSON log +-- messages to ElasticSearch. These logs can be explored easily using +-- or your tool of +-- choice. +-- +-- == __Important Note on Index Settings__ +-- +-- 'defaultEsScribeCfg' inherits a set of default index settings from +-- the @bloodhound@ package. These settings at this time of writing +-- set the indices up to have 3 shards and 2 replicas. This is an +-- arguably reasonable default setting for production but may cause +-- problems for development. In development, your cluster may be +-- configured to seek a write quorum greater than 1. If you're running +-- ElasticSearch on a single node, this could cause your writes to +-- wait for a bit and then fail due to a lack of quorum. __For development, we recommend setting your replica count to 0 or modifying your write quorum settings__. For production, we recommend reading the +-- and choosing the appropriate settings, +-- keeping in mind that you can chage replica counts on a live index +-- but that changing shard counts requires recreating the index. module Katip.Scribes.ElasticSearch (-- * Building a scribe mkEsScribe @@ -57,6 +75,7 @@ import Database.Bloodhound import Network.HTTP.Client import Network.HTTP.Types.Status +import Text.Printf (printf) ------------------------------------------------------------------------------- import Katip.Core import Katip.Scribes.ElasticSearch.Annotations @@ -213,8 +232,10 @@ ------------------------------------------------------------------------------- -sis :: Show a => a -> IndexNameSegment -sis = IndexNameSegment . T.pack . show +sis :: Integral a => a -> IndexNameSegment +sis = IndexNameSegment . T.pack . fmt + where + fmt = printf "%02d" . toInteger ------------------------------------------------------------------------------- diff --git a/src/Katip/Scribes/ElasticSearch/Annotations.hs b/src/Katip/Scribes/ElasticSearch/Annotations.hs --- a/src/Katip/Scribes/ElasticSearch/Annotations.hs +++ b/src/Katip/Scribes/ElasticSearch/Annotations.hs @@ -31,7 +31,8 @@ toJSON = annotateValue . toJSON . typeAnnotatedValue -instance ToObject a => ToObject (TypeAnnotated a) +instance ToObject a => ToObject (TypeAnnotated a) where + toObject = annotateKeys . toObject . typeAnnotatedValue instance FromJSON a => FromJSON (TypeAnnotated a) where diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -100,8 +100,8 @@ $(logT) (ExampleCtx True) mempty InfoS "tomorrow" liftIO $ do void done - todayLogs <- getLogsByIndex (IndexName "katip-elasticsearch-tests-2016-1-2") - tomorrowLogs <- getLogsByIndex (IndexName "katip-elasticsearch-tests-2016-1-3") + todayLogs <- getLogsByIndex (IndexName "katip-elasticsearch-tests-2016-01-02") + tomorrowLogs <- getLogsByIndex (IndexName "katip-elasticsearch-tests-2016-01-03") assertBool ("todayLogs has " <> show (length todayLogs) <> " items") (length todayLogs == 1) assertBool ("tomorrowLogs has " <> show (length tomorrowLogs) <> " items") (length tomorrowLogs == 1) let logToday = head todayLogs @@ -118,8 +118,8 @@ $(logT) (ExampleCtx True) mempty InfoS "tomorrow" liftIO $ do void done - todayLogs <- getLogsByIndex (IndexName "katip-elasticsearch-tests-2016-2-28") -- rounds back to previous sunday - tomorrowLogs <- getLogsByIndex (IndexName "katip-elasticsearch-tests-2016-3-6") -- is on sunday, so uses current date + todayLogs <- getLogsByIndex (IndexName "katip-elasticsearch-tests-2016-02-28") -- rounds back to previous sunday + tomorrowLogs <- getLogsByIndex (IndexName "katip-elasticsearch-tests-2016-03-06") -- is on sunday, so uses current date assertBool ("todayLogs has " <> show (length todayLogs) <> " items") (length todayLogs == 1) assertBool ("tomorrowLogs has " <> show (length tomorrowLogs) <> " items") (length tomorrowLogs == 1) let logToday = head todayLogs @@ -163,12 +163,18 @@ typeAnnotatedTests :: TestTree typeAnnotatedTests = testGroup "TypeAnnotated" [ - testCase "annotates values on toJSON" $ do + testCase "annotates values on toJSON" $ toJSON (TypeAnnotated exampleValue) @?= annotatedExampleValue - , testCase "deannotates on parseJSON" $ do + + , testCase "annotates values on toObject" $ + toObject (TypeAnnotated exampleObject) @?= annotatedExampleObject + + , testCase "deannotates on parseJSON" $ parseEither parseJSON (toJSON exampleValue) @?= Right exampleValue + , testProperty "roundtrips the same as raw" $ \(v :: Value) -> - let res = typeAnnotatedValue <$> (parseEither parseJSON (toJSON (TypeAnnotated v))) + let res = typeAnnotatedValue + <$> parseEither parseJSON (toJSON (TypeAnnotated v)) in res === Right v ] @@ -187,33 +193,41 @@ where getDOW = view _3 . toWeekDate + ------------------------------------------------------------------------------- +exampleObject :: Object +exampleObject = HM.fromList + [ ("a bool", Bool False) + , ("a long", Number 24) + , ("a double", Number 52.3) + , ("a string", String "s") + , ("a null", Null) + , ("a map", Object (HM.singleton "baz" (Bool True))) + ] + + +------------------------------------------------------------------------------- +annotatedExampleObject :: Object +annotatedExampleObject = HM.fromList + [ ("a map",Object $ HM.fromList [("baz::b", Bool True)]) + , ("a bool::b", Bool False) + , ("a null::n", Null) + , ("a string::s", String "s") + , ("a double::d", Number 52.3) + , ("a long::l", Number 24.0) + ] + + +------------------------------------------------------------------------------- exampleValue :: Value -exampleValue = Array $ V.fromList [Null, Object ob] - where - ob = HM.fromList [ ("a bool", Bool False) - , ("a long", Number 24) - , ("a double", Number 52.3) - , ("a string", String "s") - , ("a null", Null) - , ("a map", Object (HM.singleton "baz" (Bool True))) - ] +exampleValue = Array $ V.fromList [Null, Object exampleObject] ------------------------------------------------------------------------------- annotatedExampleValue :: Value annotatedExampleValue = Array $ V.fromList - [ - Null - , Object $ HM.fromList - [ - ("a map",Object $ HM.fromList [("baz::b", Bool True)]) - , ("a bool::b", Bool False) - , ("a null::n", Null) - , ("a string::s", String "s") - , ("a double::d", Number 52.3) - , ("a long::l", Number 24.0) - ] + [ Null + , Object annotatedExampleObject ]