hnormalise 0.2.0.0 → 0.3.0.0

raw patch · 12 files changed

+63/−45 lines, 12 filesdep +attoparsec-iso8601dep +timePVP ok

version bump matches the API change (PVP)

Dependencies added: attoparsec-iso8601, time

API changes (from Hackage documentation)

Files

• hnormalise.cabal +7/−1
• src/HNormalise.hs +2/−2
• src/HNormalise/Internal.hs +6/−5
• src/HNormalise/Json.hs +1/−0
• src/HNormalise/Lmod/Parser.hs +4/−3
• src/HNormalise/Parser.hs +20/−12
• src/HNormalise/Shorewall/Parser.hs +10/−9
• src/HNormalise/Torque/Parser.hs +3/−3
• test/Bench.hs +3/−3
• test/HNormalise/Lmod/ParserSpec.hs +2/−2
• test/HNormalise/Shorewall/ParserSpec.hs +3/−3
• test/HNormalise/Torque/ParserSpec.hs +2/−2

@@ -1,5 +1,5 @@ name:                hnormalise-version:             0.2.0.0+version:             0.3.0.0 synopsis:            Log message normalisation tool producing structured JSON messages description:         Log message normalisation tool producing structured JSON messages homepage:            https://github.com/itkovian/hnormalise#readme@@ -39,12 +39,14 @@                      , aeson                      , aeson-pretty                      , attoparsec+                     , attoparsec-iso8601                      , bytestring                      , containers                      , directory                      , ip                      , permute                      , text+                     , time                      , yaml   default-language:    Haskell2010 @@ -57,6 +59,7 @@                      , aeson                      , aeson-pretty                      , attoparsec+                     , attoparsec-iso8601                      , bytestring                      , conduit                      , conduit-combinators@@ -67,6 +70,7 @@                      , optparse-applicative                      , resourcet                      , text+                     , time                      , word8                      , yaml   default-language:    Haskell2010@@ -82,6 +86,7 @@   build-depends:       base                      , aeson                      , attoparsec+                     , attoparsec-iso8601                      , conduit-extra                      , hnormalise                      , hspec@@ -90,6 +95,7 @@                      , hspec-attoparsec                      , ip                      , text+                     , time   ghc-options:         -threaded -rtsopts -with-rtsopts=-N   default-language:    Haskell2010 

@@ -70,9 +70,9 @@ convertMessage :: Text -> Maybe ParseResult convertMessage message =     case parse parseMessage message of-        Done _ pm -> Just pm+        Done _ (_, pm) -> Just pm         Partial c -> case c empty of-            Done _ pm -> Just pm+            Done _ (_, pm) -> Just pm             _ -> Nothing         _         -> Nothing 

@@ -9,6 +9,7 @@ import           Data.Aeson                 (FromJSON, ToJSON, toEncoding,                                              toJSON) import           Data.Text+import           Data.Time.LocalTime import           GHC.Generics               (Generic)  --------------------------------------------------------------------------------@@ -44,16 +45,16 @@ data Rsyslog = Rsyslog     { msg              :: !Text     --, rawmsg           :: !Text-    , timereported     :: !Text+    , timereported     :: !(Maybe ZonedTime)     , hostname         :: !Text     , syslogtag        :: !Text  -- Could be a list?     , inputname        :: !Text     , fromhost         :: !Text     , fromhost_ip      :: !Text-    , pri              :: !Text+    , pri              :: !(Maybe Int)     , syslogfacility   :: !Text     , syslogseverity   :: !Text-    , timegenerated    :: !Text+    , timegenerated    :: !ZonedTime     , programname      :: !Text     , protocol_version :: !Text     --, structured_data  :: !Text@@ -62,11 +63,11 @@     --, msgid            :: !Text     --, uuid             :: !(Maybe Text)     --, all_json         :: !(Maybe Text)-    } deriving (Eq, Show, Generic)+    } deriving (Show, Generic)  -------------------------------------------------------------------------------- data NormalisedRsyslog = NRsyslog     { rsyslog    :: Rsyslog            -- ^ The original rsyslog message in a parsed form     , normalised :: ParseResult        -- ^ The normalised message     , jsonkey    :: Text               -- ^ The key under which the normalised info will appear in the JSON result-    } deriving (Eq, Show, Generic)+    } deriving (Show, Generic)

@@ -47,5 +47,6 @@         pairs             (  "message" .= msg r             <> "syslog_abspri" .= syslogseverity r+            <> "program" .= app_name r             <> k .= n             )

@@ -9,6 +9,7 @@ import           Control.Applicative        ((<|>)) import           Data.Attoparsec.Combinator (lookAhead, manyTill) import           Data.Attoparsec.Text+import           Data.Text                  (Text) --------------------------------------------------------------------------------  import           HNormalise.Common.Parser@@ -35,16 +36,16 @@         , version = version         } -parseLmodLoad :: Parser LmodLoad+parseLmodLoad :: Parser (Text, LmodLoad) parseLmodLoad = do     string "lmod::"     info <- skipSpace *> parseLmodInfo     userload <- char ',' *> skipSpace *> kvYesNoParser "userload"     m <- char ',' *> skipSpace *> parseLmodModule     filename <- char ',' *> skipSpace *> kvTextParser "fn"-    return LmodLoad+    return ("lmod", LmodLoad         { info = info         , userload = userload         , modul = m         , filename = filename-        }+        })

@@ -9,6 +9,7 @@ import           Control.Applicative        ((<|>)) import           Data.Aeson                 (encode) import           Data.Attoparsec.Text+import           Data.Attoparsec.Time import qualified Data.ByteString.Char8      as SBS import qualified Data.ByteString.Lazy.Char8 as BS import           Data.Char@@ -16,6 +17,7 @@ import qualified Data.Text                  as T import qualified Data.Text.Encoding         as TE --------------------------------------------------------------------------------+import           HNormalise.Common.Parser import           HNormalise.Huppel.Parser import           HNormalise.Internal import           HNormalise.Json@@ -27,11 +29,11 @@  -------------------------------------------------------------------------------- -- | The 'parseMessage' function will try and use each configured parser to normalise the input it's given-parseMessage :: Parser ParseResult+parseMessage :: Parser (Text, ParseResult) parseMessage =-        (parseLmodLoad >>= (\v -> return $ PR_L v))-    <|> (parseShorewall >>= (\v -> return $ PR_S v))-    <|> (parseTorqueExit >>= (\v -> return $ PR_T v))+        (parseLmodLoad >>= (\(a, v) -> return $ (a, PR_L v)))+    <|> (parseShorewall >>= (\(a, v) -> return $ (a, PR_S v)))+    <|> (parseTorqueExit >>= (\(a, v) -> return $ (a, PR_T v)))  -------------------------------------------------------------------------------- -- | The 'getJsonKey' function return the key under which the normalised message should appear when JSON is produced@@ -44,27 +46,33 @@ -------------------------------------------------------------------------------- -- | The 'parseRsyslogLogstashString' currently is a placeholder function that will convert the incoming rsyslog message -- if it is encoded as expected in a plain string format+-- <%PRI%>1 %timegenerated:::date-rfc3339% %HOSTNAME% %syslogtag% - %APP-NAME%: %msg% parseRsyslogLogstashString :: Parser SBS.ByteString parseRsyslogLogstashString = do-    pri <- char '<' *> takeTill (== '>')-    char '>' *> decimal-    timegenerated <- skipSpace *> takeTill isSpace+    abspri <- maybeOption $ do+        char '<'+        p <- decimal+        char '>'+        v <- maybeOption $ decimal+        return (p, v)+    timegenerated <- skipSpace *> zonedTime     hostname <- skipSpace *> takeTill isSpace     syslogtag <- skipSpace *> takeTill isSpace  -- FIXME: this might be incorrect-    skipSpace *> char '-'-    appname <- skipSpace *> takeTill (== ':')-    (original, parsed) <- match $ char ':' *> skipSpace *> parseMessage+    skipSpace *> char '-' *> skipSpace+    (original, (appname, parsed)) <- match parseMessage     return $ let jsonkey = getJsonKey parsed              in BS.toStrict $ encode $ NRsyslog                     { rsyslog = Rsyslog                         { msg              = original-                        , timereported     = T.empty+                        , timereported     = Nothing                         , hostname         = hostname                         , syslogtag        = syslogtag                         , inputname        = T.empty                         , fromhost         = T.empty                         , fromhost_ip      = T.empty-                        , pri              = pri+                        , pri              = case abspri of+                                Just (p, _) -> Just p+                                Nothing     -> Nothing                         , syslogfacility   = T.empty                         , syslogseverity   = T.empty                         , timegenerated    = timegenerated

@@ -9,7 +9,7 @@ import           Control.Applicative        ((<|>)) import           Data.Attoparsec.Combinator (lookAhead, manyTill) import           Data.Attoparsec.Text-import qualified Data.Text                  as T+import           Data.Text                  (Text)  -------------------------------------------------------------------------------- import           HNormalise.Common.Parser@@ -18,7 +18,7 @@ -------------------------------------------------------------------------------- parseShorewallTCP :: Parser Shorewall parseShorewallTCP = do-    string " - kernel:: Shorewall:"+    string "kernel:: Shorewall:"     fwrule <- takeTill (== ':')     fwtarget <- char ':' *> takeTill (== ':')     fwin <- char ':' *> kvTextParser "IN"@@ -52,7 +52,7 @@ -------------------------------------------------------------------------------- parseShorewallUDP :: Parser Shorewall parseShorewallUDP = do-    string " - kernel:: Shorewall:"+    string "kernel:: Shorewall:"     fwrule <- takeTill (== ':')     fwtarget <- char ':' *> takeTill (== ':')     fwin <- char ':' *> kvTextParser "IN"@@ -87,7 +87,7 @@ -------------------------------------------------------------------------------- parseShorewallICMP :: Parser Shorewall parseShorewallICMP = do-    string " - kernel:: Shorewall:"+    string "kernel:: Shorewall:"     fwrule <- takeTill (== ':')     fwtarget <- char ':' *> takeTill (== ':')     fwin <- char ':' *> kvTextParser "IN"@@ -110,8 +110,9 @@         , fwdpt = Nothing         } -parseShorewall :: Parser Shorewall-parseShorewall =-        parseShorewallTCP-    <|> parseShorewallUDP-    <|> parseShorewallICMP+parseShorewall :: Parser (Text, Shorewall)+parseShorewall = do+    s <-    parseShorewallTCP+        <|> parseShorewallUDP+        <|> parseShorewallICMP+    return ("kernel", s)

@@ -141,7 +141,7 @@             return (lower, lower)  ---------------------------------------------------------------------------------parseTorqueExit :: Parser TorqueJobExit+parseTorqueExit :: Parser (Text, TorqueJobExit) parseTorqueExit = do     takeTill (== ';') *> string ";E;"   -- drop the prefix     name <- parseTorqueJobName@@ -164,7 +164,7 @@     exit_status <- skipSpace *> kvNumParser "Exit_status"     usage <- skipSpace *> parseTorqueResourceUsage -    return $ TorqueJobExit+    return $ ("torque", TorqueJobExit         { name = name         , user = user         , group = group@@ -185,4 +185,4 @@         , totalExecutionSlots = total_execution_slots         , uniqueNodeCount = unique_node_count         , exitStatus = exit_status-        }+        })

@@ -22,9 +22,9 @@  lmodLoadInput1 = "lmod::  username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text -shorewallTCPInput = " - kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0"-shorewallUDPInput = " - kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37"-shorewallICMPInput = " - kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421"+shorewallTCPInput = "kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0"+shorewallUDPInput = "kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37"+shorewallICMPInput = "kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421"  -------------------------------------------------------------------------------- main :: IO ()

@@ -31,7 +31,7 @@          it "parse module load" $ do             let s = "lmod::  username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text-            s ~> parseLmodLoad `shouldParse` LmodLoad+            s ~> parseLmodLoad `shouldParse` ("lmod", LmodLoad                 { info = LmodInfo                     { username = "myuser"                     , cluster = "mycluster"@@ -43,4 +43,4 @@                     , version = "2.3-intel-2016b"                     }                 , filename = "/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b"-                }+                })

@@ -23,7 +23,7 @@ spec = do     describe "parseShorewall" $ do         it "Shorewall UDP message" $ do-            let s = " - kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37" :: Text+            let s = "kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37" :: Text             s ~> parseShorewallUDP `shouldParse` Shorewall                 { fwrule = "ipmi2int"                 , fwtarget = "REJECT"@@ -38,7 +38,7 @@                 }          it "Shorewall TCP message" $ do-            let s = " - kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0" :: Text+            let s = "kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0" :: Text             s ~> parseShorewallTCP `shouldParse` Shorewall                 { fwrule = "ext2fw"                 , fwtarget = "REJECT"@@ -53,7 +53,7 @@                 }          it "Shorewall ICMP message" $ do-            let s = " - kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421" :: Text+            let s = "kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421" :: Text             s ~> parseShorewallICMP `shouldParse` Shorewall                 { fwrule = "ipmi2ext"                 , fwtarget = "REJECT"

@@ -107,7 +107,7 @@     describe "parseTorqueExit" $ do         it "parse job exit log line" $ do             let s = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text-            s ~> parseTorqueExit `shouldParse` TorqueJobExit+            s ~> parseTorqueExit `shouldParse` ("torque", TorqueJobExit                 { name = TorqueJobName { number = 45, array_id = Nothing, master = "master23", cluster = "banette" }                 , user = "vsc40075"                 , group = "vsc40075"@@ -159,4 +159,4 @@                 , totalExecutionSlots = 4                 , uniqueNodeCount = 2                 , exitStatus = 0-                }+                })