diff --git a/hnormalise.cabal b/hnormalise.cabal --- a/hnormalise.cabal +++ b/hnormalise.cabal @@ -1,5 +1,5 @@ name: hnormalise -version: 0.2.0.0 +version: 0.3.0.0 synopsis: Log message normalisation tool producing structured JSON messages description: Log message normalisation tool producing structured JSON messages homepage: https://github.com/itkovian/hnormalise#readme @@ -39,12 +39,14 @@ , aeson , aeson-pretty , attoparsec + , attoparsec-iso8601 , bytestring , containers , directory , ip , permute , text + , time , yaml default-language: Haskell2010 @@ -57,6 +59,7 @@ , aeson , aeson-pretty , attoparsec + , attoparsec-iso8601 , bytestring , conduit , conduit-combinators @@ -67,6 +70,7 @@ , optparse-applicative , resourcet , text + , time , word8 , yaml default-language: Haskell2010 @@ -82,6 +86,7 @@ build-depends: base , aeson , attoparsec + , attoparsec-iso8601 , conduit-extra , hnormalise , hspec @@ -90,6 +95,7 @@ , hspec-attoparsec , ip , text + , time ghc-options: -threaded -rtsopts -with-rtsopts=-N default-language: Haskell2010 diff --git a/src/HNormalise.hs b/src/HNormalise.hs --- a/src/HNormalise.hs +++ b/src/HNormalise.hs @@ -70,9 +70,9 @@ convertMessage :: Text -> Maybe ParseResult convertMessage message = case parse parseMessage message of - Done _ pm -> Just pm + Done _ (_, pm) -> Just pm Partial c -> case c empty of - Done _ pm -> Just pm + Done _ (_, pm) -> Just pm _ -> Nothing _ -> Nothing diff --git a/src/HNormalise/Internal.hs b/src/HNormalise/Internal.hs --- a/src/HNormalise/Internal.hs +++ b/src/HNormalise/Internal.hs @@ -9,6 +9,7 @@ import Data.Aeson (FromJSON, ToJSON, toEncoding, toJSON) import Data.Text +import Data.Time.LocalTime import GHC.Generics (Generic) -------------------------------------------------------------------------------- @@ -44,16 +45,16 @@ data Rsyslog = Rsyslog { msg :: !Text --, rawmsg :: !Text - , timereported :: !Text + , timereported :: !(Maybe ZonedTime) , hostname :: !Text , syslogtag :: !Text -- Could be a list? , inputname :: !Text , fromhost :: !Text , fromhost_ip :: !Text - , pri :: !Text + , pri :: !(Maybe Int) , syslogfacility :: !Text , syslogseverity :: !Text - , timegenerated :: !Text + , timegenerated :: !ZonedTime , programname :: !Text , protocol_version :: !Text --, structured_data :: !Text @@ -62,11 +63,11 @@ --, msgid :: !Text --, uuid :: !(Maybe Text) --, all_json :: !(Maybe Text) - } deriving (Eq, Show, Generic) + } deriving (Show, Generic) -------------------------------------------------------------------------------- data NormalisedRsyslog = NRsyslog { rsyslog :: Rsyslog -- ^ The original rsyslog message in a parsed form , normalised :: ParseResult -- ^ The normalised message , jsonkey :: Text -- ^ The key under which the normalised info will appear in the JSON result - } deriving (Eq, Show, Generic) + } deriving (Show, Generic) diff --git a/src/HNormalise/Json.hs b/src/HNormalise/Json.hs --- a/src/HNormalise/Json.hs +++ b/src/HNormalise/Json.hs @@ -47,5 +47,6 @@ pairs ( "message" .= msg r <> "syslog_abspri" .= syslogseverity r + <> "program" .= app_name r <> k .= n ) diff --git a/src/HNormalise/Lmod/Parser.hs b/src/HNormalise/Lmod/Parser.hs --- a/src/HNormalise/Lmod/Parser.hs +++ b/src/HNormalise/Lmod/Parser.hs @@ -9,6 +9,7 @@ import Control.Applicative ((<|>)) import Data.Attoparsec.Combinator (lookAhead, manyTill) import Data.Attoparsec.Text +import Data.Text (Text) -------------------------------------------------------------------------------- import HNormalise.Common.Parser @@ -35,16 +36,16 @@ , version = version } -parseLmodLoad :: Parser LmodLoad +parseLmodLoad :: Parser (Text, LmodLoad) parseLmodLoad = do string "lmod::" info <- skipSpace *> parseLmodInfo userload <- char ',' *> skipSpace *> kvYesNoParser "userload" m <- char ',' *> skipSpace *> parseLmodModule filename <- char ',' *> skipSpace *> kvTextParser "fn" - return LmodLoad + return ("lmod", LmodLoad { info = info , userload = userload , modul = m , filename = filename - } + }) diff --git a/src/HNormalise/Parser.hs b/src/HNormalise/Parser.hs --- a/src/HNormalise/Parser.hs +++ b/src/HNormalise/Parser.hs @@ -9,6 +9,7 @@ import Control.Applicative ((<|>)) import Data.Aeson (encode) import Data.Attoparsec.Text +import Data.Attoparsec.Time import qualified Data.ByteString.Char8 as SBS import qualified Data.ByteString.Lazy.Char8 as BS import Data.Char @@ -16,6 +17,7 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as TE -------------------------------------------------------------------------------- +import HNormalise.Common.Parser import HNormalise.Huppel.Parser import HNormalise.Internal import HNormalise.Json @@ -27,11 +29,11 @@ -------------------------------------------------------------------------------- -- | The 'parseMessage' function will try and use each configured parser to normalise the input it's given -parseMessage :: Parser ParseResult +parseMessage :: Parser (Text, ParseResult) parseMessage = - (parseLmodLoad >>= (\v -> return $ PR_L v)) - <|> (parseShorewall >>= (\v -> return $ PR_S v)) - <|> (parseTorqueExit >>= (\v -> return $ PR_T v)) + (parseLmodLoad >>= (\(a, v) -> return $ (a, PR_L v))) + <|> (parseShorewall >>= (\(a, v) -> return $ (a, PR_S v))) + <|> (parseTorqueExit >>= (\(a, v) -> return $ (a, PR_T v))) -------------------------------------------------------------------------------- -- | The 'getJsonKey' function return the key under which the normalised message should appear when JSON is produced @@ -44,27 +46,33 @@ -------------------------------------------------------------------------------- -- | The 'parseRsyslogLogstashString' currently is a placeholder function that will convert the incoming rsyslog message -- if it is encoded as expected in a plain string format +-- <%PRI%>1 %timegenerated:::date-rfc3339% %HOSTNAME% %syslogtag% - %APP-NAME%: %msg% parseRsyslogLogstashString :: Parser SBS.ByteString parseRsyslogLogstashString = do - pri <- char '<' *> takeTill (== '>') - char '>' *> decimal - timegenerated <- skipSpace *> takeTill isSpace + abspri <- maybeOption $ do + char '<' + p <- decimal + char '>' + v <- maybeOption $ decimal + return (p, v) + timegenerated <- skipSpace *> zonedTime hostname <- skipSpace *> takeTill isSpace syslogtag <- skipSpace *> takeTill isSpace -- FIXME: this might be incorrect - skipSpace *> char '-' - appname <- skipSpace *> takeTill (== ':') - (original, parsed) <- match $ char ':' *> skipSpace *> parseMessage + skipSpace *> char '-' *> skipSpace + (original, (appname, parsed)) <- match parseMessage return $ let jsonkey = getJsonKey parsed in BS.toStrict $ encode $ NRsyslog { rsyslog = Rsyslog { msg = original - , timereported = T.empty + , timereported = Nothing , hostname = hostname , syslogtag = syslogtag , inputname = T.empty , fromhost = T.empty , fromhost_ip = T.empty - , pri = pri + , pri = case abspri of + Just (p, _) -> Just p + Nothing -> Nothing , syslogfacility = T.empty , syslogseverity = T.empty , timegenerated = timegenerated diff --git a/src/HNormalise/Shorewall/Parser.hs b/src/HNormalise/Shorewall/Parser.hs --- a/src/HNormalise/Shorewall/Parser.hs +++ b/src/HNormalise/Shorewall/Parser.hs @@ -9,7 +9,7 @@ import Control.Applicative ((<|>)) import Data.Attoparsec.Combinator (lookAhead, manyTill) import Data.Attoparsec.Text -import qualified Data.Text as T +import Data.Text (Text) -------------------------------------------------------------------------------- import HNormalise.Common.Parser @@ -18,7 +18,7 @@ -------------------------------------------------------------------------------- parseShorewallTCP :: Parser Shorewall parseShorewallTCP = do - string " - kernel:: Shorewall:" + string "kernel:: Shorewall:" fwrule <- takeTill (== ':') fwtarget <- char ':' *> takeTill (== ':') fwin <- char ':' *> kvTextParser "IN" @@ -52,7 +52,7 @@ -------------------------------------------------------------------------------- parseShorewallUDP :: Parser Shorewall parseShorewallUDP = do - string " - kernel:: Shorewall:" + string "kernel:: Shorewall:" fwrule <- takeTill (== ':') fwtarget <- char ':' *> takeTill (== ':') fwin <- char ':' *> kvTextParser "IN" @@ -87,7 +87,7 @@ -------------------------------------------------------------------------------- parseShorewallICMP :: Parser Shorewall parseShorewallICMP = do - string " - kernel:: Shorewall:" + string "kernel:: Shorewall:" fwrule <- takeTill (== ':') fwtarget <- char ':' *> takeTill (== ':') fwin <- char ':' *> kvTextParser "IN" @@ -110,8 +110,9 @@ , fwdpt = Nothing } -parseShorewall :: Parser Shorewall -parseShorewall = - parseShorewallTCP - <|> parseShorewallUDP - <|> parseShorewallICMP +parseShorewall :: Parser (Text, Shorewall) +parseShorewall = do + s <- parseShorewallTCP + <|> parseShorewallUDP + <|> parseShorewallICMP + return ("kernel", s) diff --git a/src/HNormalise/Torque/Parser.hs b/src/HNormalise/Torque/Parser.hs --- a/src/HNormalise/Torque/Parser.hs +++ b/src/HNormalise/Torque/Parser.hs @@ -141,7 +141,7 @@ return (lower, lower) -------------------------------------------------------------------------------- -parseTorqueExit :: Parser TorqueJobExit +parseTorqueExit :: Parser (Text, TorqueJobExit) parseTorqueExit = do takeTill (== ';') *> string ";E;" -- drop the prefix name <- parseTorqueJobName @@ -164,7 +164,7 @@ exit_status <- skipSpace *> kvNumParser "Exit_status" usage <- skipSpace *> parseTorqueResourceUsage - return $ TorqueJobExit + return $ ("torque", TorqueJobExit { name = name , user = user , group = group @@ -185,4 +185,4 @@ , totalExecutionSlots = total_execution_slots , uniqueNodeCount = unique_node_count , exitStatus = exit_status - } + }) diff --git a/test/Bench.hs b/test/Bench.hs --- a/test/Bench.hs +++ b/test/Bench.hs @@ -22,9 +22,9 @@ lmodLoadInput1 = "lmod:: username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text -shorewallTCPInput = " - kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0" -shorewallUDPInput = " - kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37" -shorewallICMPInput = " - kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421" +shorewallTCPInput = "kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0" +shorewallUDPInput = "kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37" +shorewallICMPInput = "kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421" -------------------------------------------------------------------------------- main :: IO () diff --git a/test/HNormalise/Lmod/ParserSpec.hs b/test/HNormalise/Lmod/ParserSpec.hs --- a/test/HNormalise/Lmod/ParserSpec.hs +++ b/test/HNormalise/Lmod/ParserSpec.hs @@ -31,7 +31,7 @@ it "parse module load" $ do let s = "lmod:: username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text - s ~> parseLmodLoad `shouldParse` LmodLoad + s ~> parseLmodLoad `shouldParse` ("lmod", LmodLoad { info = LmodInfo { username = "myuser" , cluster = "mycluster" @@ -43,4 +43,4 @@ , version = "2.3-intel-2016b" } , filename = "/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" - } + }) diff --git a/test/HNormalise/Shorewall/ParserSpec.hs b/test/HNormalise/Shorewall/ParserSpec.hs --- a/test/HNormalise/Shorewall/ParserSpec.hs +++ b/test/HNormalise/Shorewall/ParserSpec.hs @@ -23,7 +23,7 @@ spec = do describe "parseShorewall" $ do it "Shorewall UDP message" $ do - let s = " - kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37" :: Text + let s = "kernel:: Shorewall:ipmi2int:REJECT:IN=em4 OUT=em1 SRC=10.0.0.2 DST=10.0.0.1 LEN=57 TOS=0x00 PREC=0x00 TTL=63 ID=62392 PROTO=UDP SPT=57002 DPT=53 LEN=37" :: Text s ~> parseShorewallUDP `shouldParse` Shorewall { fwrule = "ipmi2int" , fwtarget = "REJECT" @@ -38,7 +38,7 @@ } it "Shorewall TCP message" $ do - let s = " - kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0" :: Text + let s = "kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0" :: Text s ~> parseShorewallTCP `shouldParse` Shorewall { fwrule = "ext2fw" , fwtarget = "REJECT" @@ -53,7 +53,7 @@ } it "Shorewall ICMP message" $ do - let s = " - kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421" :: Text + let s = "kernel:: Shorewall:ipmi2ext:REJECT:IN=em4 OUT=em3 SRC=10.0.0.2 DST=10.0.0.1 LEN=28 TOS=0x00 PREC=0x00 TTL=63 ID=36216 PROTO=ICMP TYPE=8 CODE=0 ID=0 SEQ=1421" :: Text s ~> parseShorewallICMP `shouldParse` Shorewall { fwrule = "ipmi2ext" , fwtarget = "REJECT" diff --git a/test/HNormalise/Torque/ParserSpec.hs b/test/HNormalise/Torque/ParserSpec.hs --- a/test/HNormalise/Torque/ParserSpec.hs +++ b/test/HNormalise/Torque/ParserSpec.hs @@ -107,7 +107,7 @@ describe "parseTorqueExit" $ do it "parse job exit log line" $ do let s = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text - s ~> parseTorqueExit `shouldParse` TorqueJobExit + s ~> parseTorqueExit `shouldParse` ("torque", TorqueJobExit { name = TorqueJobName { number = 45, array_id = Nothing, master = "master23", cluster = "banette" } , user = "vsc40075" , group = "vsc40075" @@ -159,4 +159,4 @@ , totalExecutionSlots = 4 , uniqueNodeCount = 2 , exitStatus = 0 - } + })