diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -30,7 +30,7 @@ import Debug.Trace -------------------------------------------------------------------------------- import HNormalise -import HNormalise.Config (Config (..), loadConfig) +import HNormalise.Config (Config (..), PortConfig(..), loadConfig) import HNormalise.Internal (Rsyslog (..)) import HNormalise.Json @@ -75,6 +75,7 @@ ) -------------------------------------------------------------------------------- +-- | 'messageSink' yields the parsed JSON downstream, or if parsing fails, yields the original message downstream messageSink success failure = loop where loop = do @@ -92,6 +93,7 @@ -------------------------------------------------------------------------------- +-- | 'mySink' yields the results downstream with the addition of a string mentioning success or failure -- for testing purposes mySink = loop where @@ -112,6 +114,8 @@ -------------------------------------------------------------------------------- +-- | 'main' starts a TCP server, listening to incoming data and connecting to TCP servers downstream to +-- for the pipeline. main :: IO () main = do options <- OA.execParser parserInfo @@ -121,32 +125,34 @@ exitSuccess config <- loadConfig (oConfigFilePath options) + trace (show config) $ return () - let lHost = case listenHost config of + let lHost = case ports config >>= listenHost of Just h -> T.unpack h Nothing -> "*" + let fs = fields config - runTCPServer (serverSettings (fromJust $ listenPort config) "*") $ \appData -> do + runTCPServer (serverSettings (fromJust $ (ports config >>= listenPort)) "*") $ \appData -> do case oTestFilePath options of Nothing -> - runTCPClient (clientSettings (fromJust $ successPort config) "localhost") $ \successServer -> - runTCPClient (clientSettings (fromJust $ failPort config) "localhost") $ \failServer -> + runTCPClient (clientSettings (fromJust $ ports config >>= successPort) "localhost") $ \successServer -> + runTCPClient (clientSettings (fromJust $ ports config >>= failPort) "localhost") $ \failServer -> case oJsonInput options of True -> appSource appData $= CB.lines - $= C.map normaliseJsonInput + $= C.map (normaliseJsonInput fs) $$ messageSink successServer failServer False -> appSource appData $= DCT.decode DCT.utf8 $= DCT.lines - $= C.map normaliseText + $= C.map (normaliseText fs) $$ messageSink successServer failServer Just testSinkFileName -> runResourceT $ appSource appData $= (case oJsonInput options of - True -> CB.lines $= C.map normaliseJsonInput - False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map normaliseText) + True -> CB.lines $= C.map (normaliseJsonInput fs) + False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs)) $= mySink $$ sinkFile testSinkFileName diff --git a/hnormalise.cabal b/hnormalise.cabal --- a/hnormalise.cabal +++ b/hnormalise.cabal @@ -1,5 +1,5 @@ name: hnormalise -version: 0.3.2.0 +version: 0.3.3.0 synopsis: Log message normalisation tool producing structured JSON messages description: Log message normalisation tool producing structured JSON messages homepage: https://github.com/itkovian/hnormalise#readme @@ -50,6 +50,7 @@ , permute , text , time + , unordered-containers , yaml default-language: Haskell2010 diff --git a/src/HNormalise.hs b/src/HNormalise.hs --- a/src/HNormalise.hs +++ b/src/HNormalise.hs @@ -43,20 +43,22 @@ -------------------------------------------------------------------------------- -- | The 'normaliseJsonInput' function converts a `ByteString` to a normalised message or keeps the original if -- the conversion (parsing) fails. -normaliseJsonInput :: SBS.ByteString -- ^ Input representing an rsyslog message in JSON format - -> Normalised -- ^ Transformed or Original result -normaliseJsonInput logLine = - case (Aeson.decodeStrict logLine :: Maybe Rsyslog) >>= normaliseRsyslog of +normaliseJsonInput :: Maybe [(Text, Text)] -- ^ Output fields + -> SBS.ByteString -- ^ Input representing an rsyslog message in JSON format + -> Normalised -- ^ Transformed or Original result +normaliseJsonInput fs logLine = + case (Aeson.decodeStrict logLine :: Maybe Rsyslog) >>= (normaliseRsyslog fs) of Just j -> Transformed j Nothing -> Original logLine -------------------------------------------------------------------------------- -- | The 'normaliseText' function converts a 'Text' to a normalised message or keeps the original (in 'ByteString') -- format if the conversion fails -normaliseText :: Text -- ^ Input - -> Normalised -- ^ Transformed or Original result -normaliseText logLine = - case parse parseRsyslogLogstashString logLine of +normaliseText :: Maybe [(Text, Text)] -- ^ Output fields + -> Text -- ^ Input + -> Normalised -- ^ Transformed or Original result +normaliseText fs logLine = + case parse (parseRsyslogLogstashString fs) logLine of Done _ r -> Transformed r Partial c -> case c empty of Done _ r -> Transformed r @@ -68,7 +70,8 @@ -------------------------------------------------------------------------------- -- | The 'convertMessage' function transforms the actual message to a 'Maybe' 'ParseResult'. If parsing fails, -- the result is 'Nothing'. -convertMessage :: Text -> Maybe ParseResult +convertMessage :: Text -- ^ The actual message part of the incoming JSON payload + -> Maybe ParseResult -- ^ The resulting Haskell data structure wrapped in a ParseResult convertMessage message = case parse parseMessage message of Done _ (_, pm) -> Just pm @@ -80,10 +83,11 @@ -------------------------------------------------------------------------------- -- | The 'normaliseRsyslog' function returns an 'NRSyslog' structure tranformed to a 'ByteString' or 'Nothing' -- when parsing fails -normaliseRsyslog :: Rsyslog -- ^ Incoming rsyslog information - -> Maybe SBS.ByteString -- ^ IF the conversion succeeded the JSON encoded rsyslog message to forward -normaliseRsyslog rsyslog = do +normaliseRsyslog :: Maybe [(Text, Text)] -- ^ Output fields + -> Rsyslog -- ^ Incoming rsyslog information + -> Maybe SBS.ByteString -- ^ If the conversion succeeded the JSON encoded rsyslog message to forward +normaliseRsyslog fs rsyslog = do cm <- convertMessage $ msg rsyslog return $ BS.toStrict $ Aeson.encode - $ NRsyslog { rsyslog = rsyslog, normalised = cm, jsonkey = getJsonKey cm } + $ NRsyslog { rsyslog = rsyslog, normalised = cm, jsonkey = getJsonKey cm, fields = fs } diff --git a/src/HNormalise/Config.hs b/src/HNormalise/Config.hs --- a/src/HNormalise/Config.hs +++ b/src/HNormalise/Config.hs @@ -4,6 +4,7 @@ module HNormalise.Config ( Config(..) + , PortConfig(..) , loadConfig ) where @@ -19,7 +20,7 @@ -------------------------------------------------------------------------------- -data Config = Config +data PortConfig = PortConfig { listenPort :: !(Maybe Int) -- ^ port for incoming messages , listenHost :: !(Maybe Text) -- ^ binding to this host specification (TODO: needs support for HostPreference) , successPort :: !(Maybe Int) -- ^ port to send rsyslog with successfully parsed and normalised msg part @@ -29,10 +30,10 @@ } deriving (Show) -------------------------------------------------------------------------------- -instance Monoid Config where - mempty = Config +instance Monoid PortConfig where + mempty = PortConfig Nothing Nothing Nothing Nothing Nothing Nothing - mappend l r = Config + mappend l r = PortConfig { listenPort = listenPort l `mplus` listenPort r , listenHost = listenHost l `mplus` listenHost r , successPort = successPort l `mplus` successPort r @@ -42,7 +43,7 @@ } -------------------------------------------------------------------------------- -defaultConfig = Config +defaultPortConfig = PortConfig { listenPort = Just 4019 , listenHost = Just "localhost" , successPort = Just 26002 @@ -52,6 +53,25 @@ } -------------------------------------------------------------------------------- +data Config = Config + { ports :: !(Maybe PortConfig) + , fields :: !(Maybe [(Text, Text)]) + } deriving (Show) + +-------------------------------------------------------------------------------- +instance Monoid Config where + mempty = Config Nothing Nothing + mappend l r = Config + { ports = ports l `mplus` ports r + , fields = fields l `mplus` fields r + } + +defaultConfig = Config + { ports = Just defaultPortConfig + , fields = Nothing + } + +-------------------------------------------------------------------------------- systemConfigFileLocation :: FilePath systemConfigFileLocation = "/etc/hnormalise.yaml" @@ -78,4 +98,5 @@ return $ userConfig <> systemConfig <> defaultConfig -------------------------------------------------------------------------------- +$(deriveJSON defaultOptions ''PortConfig) $(deriveJSON defaultOptions ''Config) diff --git a/src/HNormalise/Internal.hs b/src/HNormalise/Internal.hs --- a/src/HNormalise/Internal.hs +++ b/src/HNormalise/Internal.hs @@ -1,27 +1,26 @@ {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} -{-# LANGUAGE ExistentialQuantification #-} {-# LANGUAGE OverloadedStrings #-} module HNormalise.Internal where -------------------------------------------------------------------------------- -import Data.Aeson (FromJSON, ToJSON, toEncoding, - toJSON) -import Data.Text -import Data.Time.LocalTime -import GHC.Generics (Generic) +import Data.Aeson (FromJSON, ToJSON, toEncoding, + toJSON) +import Data.Text (Text) +import Data.Time.LocalTime (ZonedTime) +import GHC.Generics (Generic) -------------------------------------------------------------------------------- -import HNormalise.Huppel.Internal (Huppel) +import HNormalise.Huppel.Internal (Huppel) import HNormalise.Huppel.Json -import HNormalise.Lmod.Internal (LmodLoad) +import HNormalise.Lmod.Internal (LmodLoad) import HNormalise.Lmod.Json import HNormalise.Shorewall.Internal (Shorewall) import HNormalise.Shorewall.Json -import HNormalise.Snoopy.Internal (Snoopy) +import HNormalise.Snoopy.Internal (Snoopy) import HNormalise.Snoopy.Json -import HNormalise.Torque.Internal (TorqueJobExit) +import HNormalise.Torque.Internal (TorqueJobExit) import HNormalise.Torque.Json -------------------------------------------------------------------------------- @@ -50,16 +49,17 @@ data Rsyslog = Rsyslog { msg :: !Text --, rawmsg :: !Text - , timereported :: !(Maybe ZonedTime) + , timereported :: !ZonedTime , hostname :: !Text , syslogtag :: !Text -- Could be a list? , inputname :: !Text , fromhost :: !Text , fromhost_ip :: !Text , pri :: !(Maybe Int) + , version :: !(Maybe Int) , syslogfacility :: !Text , syslogseverity :: !Text - , timegenerated :: !ZonedTime + , timegenerated :: !(Maybe ZonedTime) , programname :: !Text , protocol_version :: !Text --, structured_data :: !Text @@ -75,4 +75,5 @@ { rsyslog :: Rsyslog -- ^ The original rsyslog message in a parsed form , normalised :: ParseResult -- ^ The normalised message , jsonkey :: Text -- ^ The key under which the normalised info will appear in the JSON result + , fields :: Maybe [(Text, Text)] -- ^ The fields we need to output when creating the JSON encoding as (key, fieldname) } deriving (Show, Generic) diff --git a/src/HNormalise/Json.hs b/src/HNormalise/Json.hs --- a/src/HNormalise/Json.hs +++ b/src/HNormalise/Json.hs @@ -7,6 +7,7 @@ -------------------------------------------------------------------------------- import Control.Monad import Data.Aeson +import qualified Data.HashMap.Lazy as M import Data.Monoid -------------------------------------------------------------------------------- @@ -25,6 +26,7 @@ (v .: "fromhost") <*> (v .: "fromhost-ip") <*> (v .: "pri") <*> + (v .: "version") <*> -- no idea with what the version matches in the JSON message format, see also http://www.rsyslog.com/doc/master/configuration/properties.html (v .: "syslogfacility") <*> (v .: "syslogseverity") <*> (v .: "timegenerated") <*> @@ -40,13 +42,21 @@ -------------------------------------------------------------------------------- instance ToJSON Rsyslog where toEncoding = genericToEncoding defaultOptions + toJSON = genericToJSON defaultOptions -------------------------------------------------------------------------------- instance ToJSON NormalisedRsyslog where - toEncoding (NRsyslog r n k) = - pairs - ( "message" .= msg r - <> "syslog_abspri" .= syslogseverity r - <> "program" .= app_name r - <> k .= n - ) + toEncoding (NRsyslog r n k fs) = + case fs of + -- this is the default + Nothing -> pairs + ( "message" .= msg r + <> "syslog_abspri" .= pri r + <> "syslog_version" .= version r + <> "program" .= app_name r + <> "@source_host" .= hostname r -- the host in ES will likely be set to the machine sending the data to logstash + <> k .= n + ) + Just fs' -> case toJSON r of + Object syslog -> pairs $ foldl (\v (key, fieldname) -> v <> key .= (M.lookupDefault Null fieldname syslog)) (k .= n) fs' + _ -> pairs (k .= n) -- FIXME: this should never happen diff --git a/src/HNormalise/Parser.hs b/src/HNormalise/Parser.hs --- a/src/HNormalise/Parser.hs +++ b/src/HNormalise/Parser.hs @@ -39,7 +39,8 @@ -------------------------------------------------------------------------------- -- | The 'getJsonKey' function return the key under which the normalised message should appear when JSON is produced -getJsonKey :: ParseResult -> Text +getJsonKey :: ParseResult -- ^ Wrapped result for which we need to get a key + -> Text -- ^ Key for use in the JSON encoding of the result getJsonKey (PR_Huppel _) = "huppel" getJsonKey (PR_Lmod _) = "lmod" getJsonKey (PR_Torque _) = "torque" @@ -50,15 +51,16 @@ -- | The 'parseRsyslogLogstashString' currently is a placeholder function that will convert the incoming rsyslog message -- if it is encoded as expected in a plain string format -- <%PRI%>1 %timegenerated:::date-rfc3339% %HOSTNAME% %syslogtag% - %APP-NAME%: %msg% -parseRsyslogLogstashString :: Parser SBS.ByteString -parseRsyslogLogstashString = do +parseRsyslogLogstashString :: Maybe [(Text, Text)] -- ^ Output fields + -> Parser SBS.ByteString -- ^ Resulting encoded JSON representation +parseRsyslogLogstashString fs = do abspri <- maybeOption $ do char '<' p <- decimal char '>' v <- maybeOption $ decimal return (p, v) - timegenerated <- skipSpace *> zonedTime + timereported <- skipSpace *> zonedTime hostname <- skipSpace *> takeTill isSpace syslogtag <- skipSpace *> takeTill isSpace -- FIXME: this might be incorrect skipSpace *> char '-' *> skipSpace @@ -67,18 +69,17 @@ in BS.toStrict $ encode $ NRsyslog { rsyslog = Rsyslog { msg = original - , timereported = Nothing + , timereported = timereported , hostname = hostname , syslogtag = syslogtag , inputname = T.empty , fromhost = T.empty , fromhost_ip = T.empty - , pri = case abspri of - Just (p, _) -> Just p - Nothing -> Nothing + , pri = abspri >>= \(p, _) -> return p + , version = abspri >>= \(_, v) -> v , syslogfacility = T.empty , syslogseverity = T.empty - , timegenerated = timegenerated + , timegenerated = Nothing , programname = T.empty , protocol_version = T.empty , app_name = appname @@ -86,4 +87,5 @@ } , normalised = parsed , jsonkey = jsonkey + , fields = fs } diff --git a/src/HNormalise/Torque/Internal.hs b/src/HNormalise/Torque/Internal.hs --- a/src/HNormalise/Torque/Internal.hs +++ b/src/HNormalise/Torque/Internal.hs @@ -25,8 +25,7 @@ -------------------------------------------------------------------------------- data TorqueExecHost = TorqueExecHost { name :: !Text - , lowerCore :: !Int - , upperCore :: !Int + , cores :: ![Int] } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- @@ -84,6 +83,7 @@ , owner :: !Text , session :: !Integer , times :: !TorqueJobTime + , execHost :: ![TorqueExecHost] , resourceRequest :: !TorqueResourceRequest , resourceUsage :: !TorqueResourceUsage , totalExecutionSlots :: !Int diff --git a/src/HNormalise/Torque/Json.hs b/src/HNormalise/Torque/Json.hs --- a/src/HNormalise/Torque/Json.hs +++ b/src/HNormalise/Torque/Json.hs @@ -23,7 +23,7 @@ toEncoding = genericToEncoding defaultOptions instance ToJSON TorqueWalltime where - toEncoding = genericToEncoding defaultOptions + toEncoding (TorqueWalltime d h m s) = toEncoding $ (((d * 24 + h) * 60) + m) * 60 + s instance ToJSON TorqueResourceRequest where toEncoding = genericToEncoding defaultOptions diff --git a/src/HNormalise/Torque/Parser.hs b/src/HNormalise/Torque/Parser.hs --- a/src/HNormalise/Torque/Parser.hs +++ b/src/HNormalise/Torque/Parser.hs @@ -20,6 +20,7 @@ import HNormalise.Torque.Internal -------------------------------------------------------------------------------- +-- | 'parseTorqueWalltime' parses [[[DD:]HH:]MM:]SS strings representing walltime parseTorqueWalltime :: Parser TorqueWalltime parseTorqueWalltime = parseTorqueDays @@ -51,6 +52,7 @@ -------------------------------------------------------------------------------- +-- | 'parseTorqueMemory' parses an decimal followed by a memory unit and return the memory in bytes parseTorqueMemory :: Parser Integer parseTorqueMemory = do v <- decimal @@ -65,6 +67,7 @@ "gb" -> v * 1024 * 1024 * 1024 -------------------------------------------------------------------------------- +-- | 'parseTorqueJobName' splits the job name in its components, i.e., ID, [ array ID,] master and cluster parseTorqueJobName :: Parser TorqueJobName parseTorqueJobName = do n <- decimal @@ -83,6 +86,7 @@ -------------------------------------------------------------------------------- +-- | 'parseTorqueResourceNodeList' parses a list of FQDN nodes and their ppn or a nodecount and its ppn parseTorqueResourceNodeList :: Parser (Either TorqueJobShortNode [TorqueJobFQNode]) parseTorqueResourceNodeList = do c <- peekChar' @@ -112,6 +116,9 @@ Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.walltime -} +-- | 'parseTorqueResourceRequest' parses all key value pairs denoting resources requested. +-- Most of these are not obligatory. Since the Torque documentation is vague on mentioning which entries occur, the last +-- 1.5 years of data we have were used to make an educated guess as to which keys might appear and in what order parseTorqueResourceRequest :: Parser TorqueResourceRequest parseTorqueResourceRequest = do permute $ TorqueResourceRequest @@ -131,6 +138,7 @@ <||> skipSpace *> string "Resource_List.walltime=" *> parseTorqueWalltime -------------------------------------------------------------------------------- +-- | 'parseTorqueResourceUsage' parses all the key value pairs denoting used resources. parseTorqueResourceUsage :: Parser TorqueResourceUsage parseTorqueResourceUsage = do cput <- skipSpace *> kvNumParser "resources_used.cput" @@ -147,24 +155,30 @@ } -------------------------------------------------------------------------------- +-- | 'parseTorqueHostList' parses a '+' separated list of hostname/coreranges +-- A core range can be of the form 1,3,5-7,9 parseTorqueHostList :: Parser [TorqueExecHost] parseTorqueHostList = flip sepBy (char '+') $ do fqdn <- Data.Attoparsec.Text.takeWhile (/= '/') char '/' - (lower, upper) <- try parseCoreRange <|> parseSingleCore - return $ TorqueExecHost { name = fqdn, lowerCore = lower, upperCore = upper} - where parseCoreRange :: Parser (Int, Int) - parseCoreRange = do + cores <- parseCores + return $ TorqueExecHost { name = fqdn, cores = cores} + where parseCores :: Parser [Int] + parseCores = do + cores <- flip sepBy1' (char ',') $ try parseRange <|> parseSingle + return $ concat cores + parseRange = do lower <- decimal char '-' upper <- decimal - return (lower, upper) + return [lower .. upper] + parseSingle = do + c <- decimal + return [c] - parseSingleCore = do - lower <- decimal - return (lower, lower) -------------------------------------------------------------------------------- +-- | 'parseTorqueExit' parses a complete log line denoting a job exit. Tested with Torque 6.1.x. parseTorqueExit :: Parser (Text, TorqueJobExit) parseTorqueExit = do takeTill (== ';') *> string ";E;" -- drop the prefix @@ -204,6 +218,7 @@ , startTime = start , endTime = end } + , execHost = exec_host , resourceRequest = request , resourceUsage = usage , totalExecutionSlots = total_execution_slots diff --git a/test/Bench.hs b/test/Bench.hs --- a/test/Bench.hs +++ b/test/Bench.hs @@ -57,7 +57,7 @@ , bgroup "parse snoopy" [ bench "successfull input" $ whnf (AT.parse SnoopyP.parseSnoopy) snoopyInput] , bgroup "normaliseText" - [ bench "snoopy" $ nf H.normaliseText fullSnoopyInput - , bench "lmod" $ nf H.normaliseText fullLmodInput + [ bench "snoopy" $ nf (H.normaliseText Nothing) fullSnoopyInput + , bench "lmod" $ nf (H.normaliseText Nothing) fullLmodInput ] ] diff --git a/test/HNormalise/ParserSpec.hs b/test/HNormalise/ParserSpec.hs --- a/test/HNormalise/ParserSpec.hs +++ b/test/HNormalise/ParserSpec.hs @@ -25,13 +25,17 @@ describe "parse full rsyslog message" $ do it "lmod load" $ do let s = "<13>1 2016-06-07T17:50:22.658452+02:00 node2159 lmod: - lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016" :: Text - s ~> parseRsyslogLogstashString `shouldParse` - "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":\"\",\"program\":\"lmod\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}" + s ~> (parseRsyslogLogstashString Nothing) `shouldParse` + "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}" it "imfile torque input" $ do let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text - s ~> parseRsyslogLogstashString `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":\"\",\"program\":\"torque\",\"torque\":{\"name\":{\"number\":0,\"array_id\":null,\"master\":\"master23\",\"cluster\":\"banette\"},\"user\":\"vsc40075\",\"group\":\"vsc40075\",\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"vsc40075@gligar03.gligar.gent.vsc\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"neednodes\":{\"Left\":{\"number\":1,\"ppn\":null}},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"Left\":{\"number\":1,\"ppn\":null}},\"select\":null,\"qos\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"walltime\":{\"days\":0,\"hours\":1,\"minutes\":0,\"seconds\":0}},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":{\"days\":0,\"hours\":0,\"minutes\":44,\"seconds\":4}},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265}}" + s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"test2802\",\"torque\":{\"name\":{\"number\":0,\"array_id\":null,\"master\":\"master23\",\"cluster\":\"banette\"},\"user\":\"vsc40075\",\"group\":\"vsc40075\",\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"vsc40075@gligar03.gligar.gent.vsc\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"execHost\":[{\"name\":\"exec_host=node2801.banette.gent.vsc\",\"cores\":[0]}],\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"neednodes\":{\"Left\":{\"number\":1,\"ppn\":null}},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"Left\":{\"number\":1,\"ppn\":null}},\"select\":null,\"qos\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"walltime\":3600},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":2644},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265}}" it "snoopy with process ID" $ do let s = "<86>1 2017-05-29T16:40:48.275334+02:00 master23 snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text - s ~> parseRsyslogLogstashString `shouldParse` "{\"message\":\"snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":\"\",\"program\":\"snoopy\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}" + s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":86,\"syslog_version\":1,\"program\":\"snoopy\",\"@source_host\":\"master23\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}" + + it "snoopy with process ID; only the @source_host" $ do + let s = "<86>1 2017-05-29T16:40:48.275334+02:00 master23 snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text + s ~> (parseRsyslogLogstashString $ Just [("@source_host", "hostname")]) `shouldParse` "{\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"},\"@source_host\":\"master23\"}" diff --git a/test/HNormalise/Torque/ParserSpec.hs b/test/HNormalise/Torque/ParserSpec.hs --- a/test/HNormalise/Torque/ParserSpec.hs +++ b/test/HNormalise/Torque/ParserSpec.hs @@ -200,6 +200,38 @@ , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } + describe "parseTorqueHostList" $ do + it "parse comma separated list of single cores" $ do + let s = "node1001.mycluster.mydomain/1,3,5,7" :: Text + s ~> parseTorqueHostList `shouldParse` [ TorqueExecHost + { name = "node1001.mycluster.mydomain" + , cores = [1,3,5,7] + }] + it "parse comma separated list of core rangess" $ do + let s = "node1001.mycluster.mydomain/1-3,5-7" :: Text + s ~> parseTorqueHostList `shouldParse` [ TorqueExecHost + { name = "node1001.mycluster.mydomain" + , cores = [1,2,3,5,6,7] + }] + it "parse comma separated list of single cores and core ranges" $ do + let s = "node1001.mycluster.mydomain/1,3,5-7,9,12-14" :: Text + s ~> parseTorqueHostList `shouldParse` [ TorqueExecHost + { name = "node1001.mycluster.mydomain" + , cores = [1,3,5,6,7,9,12,13,14] + }] + it "parse multiple nodes with one comma separated list of single cores and one core range" $ do + let s = "node1001.mycluster.mydomain/1,3,5,7+node1002.mycluster.mydomain/4-6" :: Text + s ~> parseTorqueHostList `shouldParse` [ + TorqueExecHost + { name = "node1001.mycluster.mydomain" + , cores = [1,3,5,7] + }, + TorqueExecHost + { name = "node1002.mycluster.mydomain" + , cores = [4,5,6] + } + ] + describe "parseTorqueExit" $ do it "parse job exit log line" $ do let s = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text @@ -219,6 +251,16 @@ , startTime = 1491390307 , endTime = 1491390413 } + , execHost = + [ TorqueExecHost + { name = "exec_host=node2801.banette.gent.vsc" + , cores = [0,1] + } + , TorqueExecHost + { name = "node2803.banette.gent.vsc" + , cores = [0,1] + } + ] , resourceRequest = TorqueResourceRequest { mem = Nothing , advres = Nothing