hnormalise 0.3.2.0 → 0.3.3.0

raw patch · 13 files changed

+179/−73 lines, 13 filesdep +unordered-containersPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: unordered-containers

API changes (from Hackage documentation)

Files

• app/Main.hs +15/−9
• hnormalise.cabal +2/−1
• src/HNormalise.hs +17/−13
• src/HNormalise/Config.hs +26/−5
• src/HNormalise/Internal.hs +13/−12
• src/HNormalise/Json.hs +17/−7
• src/HNormalise/Parser.hs +11/−9
• src/HNormalise/Torque/Internal.hs +2/−2
• src/HNormalise/Torque/Json.hs +1/−1
• src/HNormalise/Torque/Parser.hs +23/−8
• test/Bench.hs +2/−2
• test/HNormalise/ParserSpec.hs +8/−4
• test/HNormalise/Torque/ParserSpec.hs +42/−0

@@ -30,7 +30,7 @@ import           Debug.Trace -------------------------------------------------------------------------------- import           HNormalise-import           HNormalise.Config            (Config (..), loadConfig)+import           HNormalise.Config            (Config (..), PortConfig(..), loadConfig) import           HNormalise.Internal          (Rsyslog (..)) import           HNormalise.Json @@ -75,6 +75,7 @@     )  --------------------------------------------------------------------------------+-- | 'messageSink' yields the parsed JSON downstream, or if parsing fails, yields the original message downstream messageSink success failure = loop   where     loop = do@@ -92,6 +93,7 @@   --------------------------------------------------------------------------------+-- | 'mySink' yields the results downstream with the addition of a string mentioning success or failure -- for testing purposes mySink = loop   where@@ -112,6 +114,8 @@   --------------------------------------------------------------------------------+-- | 'main' starts a TCP server, listening to incoming data and connecting to TCP servers downstream to+-- for the pipeline. main :: IO () main = do     options <- OA.execParser parserInfo@@ -121,32 +125,34 @@         exitSuccess      config <- loadConfig (oConfigFilePath options)+    trace (show config) $ return () -    let lHost = case listenHost config of+    let lHost = case ports config >>= listenHost of                     Just h  -> T.unpack h                     Nothing -> "*" +    let fs = fields config -    runTCPServer (serverSettings (fromJust $ listenPort config) "*") $ \appData -> do+    runTCPServer (serverSettings (fromJust $ (ports config >>= listenPort)) "*") $ \appData -> do          case oTestFilePath options of             Nothing ->-                runTCPClient (clientSettings (fromJust $ successPort config) "localhost") $ \successServer ->-                runTCPClient (clientSettings (fromJust $ failPort config) "localhost") $ \failServer ->+                runTCPClient (clientSettings (fromJust $ ports config >>= successPort) "localhost") $ \successServer ->+                runTCPClient (clientSettings (fromJust $ ports config >>= failPort) "localhost") $ \failServer ->                     case oJsonInput options of                         True  -> appSource appData                                     $= CB.lines-                                    $= C.map normaliseJsonInput+                                    $= C.map (normaliseJsonInput fs)                                     $$ messageSink successServer failServer                         False -> appSource appData                                     $= DCT.decode DCT.utf8                                     $= DCT.lines-                                    $= C.map normaliseText+                                    $= C.map (normaliseText fs)                                     $$ messageSink successServer failServer             Just testSinkFileName ->                 runResourceT $ appSource appData                     $= (case oJsonInput options of-                        True  -> CB.lines $= C.map normaliseJsonInput-                        False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map normaliseText)+                        True  -> CB.lines $= C.map (normaliseJsonInput fs)+                        False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs))                     $= mySink                     $$ sinkFile testSinkFileName

@@ -1,5 +1,5 @@ name:                hnormalise-version:             0.3.2.0+version:             0.3.3.0 synopsis:            Log message normalisation tool producing structured JSON messages description:         Log message normalisation tool producing structured JSON messages homepage:            https://github.com/itkovian/hnormalise#readme@@ -50,6 +50,7 @@                      , permute                      , text                      , time+                     , unordered-containers                      , yaml   default-language:    Haskell2010 

@@ -43,20 +43,22 @@ -------------------------------------------------------------------------------- -- | The 'normaliseJsonInput' function converts a `ByteString` to a normalised message or keeps the original if -- the conversion (parsing) fails.-normaliseJsonInput :: SBS.ByteString    -- ^ Input representing an rsyslog message in JSON format-                   -> Normalised        -- ^ Transformed or Original result-normaliseJsonInput logLine =-    case (Aeson.decodeStrict logLine :: Maybe Rsyslog) >>= normaliseRsyslog of+normaliseJsonInput :: Maybe [(Text, Text)]    -- ^ Output fields+                   -> SBS.ByteString          -- ^ Input representing an rsyslog message in JSON format+                   -> Normalised              -- ^ Transformed or Original result+normaliseJsonInput fs logLine =+    case (Aeson.decodeStrict logLine :: Maybe Rsyslog) >>= (normaliseRsyslog fs) of         Just j  -> Transformed j         Nothing -> Original logLine  -------------------------------------------------------------------------------- -- | The 'normaliseText' function converts a 'Text' to a normalised message or keeps the original (in 'ByteString') -- format if the conversion fails-normaliseText :: Text          -- ^ Input-              -> Normalised    -- ^ Transformed or Original result-normaliseText logLine =-    case parse parseRsyslogLogstashString logLine of+normaliseText :: Maybe [(Text, Text)]  -- ^ Output fields+              -> Text                  -- ^ Input+              -> Normalised            -- ^ Transformed or Original result+normaliseText fs logLine =+    case parse (parseRsyslogLogstashString fs) logLine of         Done _ r    -> Transformed r         Partial c   -> case c empty of                             Done _ r -> Transformed r@@ -68,7 +70,8 @@ -------------------------------------------------------------------------------- -- | The 'convertMessage' function transforms the actual message to a 'Maybe' 'ParseResult'. If parsing fails, -- the result is 'Nothing'.-convertMessage :: Text -> Maybe ParseResult+convertMessage :: Text               -- ^ The actual message part of the incoming JSON payload+               -> Maybe ParseResult  -- ^ The resulting Haskell data structure wrapped in a ParseResult convertMessage message =     case parse parseMessage message of         Done _ (_, pm) -> Just pm@@ -80,10 +83,11 @@ -------------------------------------------------------------------------------- -- | The 'normaliseRsyslog' function returns an 'NRSyslog' structure tranformed to a 'ByteString' or 'Nothing' -- when parsing fails-normaliseRsyslog :: Rsyslog               -- ^ Incoming rsyslog information-                 -> Maybe SBS.ByteString  -- ^ IF the conversion succeeded the JSON encoded rsyslog message to forward-normaliseRsyslog rsyslog = do+normaliseRsyslog :: Maybe [(Text, Text)]   -- ^ Output fields+                 -> Rsyslog                -- ^ Incoming rsyslog information+                 -> Maybe SBS.ByteString   -- ^ If the conversion succeeded the JSON encoded rsyslog message to forward+normaliseRsyslog fs rsyslog = do     cm <- convertMessage $ msg rsyslog     return $ BS.toStrict            $ Aeson.encode-           $ NRsyslog { rsyslog = rsyslog, normalised = cm, jsonkey = getJsonKey cm }+           $ NRsyslog { rsyslog = rsyslog, normalised = cm, jsonkey = getJsonKey cm, fields = fs }

@@ -4,6 +4,7 @@  module HNormalise.Config     ( Config(..)+    , PortConfig(..)     , loadConfig     ) where @@ -19,7 +20,7 @@   ---------------------------------------------------------------------------------data Config = Config+data PortConfig = PortConfig     { listenPort  :: !(Maybe Int)    -- ^ port for incoming messages     , listenHost  :: !(Maybe Text)   -- ^ binding to this host specification (TODO: needs support for HostPreference)     , successPort :: !(Maybe Int)    -- ^ port to send rsyslog with successfully parsed and normalised msg part@@ -29,10 +30,10 @@     } deriving (Show)  ---------------------------------------------------------------------------------instance Monoid Config where-    mempty = Config+instance Monoid PortConfig where+    mempty = PortConfig                 Nothing Nothing Nothing Nothing Nothing Nothing-    mappend l r = Config+    mappend l r = PortConfig         { listenPort  = listenPort  l `mplus` listenPort  r         , listenHost  = listenHost  l `mplus` listenHost  r         , successPort = successPort l `mplus` successPort r@@ -42,7 +43,7 @@         }  ---------------------------------------------------------------------------------defaultConfig = Config+defaultPortConfig = PortConfig     { listenPort = Just 4019     , listenHost = Just "localhost"     , successPort = Just 26002@@ -52,6 +53,25 @@     }  --------------------------------------------------------------------------------+data Config = Config+    { ports :: !(Maybe PortConfig)+    , fields :: !(Maybe [(Text, Text)])+    } deriving (Show)++--------------------------------------------------------------------------------+instance Monoid Config where+    mempty = Config Nothing Nothing+    mappend l r = Config+        { ports = ports l `mplus` ports r+        , fields = fields l `mplus` fields r+        }++defaultConfig = Config+    { ports = Just defaultPortConfig+    , fields = Nothing+    }++-------------------------------------------------------------------------------- systemConfigFileLocation :: FilePath systemConfigFileLocation = "/etc/hnormalise.yaml" @@ -78,4 +98,5 @@     return $ userConfig <> systemConfig <> defaultConfig  --------------------------------------------------------------------------------+$(deriveJSON defaultOptions ''PortConfig) $(deriveJSON defaultOptions ''Config)

@@ -1,27 +1,26 @@ {-# LANGUAGE DeriveGeneric             #-} {-# LANGUAGE DuplicateRecordFields     #-}-{-# LANGUAGE ExistentialQuantification #-} {-# LANGUAGE OverloadedStrings         #-}  module HNormalise.Internal where  ---------------------------------------------------------------------------------import           Data.Aeson                 (FromJSON, ToJSON, toEncoding,-                                             toJSON)-import           Data.Text-import           Data.Time.LocalTime-import           GHC.Generics               (Generic)+import           Data.Aeson                    (FromJSON, ToJSON, toEncoding,+                                                toJSON)+import           Data.Text                     (Text)+import           Data.Time.LocalTime           (ZonedTime)+import           GHC.Generics                  (Generic)  ---------------------------------------------------------------------------------import           HNormalise.Huppel.Internal (Huppel)+import           HNormalise.Huppel.Internal    (Huppel) import           HNormalise.Huppel.Json-import           HNormalise.Lmod.Internal   (LmodLoad)+import           HNormalise.Lmod.Internal      (LmodLoad) import           HNormalise.Lmod.Json import           HNormalise.Shorewall.Internal (Shorewall) import           HNormalise.Shorewall.Json-import           HNormalise.Snoopy.Internal (Snoopy)+import           HNormalise.Snoopy.Internal    (Snoopy) import           HNormalise.Snoopy.Json-import           HNormalise.Torque.Internal (TorqueJobExit)+import           HNormalise.Torque.Internal    (TorqueJobExit) import           HNormalise.Torque.Json  --------------------------------------------------------------------------------@@ -50,16 +49,17 @@ data Rsyslog = Rsyslog     { msg              :: !Text     --, rawmsg           :: !Text-    , timereported     :: !(Maybe ZonedTime)+    , timereported     :: !ZonedTime     , hostname         :: !Text     , syslogtag        :: !Text  -- Could be a list?     , inputname        :: !Text     , fromhost         :: !Text     , fromhost_ip      :: !Text     , pri              :: !(Maybe Int)+    , version          :: !(Maybe Int)     , syslogfacility   :: !Text     , syslogseverity   :: !Text-    , timegenerated    :: !ZonedTime+    , timegenerated    :: !(Maybe ZonedTime)     , programname      :: !Text     , protocol_version :: !Text     --, structured_data  :: !Text@@ -75,4 +75,5 @@     { rsyslog    :: Rsyslog            -- ^ The original rsyslog message in a parsed form     , normalised :: ParseResult        -- ^ The normalised message     , jsonkey    :: Text               -- ^ The key under which the normalised info will appear in the JSON result+    , fields     :: Maybe [(Text, Text)]     -- ^ The fields we need to output when creating the JSON encoding as (key, fieldname)     } deriving (Show, Generic)

@@ -7,6 +7,7 @@ -------------------------------------------------------------------------------- import           Control.Monad import           Data.Aeson+import qualified Data.HashMap.Lazy     as M import           Data.Monoid  --------------------------------------------------------------------------------@@ -25,6 +26,7 @@             (v .:  "fromhost")         <*>             (v .:  "fromhost-ip")      <*>             (v .:  "pri")              <*>+            (v .:  "version")          <*>               -- no idea with what the version matches in the JSON message format, see also http://www.rsyslog.com/doc/master/configuration/properties.html             (v .:  "syslogfacility")   <*>             (v .:  "syslogseverity")   <*>             (v .:  "timegenerated")    <*>@@ -40,13 +42,21 @@ -------------------------------------------------------------------------------- instance ToJSON Rsyslog where     toEncoding = genericToEncoding defaultOptions+    toJSON = genericToJSON defaultOptions  -------------------------------------------------------------------------------- instance ToJSON NormalisedRsyslog where-    toEncoding (NRsyslog r n k) =-        pairs-            (  "message" .= msg r-            <> "syslog_abspri" .= syslogseverity r-            <> "program" .= app_name r-            <> k .= n-            )+    toEncoding (NRsyslog r n k fs) =+        case fs of+            -- this is the default+            Nothing -> pairs+                        (  "message" .= msg r+                        <> "syslog_abspri" .= pri r+                        <> "syslog_version" .= version r+                        <> "program" .= app_name r+                        <> "@source_host" .= hostname r          -- the host in ES will likely be set to the machine sending the data to logstash+                        <> k .= n+                        )+            Just fs' -> case toJSON r of+                            Object syslog -> pairs $ foldl (\v (key, fieldname) -> v <> key .= (M.lookupDefault Null fieldname syslog)) (k .= n) fs'+                            _ -> pairs (k .= n) -- FIXME: this should never happen

@@ -39,7 +39,8 @@  -------------------------------------------------------------------------------- -- | The 'getJsonKey' function return the key under which the normalised message should appear when JSON is produced-getJsonKey :: ParseResult -> Text+getJsonKey :: ParseResult  -- ^ Wrapped result for which we need to get a key+           -> Text         -- ^ Key for use in the JSON encoding of the result getJsonKey (PR_Huppel _) = "huppel" getJsonKey (PR_Lmod _) = "lmod" getJsonKey (PR_Torque _) = "torque"@@ -50,15 +51,16 @@ -- | The 'parseRsyslogLogstashString' currently is a placeholder function that will convert the incoming rsyslog message -- if it is encoded as expected in a plain string format -- <%PRI%>1 %timegenerated:::date-rfc3339% %HOSTNAME% %syslogtag% - %APP-NAME%: %msg%-parseRsyslogLogstashString :: Parser SBS.ByteString-parseRsyslogLogstashString = do+parseRsyslogLogstashString :: Maybe [(Text, Text)]    -- ^ Output fields+                           -> Parser SBS.ByteString   -- ^ Resulting encoded JSON representation+parseRsyslogLogstashString fs = do     abspri <- maybeOption $ do         char '<'         p <- decimal         char '>'         v <- maybeOption $ decimal         return (p, v)-    timegenerated <- skipSpace *> zonedTime+    timereported <- skipSpace *> zonedTime     hostname <- skipSpace *> takeTill isSpace     syslogtag <- skipSpace *> takeTill isSpace  -- FIXME: this might be incorrect     skipSpace *> char '-' *> skipSpace@@ -67,18 +69,17 @@              in BS.toStrict $ encode $ NRsyslog                     { rsyslog = Rsyslog                         { msg              = original-                        , timereported     = Nothing+                        , timereported     = timereported                         , hostname         = hostname                         , syslogtag        = syslogtag                         , inputname        = T.empty                         , fromhost         = T.empty                         , fromhost_ip      = T.empty-                        , pri              = case abspri of-                                Just (p, _) -> Just p-                                Nothing     -> Nothing+                        , pri              = abspri >>= \(p, _) -> return p+                        , version          = abspri >>= \(_, v) -> v                         , syslogfacility   = T.empty                         , syslogseverity   = T.empty-                        , timegenerated    = timegenerated+                        , timegenerated    = Nothing                         , programname      = T.empty                         , protocol_version = T.empty                         , app_name         = appname@@ -86,4 +87,5 @@                         }                     , normalised = parsed                     , jsonkey = jsonkey+                    , fields = fs                     }

@@ -25,8 +25,7 @@ -------------------------------------------------------------------------------- data TorqueExecHost = TorqueExecHost     { name      :: !Text-    , lowerCore :: !Int-    , upperCore :: !Int+    , cores     :: ![Int]     } deriving (Show, Eq, Generic)  --------------------------------------------------------------------------------@@ -84,6 +83,7 @@     , owner               :: !Text     , session             :: !Integer     , times               :: !TorqueJobTime+    , execHost            :: ![TorqueExecHost]     , resourceRequest     :: !TorqueResourceRequest     , resourceUsage       :: !TorqueResourceUsage     , totalExecutionSlots :: !Int

@@ -23,7 +23,7 @@     toEncoding = genericToEncoding defaultOptions  instance ToJSON TorqueWalltime where-    toEncoding = genericToEncoding defaultOptions+    toEncoding (TorqueWalltime d h m s) = toEncoding $ (((d * 24 + h) * 60) + m) * 60 + s  instance ToJSON TorqueResourceRequest where     toEncoding = genericToEncoding defaultOptions

@@ -20,6 +20,7 @@ import           HNormalise.Torque.Internal  --------------------------------------------------------------------------------+-- | 'parseTorqueWalltime' parses [[[DD:]HH:]MM:]SS strings representing walltime parseTorqueWalltime :: Parser TorqueWalltime parseTorqueWalltime =         parseTorqueDays@@ -51,6 +52,7 @@   --------------------------------------------------------------------------------+-- | 'parseTorqueMemory' parses an decimal followed by a memory unit and return the memory in bytes parseTorqueMemory :: Parser Integer parseTorqueMemory = do     v <- decimal@@ -65,6 +67,7 @@         "gb" -> v * 1024 * 1024 * 1024  --------------------------------------------------------------------------------+-- | 'parseTorqueJobName' splits the job name in its components, i.e., ID, [ array ID,] master and cluster parseTorqueJobName :: Parser TorqueJobName parseTorqueJobName = do     n <- decimal@@ -83,6 +86,7 @@   --------------------------------------------------------------------------------+-- | 'parseTorqueResourceNodeList' parses a list of FQDN nodes and their ppn or a nodecount and its ppn parseTorqueResourceNodeList :: Parser (Either TorqueJobShortNode [TorqueJobFQNode]) parseTorqueResourceNodeList = do     c <- peekChar'@@ -112,6 +116,9 @@ Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.walltime -}+-- | 'parseTorqueResourceRequest' parses all key value pairs denoting resources requested.+-- Most of these are not obligatory. Since the Torque documentation is vague on mentioning which entries occur, the last+-- 1.5 years of data we have were used to make an educated guess as to which keys might appear and in what order parseTorqueResourceRequest :: Parser TorqueResourceRequest parseTorqueResourceRequest = do     permute $ TorqueResourceRequest@@ -131,6 +138,7 @@         <||> skipSpace *> string "Resource_List.walltime=" *> parseTorqueWalltime  --------------------------------------------------------------------------------+-- | 'parseTorqueResourceUsage' parses all the key value pairs denoting used resources. parseTorqueResourceUsage :: Parser TorqueResourceUsage parseTorqueResourceUsage = do     cput <- skipSpace *> kvNumParser "resources_used.cput"@@ -147,24 +155,30 @@         }  --------------------------------------------------------------------------------+-- | 'parseTorqueHostList' parses a '+' separated list of hostname/coreranges+-- A core range can be of the form 1,3,5-7,9 parseTorqueHostList :: Parser [TorqueExecHost] parseTorqueHostList = flip sepBy (char '+') $ do     fqdn <- Data.Attoparsec.Text.takeWhile (/= '/')     char '/'-    (lower, upper) <- try parseCoreRange <|> parseSingleCore-    return $ TorqueExecHost { name = fqdn, lowerCore = lower, upperCore = upper}-  where parseCoreRange :: Parser (Int, Int)-        parseCoreRange = do+    cores <- parseCores+    return $ TorqueExecHost { name = fqdn, cores = cores}+  where parseCores :: Parser [Int]+        parseCores = do+            cores <- flip sepBy1' (char ',') $ try parseRange <|> parseSingle+            return $ concat cores+        parseRange = do             lower <- decimal             char '-'             upper <- decimal-            return (lower, upper)+            return [lower .. upper]+        parseSingle = do+            c <- decimal+            return [c] -        parseSingleCore = do-            lower <- decimal-            return (lower, lower)  --------------------------------------------------------------------------------+-- | 'parseTorqueExit' parses a complete log line denoting a job exit. Tested with Torque 6.1.x. parseTorqueExit :: Parser (Text, TorqueJobExit) parseTorqueExit = do     takeTill (== ';') *> string ";E;"   -- drop the prefix@@ -204,6 +218,7 @@             , startTime = start             , endTime = end             }+        , execHost = exec_host         , resourceRequest = request         , resourceUsage = usage         , totalExecutionSlots = total_execution_slots

@@ -57,7 +57,7 @@     , bgroup "parse snoopy"         [ bench "successfull input" $ whnf (AT.parse SnoopyP.parseSnoopy) snoopyInput]     , bgroup "normaliseText"-        [ bench "snoopy" $ nf H.normaliseText fullSnoopyInput-        , bench "lmod" $ nf H.normaliseText fullLmodInput+        [ bench "snoopy" $ nf (H.normaliseText Nothing) fullSnoopyInput+        , bench "lmod" $ nf (H.normaliseText Nothing) fullLmodInput         ]     ]

@@ -25,13 +25,17 @@     describe "parse full rsyslog message" $ do         it "lmod load" $ do             let s = "<13>1 2016-06-07T17:50:22.658452+02:00 node2159 lmod: - lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016" :: Text-            s ~> parseRsyslogLogstashString `shouldParse`-                "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":\"\",\"program\":\"lmod\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}"+            s ~> (parseRsyslogLogstashString Nothing) `shouldParse`+                "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}"          it "imfile torque input" $ do             let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text-            s ~> parseRsyslogLogstashString `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":\"\",\"program\":\"torque\",\"torque\":{\"name\":{\"number\":0,\"array_id\":null,\"master\":\"master23\",\"cluster\":\"banette\"},\"user\":\"vsc40075\",\"group\":\"vsc40075\",\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"vsc40075@gligar03.gligar.gent.vsc\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"neednodes\":{\"Left\":{\"number\":1,\"ppn\":null}},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"Left\":{\"number\":1,\"ppn\":null}},\"select\":null,\"qos\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"walltime\":{\"days\":0,\"hours\":1,\"minutes\":0,\"seconds\":0}},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":{\"days\":0,\"hours\":0,\"minutes\":44,\"seconds\":4}},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265}}"+            s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"test2802\",\"torque\":{\"name\":{\"number\":0,\"array_id\":null,\"master\":\"master23\",\"cluster\":\"banette\"},\"user\":\"vsc40075\",\"group\":\"vsc40075\",\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"vsc40075@gligar03.gligar.gent.vsc\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"execHost\":[{\"name\":\"exec_host=node2801.banette.gent.vsc\",\"cores\":[0]}],\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"neednodes\":{\"Left\":{\"number\":1,\"ppn\":null}},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"Left\":{\"number\":1,\"ppn\":null}},\"select\":null,\"qos\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"walltime\":3600},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":2644},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265}}"          it "snoopy with process ID" $ do             let s = "<86>1 2017-05-29T16:40:48.275334+02:00 master23 snoopy[28949]: - snoopy[28949]::  [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text-            s ~> parseRsyslogLogstashString `shouldParse` "{\"message\":\"snoopy[28949]::  [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":\"\",\"program\":\"snoopy\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}"+            s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"snoopy[28949]::  [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":86,\"syslog_version\":1,\"program\":\"snoopy\",\"@source_host\":\"master23\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}"++        it "snoopy with process ID; only the @source_host" $ do+            let s = "<86>1 2017-05-29T16:40:48.275334+02:00 master23 snoopy[28949]: - snoopy[28949]::  [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text+            s ~> (parseRsyslogLogstashString $ Just [("@source_host", "hostname")]) `shouldParse` "{\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"},\"@source_host\":\"master23\"}"

@@ -200,6 +200,38 @@                 , walltime      = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 }                 } +    describe "parseTorqueHostList" $ do+        it "parse comma separated list of single cores" $ do+            let s = "node1001.mycluster.mydomain/1,3,5,7" :: Text+            s ~> parseTorqueHostList `shouldParse` [ TorqueExecHost+                { name = "node1001.mycluster.mydomain"+                , cores = [1,3,5,7]+                }]+        it "parse comma separated list of core rangess" $ do+            let s = "node1001.mycluster.mydomain/1-3,5-7" :: Text+            s ~> parseTorqueHostList `shouldParse` [ TorqueExecHost+                { name = "node1001.mycluster.mydomain"+                , cores = [1,2,3,5,6,7]+                }]+        it "parse comma separated list of single cores and core ranges" $ do+            let s = "node1001.mycluster.mydomain/1,3,5-7,9,12-14" :: Text+            s ~> parseTorqueHostList `shouldParse` [ TorqueExecHost+                { name = "node1001.mycluster.mydomain"+                , cores = [1,3,5,6,7,9,12,13,14]+                }]+        it "parse multiple nodes with one comma separated list of single cores and one core range" $ do+            let s = "node1001.mycluster.mydomain/1,3,5,7+node1002.mycluster.mydomain/4-6" :: Text+            s ~> parseTorqueHostList `shouldParse` [+                TorqueExecHost+                    { name = "node1001.mycluster.mydomain"+                    , cores = [1,3,5,7]+                    },+                TorqueExecHost+                    { name = "node1002.mycluster.mydomain"+                    , cores = [4,5,6]+                    }+                ]+     describe "parseTorqueExit" $ do         it "parse job exit log line" $ do             let s = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text@@ -219,6 +251,16 @@                     , startTime = 1491390307                     , endTime = 1491390413                     }+                , execHost =+                    [ TorqueExecHost+                        { name = "exec_host=node2801.banette.gent.vsc"+                        , cores = [0,1]+                        }+                    , TorqueExecHost+                        { name = "node2803.banette.gent.vsc"+                        , cores = [0,1]+                        }+                    ]                 , resourceRequest = TorqueResourceRequest                     { mem           = Nothing                     , advres        = Nothing