diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ hNormalise ========== -[![Build Status](https://secure.travis-ci.org/itkovian/hnormalise.svg?branch=master)](http://travis-ci.org/itkovian/hnormalise) +[![Build Status](https://secure.travis-ci.org/hpcugent/hnormalise.svg?branch=master)](http://travis-ci.org/hpcugent/hnormalise) `hNormalise` is a small tool and accompanying library for the conversion of regular [rsyslog](http://www.rsyslog.com) to structured log messages, i.e., turning the `msg` payload into (a nested) JSON object. diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -1,12 +1,11 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-} -{-# LANGUAGE RecordWildCards #-} module Main where -------------------------------------------------------------------------------- import Control.Applicative ((<$>), (<*>)) -import Control.Concurrent.MVar (modifyMVar_, newMVar, newEmptyMVar, putMVar, readMVar, tryTakeMVar, withMVar, MVar) +import Control.Concurrent.MVar (modifyMVar_, newMVar, newEmptyMVar, putMVar, readMVar, tryTakeMVar, withMVar, takeMVar, MVar) import Control.Monad import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Loops (whileM_, untilM_) @@ -23,16 +22,17 @@ import Data.Conduit.Network import qualified Data.Conduit.Text as DCT import qualified Data.Conduit.ZMQ4 as ZMQC -import Data.Maybe (fromJust) +import Data.Maybe (fromJust, fromMaybe) import Data.Monoid ((<>)) import qualified Data.Text as T import qualified Data.Text.Encoding as TE +import Data.Time +import Data.Time.Format import Data.Version (showVersion) import qualified Options.Applicative as OA import qualified Paths_hnormalise import System.Exit (exitFailure, exitSuccess) import System.Posix.Signals (installHandler, Handler(CatchOnce), sigINT, sigTERM) -import qualified System.ZMQ4 as ZMQ (Receiver, Sender, Socket, Size, context, term, setIoThreads, Context) import qualified System.ZMQ4 as ZMQ (withContext, withSocket, bind, send, receive, connect, Push(..), Pull(..)) import Text.Printf (printf) @@ -42,15 +42,18 @@ import HNormalise.Config ( Config (..) , ConnectionType(..) , InputConfig(..) + , LoggingConfig(..) , OutputConfig(..) , TcpOutputConfig(..) , TcpPortConfig(..) , ZeroMQOutputConfig(..) , ZeroMQPortConfig(..) , connectionType + , defaultLoggingFrequency , loadConfig) import HNormalise.Internal (Rsyslog (..)) import HNormalise.Json +import HNormalise.Verbose -------------------------------------------------------------------------------- data Options = Options @@ -58,30 +61,35 @@ , oJsonInput :: !Bool , oVersion :: !Bool , oTestFilePath :: !(Maybe FilePath) + , oVerbose :: !Bool } deriving (Show) -------------------------------------------------------------------------------- parserOptions :: OA.Parser Options parserOptions = Options - <$> (OA.optional $ OA.strOption $ + <$> OA.optional ( OA.strOption $ OA.long "configfile" <> OA.short 'c' <> OA.metavar "FILENAME" <> OA.help "configuration file location ") - <*> (OA.switch $ + <*> OA.switch ( OA.long "jsoninput" <> OA.short 'j' <> OA.help "Input will be delivered as JSON (slower)") - <*> (OA.switch $ + <*> OA.switch ( OA.long "version" <> OA.short 'v' <> OA.help "Display version and exit" <> OA.hidden) - <*> (OA.optional $ OA.strOption $ + <*> OA.optional ( OA.strOption $ OA.long "test" <> OA.short 't' <> OA.metavar "OUTPUT FILENAME" <> OA.help "run in test modus, sinking output to the given file") + <*> (OA.switch $ + OA.long "verbose" <> + OA.help "Verbose output" <> + OA.hidden) -------------------------------------------------------------------------------- @@ -103,7 +111,7 @@ -------------------------------------------------------------------------------- -- | 'messageSink' yields the parsed JSON downstream, or if parsing fails, yields the original message downstream -messageSink success failure = loop +messageSink success failure messageCount frequency = loop where loop = do v <- await @@ -111,13 +119,21 @@ Just (Transformed json) -> do yield json $$ success yield (SBS.pack "\n") $$ success + increaseCount (1, 0) loop Just (Original l) -> do yield l $$ failure yield (SBS.pack "\n") $$ failure + increaseCount (0, 1) loop Nothing -> return () - + increaseCount (s, f) = + liftIO $ do + (s', f') <- takeMVar messageCount + when ((s' + f') `mod` frequency == 0) $ do + epoch_int <- (read . formatTime defaultTimeLocale "%s" <$> getCurrentTime) :: IO Int + printf "%ld - message count: %10d (success: %10d, fail: %10d)\n" epoch_int (s' + f') s' f' + putMVar messageCount (s' + s, f' + f) -------------------------------------------------------------------------------- -- | 'mySink' yields the results downstream with the addition of a string mentioning success or failure @@ -140,9 +156,10 @@ Nothing -> return () -------------------------------------------------------------------------------- -runTCPConnection options config = do +runTCPConnection options config messageCount = do let listenHost = fromJust $ input config >>= (\(InputConfig t _) -> t) >>= (\(TcpPortConfig h p) -> h) let listenPort = fromJust $ input config >>= (\(InputConfig t _) -> t) >>= (\(TcpPortConfig h p) -> p) + let frequency = fromMaybe defaultLoggingFrequency (logging config >>= (\(LoggingConfig f) -> f)) runTCPServer (serverSettings listenPort "*") $ \appData -> do let fs = fields config case oTestFilePath options of @@ -154,20 +171,13 @@ runTCPClient (clientSettings successPort successHost) $ \successServer -> runTCPClient (clientSettings failurePort failureHost) $ \failServer -> - let normalisationConduit = case oJsonInput options of - True -> CB.lines $= C.map (normaliseJsonInput fs) - False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs) - in appSource appData - $= normalisationConduit - $$ messageSink (appSink successServer) (appSink failServer) - Just testSinkFileName -> - let normalisationConduit = case oJsonInput options of - True -> CB.lines $= C.map (normaliseJsonInput fs) - False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs) - in runResourceT $ appSource appData - $= normalisationConduit - $= mySink - $$ sinkFile testSinkFileName + appSource appData + $= normalisationConduit options config + $$ messageSink (appSink successServer) (appSink failServer) messageCount frequency + Just testSinkFileName -> runResourceT $ appSource appData + $= normalisationConduit options config + $= mySink + $$ sinkFile testSinkFileName -- | 'zmqInterruptibleSource' converts a regular 0mq recieve operation on a socket into a conduit source -- The source is halted when something is put into the MVar, effectively stopping the program from checking @@ -183,17 +193,25 @@ liftIO $ putStrLn "Done!" -------------------------------------------------------------------------------- -runZeroMQConnection options config s_interrupted = do +normalisationConduit options config = let fs = fields config + in if oJsonInput options + then CB.lines $= C.map (normaliseJsonInput fs) + else DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs) + + +-------------------------------------------------------------------------------- +runZeroMQConnection options config s_interrupted messageCount verbose' = do + let fs = fields config let listenHost = fromJust $ input config >>= (\(InputConfig _ z) -> z) >>= (\(ZeroMQPortConfig m h p) -> h) let listenPort = fromJust $ input config >>= (\(InputConfig _ z) -> z) >>= (\(ZeroMQPortConfig m h p) -> p) - liftIO $ putStrLn (printf "listening on tcp://%s:%d" listenHost listenPort) + let frequency = fromMaybe defaultLoggingFrequency (logging config >>= (\(LoggingConfig f) -> f)) case oTestFilePath options of - Nothing -> do - ZMQ.withContext $ \ctx -> do + Nothing -> ZMQ.withContext $ \ctx -> ZMQ.withSocket ctx ZMQ.Pull $ \s -> do ZMQ.bind s $ printf "tcp://%s:%d" listenHost listenPort + verbose' $ printf "Listening on tcp://%s:%d" listenHost listenPort let successHost = fromJust $ output config >>= (\(OutputConfig _ z) -> z) >>= (\(ZeroMQOutputConfig s f) -> s) >>= (\(ZeroMQPortConfig m h p) -> h) let successPort = fromJust $ output config >>= (\(OutputConfig _ z) -> z) >>= (\(ZeroMQOutputConfig s f) -> s) >>= (\(ZeroMQPortConfig m h p) -> p) @@ -203,31 +221,24 @@ ZMQ.withSocket ctx ZMQ.Push $ \successSocket -> ZMQ.withSocket ctx ZMQ.Push $ \failureSocket -> do ZMQ.connect successSocket $ printf "tcp://%s:%d" successHost successPort - liftIO $ putStrLn (printf "connected to tcp://%s:%d" successHost successPort) - ZMQ.connect failureSocket $ printf "tcp://%s:%d" failureHost failurePort - liftIO $ putStrLn (printf "connected to tcp://%s:%d" failureHost failurePort) + verbose' $ printf "Pushing successful parses on tcp://%s:%d" successHost successPort - let normalisationConduit = case oJsonInput options of - True -> CB.lines $= C.map (normaliseJsonInput fs) - False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs) + ZMQ.connect failureSocket $ printf "tcp://%s:%d" failureHost failurePort + verbose' $ printf "Pushing failed parses on tcp://%s:%d" failureHost failurePort liftIO $ zmqInterruptibleSource s_interrupted s - $= normalisationConduit - $$ messageSink (ZMQC.zmqSink successSocket []) (ZMQC.zmqSink failureSocket []) - - {-Just testSinkFileName -> - let normalisationConduit = case oJsonInput options of - True -> CB.lines $= C.map (normaliseJsonInput fs) - False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs) + $= normalisationConduit options config + $$ messageSink (ZMQC.zmqSink successSocket []) (ZMQC.zmqSink failureSocket []) messageCount frequency - in ZMQ.withContext $ \ctx -> do - ZMQ.withSocket ctx ZMQ.Pull $ \s -> do - ZMQ.bind s $ printf "tcp://%s:%d" listenHost listenPort - liftIO $ zmqInterruptibleSource s_interrupted s - $= normalisationConduit - $= mySink - $$ sinkFile testSinkFileName +{- -- FIXME: This does not work, gives a type error + Just testSinkFileName -> + ZMQ.withContext $ \ctx -> + ZMQ.withSocket ctx ZMQ.Pull $ \s -> do + ZMQ.bind s $ printf "tcp://%s:%d" listenHost listenPort + liftIO $ zmqInterruptibleSource s_interrupted s + $= normalisationConduit options config + $= mySink + $$ sinkFile testSinkFileName -} - -------------------------------------------------------------------------------- -- | 'main' starts a TCP server, listening to incoming data and connecting to TCP servers downstream to -- for the pipeline. @@ -239,17 +250,23 @@ putStrLn $ showVersion Paths_hnormalise.version exitSuccess + let verbose' = makeVerbose (oVerbose options) config <- loadConfig (oConfigFilePath options) + verbose' $ "Config: " ++ show config + -- For now, we only support input and output configurations of the same type, -- i.e., both TCP, both ZeroMQ, etc. + messageCount <- newMVar ((0,0) :: (Int, Int)) case connectionType config of - TCP -> void $ runTCPConnection options config + TCP -> void $ runTCPConnection options config messageCount ZeroMQ -> do - trace "zeromq" $ return () + verbose' "ZeroMQ connections" s_interrupted <- newEmptyMVar installHandler sigINT (CatchOnce $ handler s_interrupted) Nothing + verbose' "SIGINT handler installed" installHandler sigTERM (CatchOnce $ handler s_interrupted) Nothing - runZeroMQConnection options config s_interrupted + verbose' "SIGTERM handler installed" + runZeroMQConnection options config s_interrupted messageCount verbose' exitSuccess diff --git a/hnormalise.cabal b/hnormalise.cabal --- a/hnormalise.cabal +++ b/hnormalise.cabal @@ -1,5 +1,5 @@ name: hnormalise -version: 0.4.2.0 +version: 0.4.7.0 synopsis: Log message normalisation tool producing structured JSON messages description: Log message normalisation tool producing structured JSON messages homepage: https://github.com/itkovian/hnormalise#readme @@ -7,7 +7,7 @@ license-file: LICENSE author: Andy Georges maintainer: itkovian@gmail.com -copyright: 2017 Andy Georges +copyright: 2017 Ghent University category: Command line build-type: Simple extra-source-files: README.md @@ -38,6 +38,7 @@ , HNormalise.Torque.Internal , HNormalise.Torque.Json , HNormalise.Torque.Parser + , HNormalise.Verbose build-depends: base >= 4.7 && < 5 , aeson , aeson-pretty diff --git a/src/HNormalise.hs b/src/HNormalise.hs --- a/src/HNormalise.hs +++ b/src/HNormalise.hs @@ -32,10 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} - -{-# LANGUAGE OverloadedStrings #-} - - module HNormalise ( normaliseRsyslog , normaliseJsonInput @@ -82,7 +78,7 @@ -> SBS.ByteString -- ^ Input representing an rsyslog message in JSON format -> Normalised -- ^ Transformed or Original result normaliseJsonInput fs logLine = - case (Aeson.decodeStrict logLine :: Maybe Rsyslog) >>= (normaliseRsyslog fs) of + case (Aeson.decodeStrict logLine :: Maybe Rsyslog) >>= normaliseRsyslog fs of Just j -> Transformed j Nothing -> Original logLine @@ -100,7 +96,7 @@ _ -> original _ -> original where - original = Original $ encodeUtf8 $ logLine + original = Original $ encodeUtf8 logLine -------------------------------------------------------------------------------- -- | The 'convertMessage' function transforms the actual message to a 'Maybe' 'ParseResult'. If parsing fails, @@ -124,5 +120,4 @@ normaliseRsyslog fs rsyslog = do cm <- convertMessage $ msg rsyslog return $ BS.toStrict - $ Aeson.encode - $ NRsyslog { rsyslog = rsyslog, normalised = cm, jsonkey = getJsonKey cm, fields = fs } + $ Aeson.encode NRsyslog { rsyslog = rsyslog, normalised = cm, jsonkey = getJsonKey cm, fields = fs } diff --git a/src/HNormalise/Common/Internal.hs b/src/HNormalise/Common/Internal.hs --- a/src/HNormalise/Common/Internal.hs +++ b/src/HNormalise/Common/Internal.hs @@ -32,10 +32,8 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE ExistentialQuantification #-} -{-# LANGUAGE OverloadedStrings #-} module HNormalise.Common.Internal ( Host (..) diff --git a/src/HNormalise/Common/Parser.hs b/src/HNormalise/Common/Parser.hs --- a/src/HNormalise/Common/Parser.hs +++ b/src/HNormalise/Common/Parser.hs @@ -32,8 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-} -{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE OverloadedStrings #-} @@ -91,6 +89,11 @@ kvNumParser :: Integral a => Text -> Parser a kvNumParser key = keyParser key *> decimal {-# INLINE kvNumParser #-} + +-------------------------------------------------------------------------------- +kvSignedParser :: Integral a => Text -> Parser a +kvSignedParser key = keyParser key *> signed decimal +{-# INLINE kvSignedParser #-} -------------------------------------------------------------------------------- kvYesNoParser :: Text -> Parser Bool diff --git a/src/HNormalise/Config.hs b/src/HNormalise/Config.hs --- a/src/HNormalise/Config.hs +++ b/src/HNormalise/Config.hs @@ -40,12 +40,14 @@ ( Config(..) , ConnectionType(..) , InputConfig(..) + , LoggingConfig(..) , OutputConfig(..) , TcpOutputConfig(..) , TcpPortConfig(..) , ZeroMQOutputConfig(..) , ZeroMQPortConfig(..) , connectionType + , defaultLoggingFrequency , loadConfig ) where @@ -60,6 +62,7 @@ import System.Directory import Debug.Trace + -------------------------------------------------------------------------------- data ConnectionType = TCP | ZeroMQ @@ -72,9 +75,16 @@ _ -> ZeroMQ -------------------------------------------------------------------------------- +defaultLoggingFrequency = 100000 + +data LoggingConfig = LoggingConfig + { frequency :: !(Maybe Int) -- ^ How often should we write an update (# of messages processed) + } deriving (Show) + +-------------------------------------------------------------------------------- data TcpPortConfig = TcpPortConfig - { host :: !(Maybe Text) - , port :: !(Maybe Int) + { host :: !(Maybe Text) + , port :: !(Maybe Int) } deriving (Show) -------------------------------------------------------------------------------- @@ -117,8 +127,8 @@ -------------------------------------------------------------------------------- data ZeroMQOutputConfig = ZeroMQOutputConfig - { success :: !(Maybe ZeroMQPortConfig) - , failure :: !(Maybe ZeroMQPortConfig) + { success :: !(Maybe ZeroMQPortConfig) + , failure :: !(Maybe ZeroMQPortConfig) } deriving (Show) -------------------------------------------------------------------------------- @@ -131,8 +141,8 @@ -------------------------------------------------------------------------------- data InputConfig = InputConfig - { tcp :: !(Maybe TcpPortConfig) - , zeromq :: !(Maybe ZeroMQPortConfig) + { tcp :: !(Maybe TcpPortConfig) + , zeromq :: !(Maybe ZeroMQPortConfig) } deriving (Show) -------------------------------------------------------------------------------- @@ -145,8 +155,8 @@ -------------------------------------------------------------------------------- data OutputConfig = OutputConfig - { tcp :: !(Maybe TcpOutputConfig) - , zeromq :: !(Maybe ZeroMQOutputConfig) + { tcp :: !(Maybe TcpOutputConfig) + , zeromq :: !(Maybe ZeroMQOutputConfig) } deriving (Show) -------------------------------------------------------------------------------- @@ -165,8 +175,8 @@ } defaultOutputTcpConfig = TcpOutputConfig - { success = Just $ TcpPortConfig { host = Just "localhost", port = Just 26001 } - , failure = Just $ TcpPortConfig { host = Just "localhost", port = Just 26002 } + { success = Just TcpPortConfig { host = Just "localhost", port = Just 26001 } + , failure = Just TcpPortConfig { host = Just "localhost", port = Just 26002 } } defaultInputConfig = InputConfig @@ -181,22 +191,25 @@ -------------------------------------------------------------------------------- data Config = Config - { input :: !(Maybe InputConfig) - , output :: !(Maybe OutputConfig) - , fields :: !(Maybe [(Text, Text)]) + { logging :: !(Maybe LoggingConfig) + , input :: !(Maybe InputConfig) + , output :: !(Maybe OutputConfig) + , fields :: !(Maybe [(Text, Text)]) } deriving (Show) -------------------------------------------------------------------------------- instance Monoid Config where - mempty = Config Nothing Nothing Nothing + mempty = Config Nothing Nothing Nothing Nothing mappend l r = Config - { input = input l `mplus` input r + { logging = logging l `mplus` logging r + , input = input l `mplus` input r , output = output l `mplus` output r , fields = fields l `mplus` fields r } defaultConfig = Config - { input = Nothing + { logging = Just LoggingConfig { frequency = Just defaultLoggingFrequency } + , input = Nothing , output = Nothing , fields = Nothing } @@ -229,6 +242,7 @@ return $ userConfig <> systemConfig <> defaultConfig -------------------------------------------------------------------------------- +$(deriveJSON defaultOptions ''LoggingConfig) $(deriveJSON defaultOptions ''TcpPortConfig) $(deriveJSON defaultOptions ''TcpOutputConfig) $(deriveJSON defaultOptions ''ZeroMQPortConfig) diff --git a/src/HNormalise/Internal.hs b/src/HNormalise/Internal.hs --- a/src/HNormalise/Internal.hs +++ b/src/HNormalise/Internal.hs @@ -32,9 +32,9 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveGeneric #-} -{-# LANGUAGE DuplicateRecordFields #-} -{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DuplicateRecordFields #-} +{-# LANGUAGE OverloadedStrings #-} module HNormalise.Internal where @@ -54,7 +54,7 @@ import HNormalise.Shorewall.Json import HNormalise.Snoopy.Internal (Snoopy) import HNormalise.Snoopy.Json -import HNormalise.Torque.Internal (TorqueJobExit) +import HNormalise.Torque.Internal (TorqueParseResult) import HNormalise.Torque.Json -------------------------------------------------------------------------------- @@ -67,17 +67,17 @@ | PR_Shorewall Shorewall -- | Represents a parsed 'Snoopy' message | PR_Snoopy Snoopy - -- | Represents a parsed 'TorqueJobExit' message - | PR_Torque TorqueJobExit + -- | Represents a parsed 'Torque' message + | PR_Torque TorqueParseResult deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- instance ToJSON ParseResult where - toEncoding (PR_Huppel v) = toEncoding v - toEncoding (PR_Lmod v) = toEncoding v + toEncoding (PR_Huppel v) = toEncoding v + toEncoding (PR_Lmod v) = toEncoding v toEncoding (PR_Shorewall v) = toEncoding v - toEncoding (PR_Snoopy v) = toEncoding v - toEncoding (PR_Torque v) = toEncoding v + toEncoding (PR_Snoopy v) = toEncoding v + toEncoding (PR_Torque v) = toEncoding v -------------------------------------------------------------------------------- data Rsyslog = Rsyslog diff --git a/src/HNormalise/Json.hs b/src/HNormalise/Json.hs --- a/src/HNormalise/Json.hs +++ b/src/HNormalise/Json.hs @@ -41,7 +41,7 @@ -------------------------------------------------------------------------------- import Control.Monad import Data.Aeson -import qualified Data.HashMap.Lazy as M +import qualified Data.HashMap.Lazy as M import Data.Monoid -------------------------------------------------------------------------------- diff --git a/src/HNormalise/Parser.hs b/src/HNormalise/Parser.hs --- a/src/HNormalise/Parser.hs +++ b/src/HNormalise/Parser.hs @@ -32,7 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE ExistentialQuantification #-} {-# LANGUAGE OverloadedStrings #-} @@ -40,16 +39,16 @@ module HNormalise.Parser where -------------------------------------------------------------------------------- -import Control.Applicative ((<|>)) -import Data.Aeson (encode) +import Control.Applicative ((<|>)) +import Data.Aeson (encode) import Data.Attoparsec.Text import Data.Attoparsec.Time -import qualified Data.ByteString.Char8 as SBS -import qualified Data.ByteString.Lazy.Char8 as BS +import qualified Data.ByteString.Char8 as SBS +import qualified Data.ByteString.Lazy.Char8 as BS import Data.Char -import Data.Text (Text, empty) -import qualified Data.Text as T -import qualified Data.Text.Encoding as TE +import Data.Text (Text, empty) +import qualified Data.Text as T +import qualified Data.Text.Encoding as TE -------------------------------------------------------------------------------- import HNormalise.Common.Parser import HNormalise.Huppel.Parser @@ -66,20 +65,26 @@ -- | The 'parseMessage' function will try and use each configured parser to normalise the input it's given parseMessage :: Parser (Text, ParseResult) parseMessage = - (parseLmodLoad >>= (\(a, v) -> return $ (a, PR_Lmod v))) - <|> (parseShorewall >>= (\(a, v) -> return $ (a, PR_Shorewall v))) - <|> (parseSnoopy >>= (\(a, v) -> return $ (a, PR_Snoopy v))) - <|> (parseTorqueExit >>= (\(a, v) -> return $ (a, PR_Torque v))) + let pm parser target = (parser >>= (\(a, v) -> return (a, target v))) + in pm parseLmodLoad PR_Lmod + <|> pm parseShorewall PR_Shorewall + <|> pm parseSnoopy PR_Snoopy + <|> pm parseTorqueQueue PR_Torque + <|> pm parseTorqueStart PR_Torque + <|> pm parseTorqueDelete PR_Torque + <|> pm parseTorqueExit PR_Torque + <|> pm parseTorqueAbort PR_Torque + <|> pm parseTorqueRerun PR_Torque -------------------------------------------------------------------------------- -- | The 'getJsonKey' function return the key under which the normalised message should appear when JSON is produced getJsonKey :: ParseResult -- ^ Wrapped result for which we need to get a key -> Text -- ^ Key for use in the JSON encoding of the result -getJsonKey (PR_Huppel _) = "huppel" -getJsonKey (PR_Lmod _) = "lmod" -getJsonKey (PR_Torque _) = "torque" +getJsonKey (PR_Huppel _) = "huppel" +getJsonKey (PR_Lmod _) = "lmod" +getJsonKey (PR_Torque _) = "torque" getJsonKey (PR_Shorewall _) = "shorewall" -getJsonKey (PR_Snoopy _) = "snoopy" +getJsonKey (PR_Snoopy _) = "snoopy" -------------------------------------------------------------------------------- -- | The 'parseRsyslogLogstashString' currently is a placeholder function that will convert the incoming rsyslog message @@ -92,7 +97,7 @@ char '<' p <- decimal char '>' - v <- maybeOption $ decimal + v <- maybeOption decimal return (p, v) timereported <- skipSpace *> zonedTime hostname <- skipSpace *> takeTill isSpace @@ -100,7 +105,7 @@ skipSpace *> char '-' *> skipSpace (original, (appname, parsed)) <- match parseMessage return $ let jsonkey = getJsonKey parsed - in BS.toStrict $ encode $ NRsyslog + in BS.toStrict $ encode NRsyslog { rsyslog = Rsyslog { msg = original , timereported = timereported @@ -110,7 +115,7 @@ , fromhost = T.empty , fromhost_ip = T.empty , pri = abspri >>= \(p, _) -> return p - , version = abspri >>= \(_, v) -> v + , version = abspri >>= snd , syslogfacility = T.empty , syslogseverity = T.empty , timegenerated = Nothing diff --git a/src/HNormalise/Torque/Internal.hs b/src/HNormalise/Torque/Internal.hs --- a/src/HNormalise/Torque/Internal.hs +++ b/src/HNormalise/Torque/Internal.hs @@ -32,11 +32,8 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE BangPatterns #-} -{-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} -{-# LANGUAGE OverloadedStrings #-} module HNormalise.Torque.Internal where @@ -45,6 +42,28 @@ import GHC.Generics (Generic) -------------------------------------------------------------------------------- +-- | `TorqueParseResult` encapsulates the results we get from parsing torque lines +data TorqueParseResult + = TorqueQueue TorqueJobQueue + | TorqueStart TorqueJobStart + | TorqueDelete TorqueJobDelete + | TorqueExit TorqueJobExit + | TorqueAbort TorqueJobAbort + | TorqueRerun TorqueJobRerun + deriving (Show, Eq, Generic) + +-------------------------------------------------------------------------------- +-- | `TorqueEntryType` distinguishes between accounting data entry Types +data TorqueEntryType + = TorqueQueueEntry + | TorqueStartEntry + | TorqueDeleteEntry + | TorqueExitEntry + | TorqueAbortEntry + | TorqueRerunEntry + deriving (Show, Eq, Generic) + +-------------------------------------------------------------------------------- data TorqueJobShortNode = TorqueJobShortNode { number :: !Int , ppn :: !(Maybe Int) @@ -53,7 +72,7 @@ -------------------------------------------------------------------------------- data TorqueJobFQNode = TorqueJobFQNode { name :: !Text - , ppn :: !Int + , ppn :: !(Maybe Int) } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- @@ -63,10 +82,13 @@ -------------------------------------------------------------------------------- data TorqueExecHost = TorqueExecHost - { name :: !Text - , cores :: ![Int] + { name :: !Text + , cores :: ![Int] } deriving (Show, Eq, Generic) +instance Ord TorqueExecHost where + compare (TorqueExecHost t1 _) (TorqueExecHost t2 _) = compare t1 t2 + -------------------------------------------------------------------------------- data TorqueWalltime = TorqueWalltime { days :: !Int @@ -81,22 +103,32 @@ , advres :: !(Maybe Text) , naccesspolicy :: !(Maybe Text) , ncpus :: !(Maybe Int) - , neednodes :: !TorqueJobNode + , cputime :: !(Maybe TorqueWalltime) + , prologue :: !(Maybe Text) + , epilogue :: !(Maybe Text) + , neednodes :: !(Maybe TorqueJobNode) , nice :: !(Maybe Int) , nodeCount :: !Int , nodes :: !TorqueJobNode , select :: !(Maybe Text) , qos :: !(Maybe Text) + , other :: !(Maybe Text) + , feature :: !(Maybe Text) + , host :: !(Maybe Text) + , procs :: !(Maybe Text) + , nodeset :: !(Maybe Text) + , tpn :: !(Maybe Text) , pmem :: !(Maybe Integer) , vmem :: !(Maybe Integer) , pvmem :: !(Maybe Integer) + , mppmem :: !(Maybe Integer) , walltime :: !TorqueWalltime } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueResourceUsage = TorqueResourceUsage { cputime :: !Integer - , energy :: !Integer + , energy :: !(Maybe Integer) , mem :: !Integer , vmem :: !Integer , walltime :: !TorqueWalltime @@ -113,9 +145,11 @@ -------------------------------------------------------------------------------- data TorqueJobExit = TorqueJobExit - { name :: !TorqueJobName + { torqueDatestamp :: !Text + , name :: !TorqueJobName , user :: !Text , group :: !Text + , account :: !(Maybe Text) , jobname :: !Text , queue :: !Text , startCount :: !(Maybe Int) @@ -128,43 +162,65 @@ , totalExecutionSlots :: !Int , uniqueNodeCount :: !Int , exitStatus :: !Int + , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueJobName = TorqueJobName - { number :: !Integer - , array_id :: !(Maybe Integer) - , master :: !Text - , cluster :: !Text + { number :: !Integer + , arrayId :: !(Maybe Integer) + , master :: !Text + , cluster :: !Text } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueJobQueue = TorqueJobQueue - { name :: !TorqueJobName - , queue :: !Text + { torqueDatestamp :: !Text + , name :: !TorqueJobName + , queue :: !Text + , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueJobStart = TorqueJobStart - { name :: !TorqueJobName - , user :: !Text - , group :: !Text - , jobname :: !Text - , queue :: !Text - , owner :: !Text - , times :: !TorqueJobTime - , execHost :: ![TorqueExecHost] - , resourceRequest :: !TorqueResourceRequest + { torqueDatestamp :: !Text + , name :: !TorqueJobName + , user :: !Text + , group :: !Text + , account :: !(Maybe Text) + , jobname :: !Text + , queue :: !Text + , owner :: !Text + , times :: !TorqueJobTime + , execHost :: ![TorqueExecHost] + , resourceRequest :: !TorqueResourceRequest + , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueRequestor = TorqueRequestor - { user :: !Text - , whence :: !Text + { user :: !Text + , whence :: !Text } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueJobDelete = TorqueJobDelete - { name :: !TorqueJobName - , requestor :: !TorqueRequestor + { torqueDatestamp :: !Text + , name :: !TorqueJobName + , requestor :: !TorqueRequestor + , torqueEntryType :: TorqueEntryType + } deriving (Show, Eq, Generic) + +-------------------------------------------------------------------------------- +data TorqueJobAbort = TorqueJobAbort + { torqueDatestamp :: !Text + , name :: !TorqueJobName + , torqueEntryType :: TorqueEntryType + } deriving (Show, Eq, Generic) + +-------------------------------------------------------------------------------- +data TorqueJobRerun = TorqueJobRerun + { torqueDatestamp :: !Text + , name :: !TorqueJobName + , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic) diff --git a/src/HNormalise/Torque/Json.hs b/src/HNormalise/Torque/Json.hs --- a/src/HNormalise/Torque/Json.hs +++ b/src/HNormalise/Torque/Json.hs @@ -32,21 +32,18 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-} -{-# LANGUAGE DeriveGeneric #-} -{-# LANGUAGE OverloadedStrings #-} - - module HNormalise.Torque.Json where -------------------------------------------------------------------------------- import Data.Aeson -import Data.Monoid ((<>)) +import Data.Monoid ((<>)) -------------------------------------------------------------------------------- import HNormalise.Torque.Internal -------------------------------------------------------------------------------- +instance ToJSON TorqueEntryType where + toEncoding = genericToEncoding defaultOptions instance ToJSON TorqueJobShortNode where toEncoding = genericToEncoding defaultOptions @@ -61,7 +58,7 @@ toEncoding (TorqueWalltime d h m s) = toEncoding $ (((d * 24 + h) * 60) + m) * 60 + s instance ToJSON TorqueJobNode where - toEncoding (TSN n) = toEncoding n + toEncoding (TSN n) = toEncoding n toEncoding (TFN ns) = toEncoding ns instance ToJSON TorqueResourceRequest where @@ -78,3 +75,29 @@ instance ToJSON TorqueJobName where toEncoding = genericToEncoding defaultOptions + +instance ToJSON TorqueRequestor where + toEncoding = genericToEncoding defaultOptions + +instance ToJSON TorqueJobStart where + toEncoding = genericToEncoding defaultOptions + +instance ToJSON TorqueJobQueue where + toEncoding = genericToEncoding defaultOptions + +instance ToJSON TorqueJobDelete where + toEncoding = genericToEncoding defaultOptions + +instance ToJSON TorqueJobAbort where + toEncoding = genericToEncoding defaultOptions + +instance ToJSON TorqueJobRerun where + toEncoding = genericToEncoding defaultOptions + +instance ToJSON TorqueParseResult where + toEncoding (TorqueQueue ts) = toEncoding ts + toEncoding (TorqueStart ts) = toEncoding ts + toEncoding (TorqueDelete ts) = toEncoding ts + toEncoding (TorqueExit ts) = toEncoding ts + toEncoding (TorqueAbort ts) = toEncoding ts + toEncoding (TorqueRerun ts) = toEncoding ts diff --git a/src/HNormalise/Torque/Parser.hs b/src/HNormalise/Torque/Parser.hs --- a/src/HNormalise/Torque/Parser.hs +++ b/src/HNormalise/Torque/Parser.hs @@ -32,8 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-} -{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE OverloadedStrings #-} @@ -41,13 +39,17 @@ -------------------------------------------------------------------------------- import Control.Applicative ((<|>)) +import Control.Monad (join) import Data.Attoparsec.Combinator (lookAhead, manyTill) import Data.Attoparsec.Text import Data.Char (isDigit, isSpace) +import Data.List (concatMap, groupBy, sort) import qualified Data.Map as M +import Data.Maybe (fromMaybe) import Data.Text (Text) import qualified Data.Text as T -import Text.ParserCombinators.Perm ((<$$>), (<||>), (<$?>), (<|?>), permute) +import Text.ParserCombinators.Perm (permute, (<$$>), (<$?>), (<|?>), + (<||>)) -------------------------------------------------------------------------------- import HNormalise.Common.Parser @@ -109,29 +111,34 @@ m <- char '.' *> takeTill (== '.') c <- char '.' *> takeTill (== '.') manyTill anyChar (lookAhead ";") *> char ';' - return $ TorqueJobName { number = n, array_id = a, master = m, cluster = c} + return TorqueJobName { number = n, arrayId = a, master = m, cluster = c} where parseArrayId :: Parser (Maybe Integer) - parseArrayId = try $ maybeOption $ do - char '[' - i <- decimal - char ']' - return i + parseArrayId = try parseArrayIdBracket <|> parseArrayIdDash + where parseArrayIdBracket = do + char '[' + i <- maybeOption decimal + char ']' + return i + parseArrayIdDash = maybeOption $ do + char '-' + decimal -------------------------------------------------------------------------------- -- | 'parseTorqueResourceNodeList' parses a list of FQDN nodes and their ppn or a nodecount and its ppn +-- FIXME: Add support for resource lists of the form Resource_List.neednodes=3:ppn=8+1:ppn=1 parseTorqueResourceNodeList :: Parser TorqueJobNode parseTorqueResourceNodeList = do c <- peekChar' if Data.Char.isDigit c then do number <- decimal ppn <- maybeOption $ char ':' *> string "ppn=" *> decimal - return $ TSN $ TorqueJobShortNode { number = number, ppn = ppn } - else TFN <$> (flip sepBy (char '+') $ do - fqdn <- Data.Attoparsec.Text.takeWhile (/= ':') - ppn <- char ':' *> kvNumParser "ppn" - return TorqueJobFQNode { name = fqdn, ppn = ppn}) + return $ TSN TorqueJobShortNode { number = number, ppn = ppn } + else TFN <$> sepBy (do + fqdn <- Data.Attoparsec.Text.takeWhile (\c -> c /= ':' && c /= ' ') + ppn <- maybeOption $ char ':' *> kvNumParser "ppn" + return TorqueJobFQNode { name = fqdn, ppn = ppn}) (char '+') -------------------------------------------------------------------------------- {- Examples found in the 2016 logs @@ -149,38 +156,61 @@ Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.qos Resource_List.vmem Resource_List.walltime Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.walltime + +-- 2014 logs +Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime +Resource_List.cput Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime -} -- | 'parseTorqueResourceRequest' parses all key value pairs denoting resources requested. --- Most of these are not obligatory. Since the Torque documentation is vague on mentioning which entries occur, the last --- 1.5 years of data we have were used to make an educated guess as to which keys might appear and in what order +-- Most of these are not obligatory. Since the Torque documentation is vague on mentioning which entries occur, +-- the guesses as to the most common ordering and the mandatory fields is based on 5 years of log data from Torque +-- 4.x to 6.0 parseTorqueResourceRequest :: Parser TorqueResourceRequest -parseTorqueResourceRequest = do +parseTorqueResourceRequest = permute $ TorqueResourceRequest <$?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.mem=" *> parseTorqueMemory)) <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.advres")) <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.naccesspolicy")) <|?> (Nothing, Just `fmap` (skipSpace *> kvNumParser "Resource_List.ncpus")) - <||> skipSpace *> string "Resource_List.neednodes=" *> parseTorqueResourceNodeList + <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.cput=" *> parseTorqueWalltime)) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.prologue")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.epilogue")) + <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.neednodes=" *> parseTorqueResourceNodeList)) <|?> (Nothing, Just `fmap` (skipSpace *> kvNumParser "Resource_List.nice")) <||> skipSpace *> kvNumParser "Resource_List.nodect" <||> skipSpace *> string "Resource_List.nodes=" *> parseTorqueResourceNodeList <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.select")) <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.qos")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.other")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.feature")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.host")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.procs")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.nodeset")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.tpn")) <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.pmem=" *> parseTorqueMemory)) <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.vmem=" *> parseTorqueMemory)) <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.pvmem=" *> parseTorqueMemory)) + <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.mppmem=" *> parseTorqueMemory)) <||> skipSpace *> string "Resource_List.walltime=" *> parseTorqueWalltime -------------------------------------------------------------------------------- +-- | 'parseTorqueCpuTime' parses the cpu time spent +-- This value is either given in seconds or in a torque timestamp format +-- We always convert the value to seconds if needed +parseTorqueCpuTime :: Parser Integer +parseTorqueCpuTime = + try (parseTorqueWalltime >>= \(TorqueWalltime d h m s) -> return $ fromIntegral (((d*24+h)*60+m)*60+s)) <|> decimal + +-------------------------------------------------------------------------------- -- | 'parseTorqueResourceUsage' parses all the key value pairs denoting used resources. parseTorqueResourceUsage :: Parser TorqueResourceUsage parseTorqueResourceUsage = do - cput <- skipSpace *> kvNumParser "resources_used.cput" - energy <- skipSpace *> kvNumParser "resources_used.energy_used" + cput <- skipSpace *> string "resources_used.cput=" *> parseTorqueCpuTime + energy <- skipSpace *> maybeOption (kvNumParser "resources_used.energy_used") mem <- skipSpace *> string "resources_used.mem=" *> parseTorqueMemory vmem <- skipSpace *> string "resources_used.vmem=" *> parseTorqueMemory walltime <- skipSpace *> string "resources_used.walltime=" *> parseTorqueWalltime - return $ TorqueResourceUsage + return TorqueResourceUsage { cputime = cput , energy = energy , mem = mem @@ -189,16 +219,30 @@ } -------------------------------------------------------------------------------- +-- | `aggregateHosts` take a list of TorqueExecHost and condenses them to a minimal form +-- There will be one entry for each different host, each time with the cores combined +aggregateHosts :: [TorqueExecHost] -> [TorqueExecHost] +aggregateHosts ths = + let ths' = groupBy (\(TorqueExecHost n1 _) (TorqueExecHost n2 _) -> n1 == n2) $ sort ths + in map aggCores ths' + where aggCores :: [TorqueExecHost] -> TorqueExecHost + aggCores ths@(TorqueExecHost n _:_) = TorqueExecHost + { name = n + , cores = sort . concatMap cores $ ths + } + +-------------------------------------------------------------------------------- -- | 'parseTorqueHostList' parses a '+' separated list of hostname/coreranges -- A core range can be of the form 1,3,5-7,9 parseTorqueHostList :: Parser [TorqueExecHost] parseTorqueHostList = do string "exec_host=" - flip sepBy (char '+') $ do + hosts <- flip sepBy (char '+') $ do fqdn <- Data.Attoparsec.Text.takeWhile (/= '/') char '/' cores <- parseCores - return $ TorqueExecHost { name = fqdn, cores = cores} + return TorqueExecHost { name = fqdn, cores = cores} + return $ aggregateHosts hosts where parseCores :: Parser [Int] parseCores = do cores <- flip sepBy1' (char ',') $ try parseRange <|> parseSingle @@ -225,36 +269,74 @@ { user = user , whence = whence } +-------------------------------------------------------------------------------- +-- | `parseTorqueAccountingDatestamp` parses the datestamp and the given log line tag +parseTorqueAccountingDatestamp :: Text -> Parser Text +parseTorqueAccountingDatestamp tag = do + string "torque: " + torqueDatestamp <- takeTill (== ';') + string tag -- drop the prefix + return torqueDatestamp -------------------------------------------------------------------------------- --- | 'parseTorqueExit' parses a complete log line denoting a job exit. Tested with Torque 6.1.x. -parseTorqueExit :: Parser (Text, TorqueJobExit) -parseTorqueExit = do - takeTill (== ';') *> string ";E;" -- drop the prefix +-- | 'parseCommonAccountingInfo' parser the initial part that is common between start and exit lines +parseCommonAccountingInfo :: Parser + (TorqueJobName + , Text + , Text + , Maybe Text + , Text + , Text + , Integer + , Integer + , Integer) +parseCommonAccountingInfo = do name <- parseTorqueJobName user <- kvTextParser "user" group <- skipSpace *> kvTextParser "group" + account <- skipSpace *> maybeOption (kvTextParser "account") jobname <- skipSpace *> kvTextParser "jobname" queue <- skipSpace *> kvTextParser "queue" ctime <- skipSpace *> kvNumParser "ctime" qtime <- skipSpace *> kvNumParser "qtime" etime <- skipSpace *> kvNumParser "etime" - start_count <- maybeOption $ skipSpace *> kvNumParser "start_count" + return (name, user, group, account, jobname, queue, ctime, qtime, etime) + +-------------------------------------------------------------------------------- +-- | 'parseCommonStartInfo' parses the start information that is common between start and exit lines +parseCommonStartInfo :: Parser + ( Integer + , Text + , [TorqueExecHost] + , TorqueResourceRequest) +parseCommonStartInfo = do start <- skipSpace *> kvNumParser "start" owner <- skipSpace *> kvTextParser "owner" exec_host <- skipSpace *> parseTorqueHostList request <- parseTorqueResourceRequest + return (start, owner, exec_host, request) + +-------------------------------------------------------------------------------- +-- | 'parseTorqueExit' parses a complete log line denoting a job exit. Tested with Torque 6.1.x. +parseTorqueExit :: Parser (Text, TorqueParseResult) +parseTorqueExit = do + torqueDatestamp <- parseTorqueAccountingDatestamp ";E;" + (name, user, group, account, jobname, queue, ctime, qtime, etime) <- parseCommonAccountingInfo + start_count <- maybeOption $ skipSpace *> kvNumParser "start_count" + (start, owner, exec_host, request) <- parseCommonStartInfo session <- skipSpace *> kvNumParser "session" - total_execution_slots <- skipSpace *> kvNumParser "total_execution_slots" - unique_node_count <- skipSpace *> kvNumParser "unique_node_count" + total_execution_slots <- skipSpace *> maybeOption (kvNumParser "total_execution_slots") + unique_node_count <- skipSpace *> maybeOption (kvNumParser "unique_node_count") end <- skipSpace *> kvNumParser "end" - exit_status <- skipSpace *> kvNumParser "Exit_status" + exit_status <- skipSpace *> kvSignedParser "Exit_status" usage <- skipSpace *> parseTorqueResourceUsage - return $ ("torque", TorqueJobExit - { name = name + return ("torque", TorqueExit TorqueJobExit + { torqueDatestamp = torqueDatestamp + , name = name , user = user , group = group + , account = account , jobname = jobname , queue = queue , startCount = start_count @@ -270,59 +352,84 @@ , execHost = exec_host , resourceRequest = request , resourceUsage = usage - , totalExecutionSlots = total_execution_slots - , uniqueNodeCount = unique_node_count + , totalExecutionSlots = fromMaybe (compute_total_execution_slots exec_host) total_execution_slots + , uniqueNodeCount = fromMaybe (length exec_host) unique_node_count , exitStatus = exit_status + , torqueEntryType = TorqueExitEntry }) + where compute_total_execution_slots = sum . map (\(TorqueExecHost _ cs) -> length cs) -------------------------------------------------------------------------------- -- | `parseTorqueDelete` parses a complete log line denoting a deleted job. Tested with Torue 6.1.x -parseTorqueDelete :: Parser (Text, TorqueJobDelete) +parseTorqueDelete :: Parser (Text, TorqueParseResult) parseTorqueDelete = do - takeTill (== ';') *> string ";D;" -- drop the prefix + torqueDatestamp <- parseTorqueAccountingDatestamp ";D;" name <- parseTorqueJobName requestor <- parseTorqueRequestor - return ("torque", TorqueJobDelete - { name = name + return ("torque", TorqueDelete TorqueJobDelete + { torqueDatestamp = torqueDatestamp + , name = name , requestor = requestor + , torqueEntryType = TorqueDeleteEntry }) -------------------------------------------------------------------------------- +-- | `parseTorqueAbort` parses a complete log line denoting an aborted job. Tested with Torque 4.x +parseTorqueAbort :: Parser (Text, TorqueParseResult) +parseTorqueAbort = do + torqueDatestamp <- parseTorqueAccountingDatestamp ";A;" + name <- parseTorqueJobName + + return ("torque", TorqueAbort TorqueJobAbort + { torqueDatestamp = torqueDatestamp + , name = name + , torqueEntryType = TorqueAbortEntry + }) + +-------------------------------------------------------------------------------- +-- | `parseTorqueRerun` parses a complete log line denoting a rerun job. Tested with Torque 4.x +parseTorqueRerun :: Parser (Text, TorqueParseResult) +parseTorqueRerun = do + torqueDatestamp <- parseTorqueAccountingDatestamp ";R;" + name <- parseTorqueJobName + + return ("torque", TorqueRerun TorqueJobRerun + { torqueDatestamp = torqueDatestamp + , name = name + , torqueEntryType = TorqueRerunEntry + }) + + +-------------------------------------------------------------------------------- -- | `parseTorqueQueue` parses a complete log line denoting a queued job. Tested with Torue 6.1.x -parseTorqueQueue :: Parser (Text, TorqueJobQueue) +parseTorqueQueue :: Parser (Text, TorqueParseResult) parseTorqueQueue = do - takeTill (== ';') *> string ";Q;" -- drop the prefix + torqueDatestamp <- parseTorqueAccountingDatestamp ";Q;" name <- parseTorqueJobName queue <- kvTextParser "queue" - return ("torque", TorqueJobQueue - { name = name + return ("torque", TorqueQueue TorqueJobQueue + { torqueDatestamp = torqueDatestamp + , name = name , queue = queue + , torqueEntryType = TorqueQueueEntry }) -------------------------------------------------------------------------------- -- | `parseTorqueStart` parses a complete log line denoting a started job. Tested with Torque 6.1.x -parseTorqueStart :: Parser (Text, TorqueJobStart) +parseTorqueStart :: Parser (Text, TorqueParseResult) parseTorqueStart = do - takeTill (== ';') *> string ";S;" -- drop the prefix - name <- parseTorqueJobName - user <- kvTextParser "user" - group <- skipSpace *> kvTextParser "group" - jobname <- skipSpace *> kvTextParser "jobname" - queue <- skipSpace *> kvTextParser "queue" - ctime <- skipSpace *> kvNumParser "ctime" - qtime <- skipSpace *> kvNumParser "qtime" - etime <- skipSpace *> kvNumParser "etime" - start <- skipSpace *> kvNumParser "start" - owner <- skipSpace *> kvTextParser "owner" - exec_host <- skipSpace *> parseTorqueHostList - request <- parseTorqueResourceRequest + torqueDatestamp <- parseTorqueAccountingDatestamp ";S;" + (name, user, group, account, jobname, queue, ctime, qtime, etime) <- parseCommonAccountingInfo + (start, owner, exec_host, request) <- parseCommonStartInfo - return $ ("torque", TorqueJobStart - { name = name + return ("torque", TorqueStart TorqueJobStart + { torqueDatestamp = torqueDatestamp + , name = name , user = user , group = group + , account = account , jobname = jobname , queue = queue , owner = owner @@ -335,4 +442,5 @@ } , execHost = exec_host , resourceRequest = request + , torqueEntryType = TorqueStartEntry }) diff --git a/src/HNormalise/Verbose.hs b/src/HNormalise/Verbose.hs new file mode 100644 --- /dev/null +++ b/src/HNormalise/Verbose.hs @@ -0,0 +1,54 @@ +{- hnormalise - a log normalisation library + - + - Copyright Andy Georges (c) 2017 + - + - All rights reserved. + - + - Redistribution and use in source and binary forms, with or without + - modification, are permitted provided that the following conditions are met: + - + - * Redistributions of source code must retain the above copyright + - notice, this list of conditions and the following disclaimer. + - + - * Redistributions in binary form must reproduce the above + - copyright notice, this list of conditions and the following + - disclaimer in the documentation and/or other materials provided + - with the distribution. + - + - * Neither the name of Author name here nor the names of other + - contributors may be used to endorse or promote products derived + - from this software without specific prior written permission. + - + - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +-} + +{- This code was taken from https://github.com/jaspervdj/stylish-haskell/blob/master/lib/Language/Haskell/Stylish/Verbose.hs -} +module HNormalise.Verbose + ( Verbose + , makeVerbose + ) where + + +-------------------------------------------------------------------------------- +import System.IO (hPutStrLn, stderr) + + +-------------------------------------------------------------------------------- +type Verbose = String -> IO () + + +-------------------------------------------------------------------------------- +makeVerbose :: Bool -> Verbose +makeVerbose verbose + | verbose = hPutStrLn stderr + | otherwise = const $ return () diff --git a/test/Bench.hs b/test/Bench.hs --- a/test/Bench.hs +++ b/test/Bench.hs @@ -57,14 +57,14 @@ rnf (H.Original s) = rnf s -------------------------------------------------------------------------------- -torqueJobExitInput1 = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text -torqueJobExitInput2 = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text +torqueJobExitInput1 = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text +torqueJobExitInput2 = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text -torqueJobExitFailInput1 = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short HUPPEL" +torqueJobExitFailInput1 = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short HUPPEL" -torqueJobQueueInput = "06/28/2017 14:31:09;Q;80.master23.banette.gent.vsc;queue=default" -torqueJobDeleteInput = "06/28/2017 15:44:02;D;81.master23.banette.gent.vsc;requestor=vsc40075@gligar02.gligar.gent.vsc" -torqueJobStartInput = "06/20/2017 11:24:49;S;63.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1497950675 qtime=1497950675 etime=1497950675 start=1497950689 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.vmem=4224531456b Resource_List.nodes=1:ppn=1 Resource_List.walltime=00:10:00 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1 Resource_List.nice=0" +torqueJobQueueInput = "torque: 06/28/2017 14:31:09;Q;80.master23.banette.gent.vsc;queue=default" +torqueJobDeleteInput = "torque: 06/28/2017 15:44:02;D;81.master23.banette.gent.vsc;requestor=vsc40075@gligar02.gligar.gent.vsc" +torqueJobStartInput = "torque: 06/20/2017 11:24:49;S;63.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1497950675 qtime=1497950675 etime=1497950675 start=1497950689 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.vmem=4224531456b Resource_List.nodes=1:ppn=1 Resource_List.walltime=00:10:00 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1 Resource_List.nice=0" lmodLoadInput1 = "lmod:: username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text fullLmodInput = "<13>1 2016-06-07T17:50:22.495571+02:00 node2159 lmod: - lmod:: username=vsc40307, cluster=delcatty, jobid=434.master16.delcatty.gent.vsc, userload=no, module=binutils/2.25-GCCcore-4.9.3, fn=/apps/gent/SL6/sandybridge/modules/all/binutils/2.25-GCCcore-4.9.3" :: Text diff --git a/test/HNormalise/ParserSpec.hs b/test/HNormalise/ParserSpec.hs --- a/test/HNormalise/ParserSpec.hs +++ b/test/HNormalise/ParserSpec.hs @@ -38,16 +38,16 @@ module HNormalise.ParserSpec (main, spec) where -------------------------------------------------------------------------------- -import Data.Text (Text) -import qualified Data.Text.Read as TR +import Data.Text (Text) +import qualified Data.Text.Read as TR +import qualified Net.IPv4 as NT import Test.Hspec import Test.Hspec.Attoparsec -import qualified Net.IPv4 as NT -------------------------------------------------------------------------------- import HNormalise.Common.Internal -import HNormalise.Lmod.Internal import HNormalise.Internal +import HNormalise.Lmod.Internal import HNormalise.Parser -------------------------------------------------------------------------------- main :: IO () @@ -55,7 +55,7 @@ -------------------------------------------------------------------------------- spec :: Spec -spec = do +spec = describe "parse full rsyslog message" $ do it "lmod load" $ do let s = "<13>1 2016-06-07T17:50:22.658452+02:00 node2159 lmod: - lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016" :: Text @@ -63,13 +63,17 @@ "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}" it "imfile torque input" $ do - let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text - s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"test2802\",\"torque\":{\"name\":{\"number\":0,\"array_id\":null,\"master\":\"master23\",\"cluster\":\"banette\"},\"user\":\"vsc40075\",\"group\":\"vsc40075\",\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"vsc40075@gligar03.gligar.gent.vsc\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"execHost\":[{\"name\":\"node2801.banette.gent.vsc\",\"cores\":[0]}],\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"neednodes\":{\"number\":1,\"ppn\":null},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"number\":1,\"ppn\":null},\"select\":null,\"qos\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"walltime\":3600},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":2644},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265}}" + let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text + s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"test2802\",\"torque\":{\"torqueDatestamp\":\"01/25/2017 15:04:10\",\"name\":{\"number\":0,\"arrayId\":null,\"master\":\"mymaster\",\"cluster\":\"somepokemon\"},\"user\":\"huppelde\",\"group\":\"huppelde\",\"account\":null,\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"huppelde@mymachine.mydomain.com\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"execHost\":[{\"name\":\"node2801.somepokemon.mydomain\",\"cores\":[0]}],\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"cputime\":null,\"prologue\":null,\"epilogue\":null,\"neednodes\":{\"number\":1,\"ppn\":null},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"number\":1,\"ppn\":null},\"select\":null,\"qos\":null,\"other\":null,\"feature\":null,\"host\":null,\"procs\":null,\"nodeset\":null,\"tpn\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"mppmem\":null,\"walltime\":3600},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":2644},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265,\"torqueEntryType\":\"TorqueExitEntry\"}}" + it "imfile torque abort job input" $ do + let s = "<133>1 2017-09-14T21:13:09.181048+02:00 master15 torque - torque: 10/17/2013 02:46:14;A;86264.master15.delcatty.gent.vsc;\n" :: Text + s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 10/17/2013 02:46:14;A;86264.master15.delcatty.gent.vsc;\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"master15\",\"torque\":{\"torqueDatestamp\":\"10/17/2013 02:46:14\",\"name\":{\"number\":86264,\"arrayId\":null,\"master\":\"master15\",\"cluster\":\"delcatty\"},\"torqueEntryType\":\"TorqueAbortEntry\"}}" + it "snoopy with process ID" $ do - let s = "<86>1 2017-05-29T16:40:48.275334+02:00 master23 snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text - s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":86,\"syslog_version\":1,\"program\":\"snoopy\",\"@source_host\":\"master23\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}" + let s = "<86>1 2017-05-29T16:40:48.275334+02:00 mymaster snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text + s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":86,\"syslog_version\":1,\"program\":\"snoopy\",\"@source_host\":\"mymaster\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}" it "snoopy with process ID; only the @source_host" $ do - let s = "<86>1 2017-05-29T16:40:48.275334+02:00 master23 snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text - s ~> (parseRsyslogLogstashString $ Just [("@source_host", "hostname")]) `shouldParse` "{\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"},\"@source_host\":\"master23\"}" + let s = "<86>1 2017-05-29T16:40:48.275334+02:00 mymaster snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text + s ~> (parseRsyslogLogstashString $ Just [("@source_host", "hostname")]) `shouldParse` "{\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"},\"@source_host\":\"mymaster\"}" diff --git a/test/HNormalise/Torque/ParserSpec.hs b/test/HNormalise/Torque/ParserSpec.hs --- a/test/HNormalise/Torque/ParserSpec.hs +++ b/test/HNormalise/Torque/ParserSpec.hs @@ -38,14 +38,14 @@ module HNormalise.Torque.ParserSpec (main, spec) where -------------------------------------------------------------------------------- -import Data.Text (Text) -import qualified Data.Text.Read as TR +import Data.Text (Text) +import qualified Data.Text.Read as TR import Test.Hspec import Test.Hspec.Attoparsec -------------------------------------------------------------------------------- -import HNormalise.Torque.Parser import HNormalise.Torque.Internal +import HNormalise.Torque.Parser -------------------------------------------------------------------------------- main :: IO () @@ -56,20 +56,20 @@ spec = do describe "parseTorqueWalltime" $ do it "parse walltime given as SS" $ do - let s = ("1234567" :: Text) + let s = "1234567" :: Text Right (s', _) = TR.decimal s s ~> parseTorqueWalltime `shouldParse` TorqueWalltime { days = 0, hours = 0, minutes = 0, seconds = s' } it "parse walltime given as MM:SS" $ do - let s = ("12:13") :: Text + let s = "12:13" :: Text s ~> parseTorqueWalltime `shouldParse` TorqueWalltime { days = 0, hours = 0, minutes = 12, seconds = 13} it "parse walltime given as HH:MM:SS" $ do - let s = ("11:12:13") :: Text + let s = "11:12:13" :: Text s ~> parseTorqueWalltime `shouldParse` TorqueWalltime { days = 0, hours = 11, minutes = 12, seconds = 13 } it "parse walltime given as MM:SS" $ do - let s = ("10:11:12:13") :: Text + let s = "10:11:12:13" :: Text s ~> parseTorqueWalltime `shouldParse` TorqueWalltime { days = 10, hours = 11, minutes = 12, seconds = 13 } describe "parseTorqueMemory" $ do @@ -132,11 +132,11 @@ describe "parseTorqueJobName" $ do it "parse regular torque job name" $ do let s = "123456789.master.mycluster.mydomain;" :: Text - s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456789, array_id = Nothing, master = "master", cluster = "mycluster" } + s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456789, arrayId = Nothing, master = "master", cluster = "mycluster" } it "parse array torque job name" $ do let s = "123456[789].master.mycluster.mydomain;" :: Text - s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456, array_id = Just 789, master = "master", cluster = "mycluster" } + s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456, arrayId = Just 789, master = "master", cluster = "mycluster" } describe "parseTorqueResourceRequest" $ do it "parse mandatory fields in expected order" $ do @@ -146,15 +146,25 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing - , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1 - , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing , pmem = Nothing , vmem = Nothing , pvmem = Nothing + , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } @@ -165,15 +175,25 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing - , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1 - , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing , pmem = Nothing , vmem = Nothing , pvmem = Nothing + , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } @@ -184,15 +204,25 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing - , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1 - , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing , pmem = Just $ 200 * 1024 , vmem = Just $ 1 * 1024 * 1024 , pvmem = Just $ 400 * 1024 + , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } @@ -203,15 +233,25 @@ , advres = Just "myreservation.1" , naccesspolicy = Nothing , ncpus = Nothing - , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1 - , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing , pmem = Nothing , vmem = Nothing , pvmem = Nothing + , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } @@ -222,18 +262,113 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing - , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1 - , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 } + , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Just "someqos" + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing , pmem = Nothing , vmem = Nothing , pvmem = Nothing + , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } + it "parse 2014 resource List" $ do + let s = "Resource_List.neednodes=1:ppn=16 Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1:ppn=16 Resource_List.vmem=74737mb Resource_List.walltime=05:00:00" :: Text + s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 16 } + , nice = Just 0 + , nodeCount = 1 + , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 16 } + , select = Nothing + , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing + , pmem = Nothing + , vmem = Just $ 74737 * 1024 * 1024 + , pvmem = Nothing + , mppmem = Nothing + , walltime = TorqueWalltime { days = 0, hours = 5, minutes = 0, seconds = 0 } + } + it "parse resource list with FQDN node and no ppn specified" $ do + let s = "Resource_List.neednodes=somenode.somecluster.somedomain Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.walltime=01:00:00" :: Text + s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TFN [TorqueJobFQNode {name = "somenode.somecluster.somedomain", ppn = Nothing}] + , nice = Just 0 + , nodeCount = 1 + , nodes = TSN (TorqueJobShortNode {number = 1, ppn = Nothing}) + , select = Nothing + , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing + , pmem = Nothing + , vmem = Nothing + , pvmem = Nothing + , mppmem = Nothing + , walltime = TorqueWalltime {days = 0, hours = 1, minutes = 0, seconds = 0} + } + it "parse resource list without neednodes specified" $ do + let s = "Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 Resource_List.walltime=03:00:00" :: Text + s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Nothing + , nice = Just 0 + , nodeCount = 1 + , nodes = TSN (TorqueJobShortNode {number = 1, ppn = Just 8}) + , select = Nothing + , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing + , pmem = Nothing + , vmem = Nothing + , pvmem = Nothing + , mppmem = Nothing + , walltime = TorqueWalltime {days = 0, hours = 3, minutes = 0, seconds = 0} + } + describe "parseTorqueHostList" $ do it "parse comma separated list of single cores" $ do let s = "exec_host=node1001.mycluster.mydomain/1,3,5,7" :: Text @@ -267,16 +402,18 @@ ] describe "parseTorqueExit" $ do - it "parse job exit log line" $ do - let s = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text - s ~> parseTorqueExit `shouldParse` ("torque", TorqueJobExit - { name = TorqueJobName { number = 45, array_id = Nothing, master = "master23", cluster = "banette" } + it "parse torque 6.0 job exit log line" $ do + let s = "torque: 04/05/2017 13:06:53;E;45.mymaster.somecluster.somedomain;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@submitnode01.submitnode.somedomain exec_host=node2801.somecluster.somedomain/0-1+node2803.somecluster.somedomain/0-1 Resource_List.nodes=node2801.somecluster.somedomain:ppn=2+node2803.somecluster.somedomain:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.somecluster.somedomain:ppn=2+node2803.somecluster.somedomain:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text + s ~> parseTorqueExit `shouldParse` ("torque", TorqueExit TorqueJobExit + { torqueDatestamp = "04/05/2017 13:06:53" + , name = TorqueJobName { number = 45, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } , user = "vsc40075" , group = "vsc40075" + , account = Nothing , jobname = "STDIN" , queue = "short" , startCount = Nothing - , owner = "vsc40075@gligar01.gligar.gent.vsc" + , owner = "vsc40075@submitnode01.submitnode.somedomain" , session = 15273 , times = TorqueJobTime { ctime = 1491390300 @@ -287,11 +424,11 @@ } , execHost = [ TorqueExecHost - { name = "node2801.banette.gent.vsc" + { name = "node2801.somecluster.somedomain" , cores = [0,1] } , TorqueExecHost - { name = "node2803.banette.gent.vsc" + { name = "node2803.somecluster.somedomain" , cores = [0,1] } ] @@ -300,38 +437,48 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing - , neednodes = TFN + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TFN [ TorqueJobFQNode - { name = "node2801.banette.gent.vsc" - , ppn = 2 + { name = "node2801.somecluster.somedomain" + , ppn = Just 2 } , TorqueJobFQNode - { name = "node2803.banette.gent.vsc" - , ppn = 2 + { name = "node2803.somecluster.somedomain" + , ppn = Just 2 } ] , nice = Just 0 , nodeCount = 2 , nodes = TFN [ TorqueJobFQNode - { name = "node2801.banette.gent.vsc" - , ppn = 2 + { name = "node2801.somecluster.somedomain" + , ppn = Just 2 } , TorqueJobFQNode - { name = "node2803.banette.gent.vsc" - , ppn = 2 + { name = "node2803.somecluster.somedomain" + , ppn = Just 2 } ] , select = Nothing , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing , vmem = Just $ 1 * 1024 * 1024 * 1024 , pmem = Nothing , pvmem = Nothing + , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0} } , resourceUsage = TorqueResourceUsage { cputime = 0 - , energy = 0 + , energy = Just 0 , mem = 55048 * 1024 , vmem = 92488 * 1024 , walltime = TorqueWalltime { days = 0, hours = 0, minutes = 1, seconds = 44 } @@ -339,34 +486,265 @@ , totalExecutionSlots = 4 , uniqueNodeCount = 2 , exitStatus = 0 + , torqueEntryType = TorqueExitEntry }) + it "parse 2014 torque job exit line" $ do + let s = "torque: 01/12/2014 23:57:07;E;161299[389].mymaster.somecluster.somedomain;user=vsc40909 group=vsc40909 jobname=30by40XconChoicesResults-389 queue=short ctime=1389546423 qtime=1389546423 etime=1389546423 start=1389567229 owner=vsc40909@submitnode02.submitnode.somedomain exec_host=node2135.somecluster.somedomain/0+node2135.somecluster.somedomain/1+node2135.somecluster.somedomain/2+node2135.somecluster.somedomain/3+node2135.somecluster.somedomain/4+node2135.somecluster.somedomain/5+node2135.somecluster.somedomain/6+node2135.somecluster.somedomain/7+node2135.somecluster.somedomain/8+node2135.somecluster.somedomain/9+node2135.somecluster.somedomain/10+node2135.somecluster.somedomain/11+node2135.somecluster.somedomain/12+node2135.somecluster.somedomain/13+node2135.somecluster.somedomain/14+node2135.somecluster.somedomain/15 Resource_List.neednodes=1:ppn=16 Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1:ppn=16 Resource_List.vmem=74737mb Resource_List.walltime=05:00:00 session=32698 end=1389567427 Exit_status=0 resources_used.cput=00:48:40 resources_used.mem=307504kb resources_used.vmem=1985904kb resources_used.walltime=00:03:21" :: Text + s ~> parseTorqueExit `shouldParse` ("torque", TorqueExit TorqueJobExit + { torqueDatestamp = "01/12/2014 23:57:07" + , name = TorqueJobName { number = 161299, arrayId = Just 389, master = "mymaster", cluster = "somecluster" } + , user = "vsc40909" + , group = "vsc40909" + , account = Nothing + , jobname = "30by40XconChoicesResults-389" + , queue = "short" + , startCount = Nothing + , owner = "vsc40909@submitnode02.submitnode.somedomain" + , session = 32698 + , times = TorqueJobTime + { ctime = 1389546423 + , qtime = 1389546423 + , etime = 1389546423 + , startTime = 1389567229 + , endTime = Just 1389567427 + } + , execHost = + [ TorqueExecHost + { name = "node2135.somecluster.somedomain" + , cores = [0 .. 15] + } + ] + , resourceRequest = TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode + { number = 1 + , ppn = Just 16 + } + , nice = Just 0 + , nodeCount = 1 + , nodes = TSN TorqueJobShortNode + { number = 1 + , ppn = Just 16 + } + , select = Nothing + , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing + , pmem = Nothing + , vmem = Just 78367424512 + , pvmem = Nothing + , mppmem = Nothing + , walltime = TorqueWalltime { days = 0, hours = 5, minutes = 0, seconds = 0} + } + , resourceUsage = TorqueResourceUsage + { cputime = 2920 + , energy = Nothing + , mem = 314884096 + , vmem = 2033565696 + , walltime = TorqueWalltime { days = 0, hours = 0, minutes = 3, seconds = 21} + } + , totalExecutionSlots = 16 + , uniqueNodeCount = 1 + , exitStatus = 0 + , torqueEntryType = TorqueExitEntry + }) + it "parse torque job exit line with cput resource request" $ do + let s = "torque: 07/22/2014 11:00:03;E;621344.master15.delcatty.gent.vsc;user=vsc40035 group=vsc40035 jobname=NB03N queue=long ctime=1406019524 qtime=1406019524 etime=1406019524 start=1406019532 owner=vsc40035@gligar03.gligar.gent.vsc exec_host=node2142.delcatty.gent.vsc/0+node2142.delcatty.gent.vsc/1+node2142.delcatty.gent.vsc/2+node2142.delcatty.gent.vsc/3+node2142.delcatty.gent.vsc/4+node2142.delcatty.gent.vsc/5+node2142.delcatty.gent.vsc/6+node2142.delcatty.gent.vsc/7+node2142.delcatty.gent.vsc/8+node2142.delcatty.gent.vsc/9+node2142.delcatty.gent.vsc/10+node2142.delcatty.gent.vsc/11+node2142.delcatty.gent.vsc/12+node2142.delcatty.gent.vsc/13+node2142.delcatty.gent.vsc/14+node2142.delcatty.gent.vsc/15 Resource_List.cput=72:00:00 Resource_List.neednodes=1:ppn=16 Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1:ppn=16 Resource_List.vmem=74737mb Resource_List.walltime=72:00:00 session=117962 end=1406019603 Exit_status=271 resources_used.cput=00:00:25 resources_used.mem=5316kb resources_used.vmem=78756kb resources_used.walltime=00:01:14" :: Text + s ~> parseTorqueExit `shouldParse` ("torque", TorqueExit TorqueJobExit + { torqueDatestamp = "07/22/2014 11:00:03" + , name = TorqueJobName {number = 621344, arrayId = Nothing, master = "master15", cluster = "delcatty"} + , user = "vsc40035" + , group = "vsc40035" + , account = Nothing + , jobname = "NB03N" + , queue = "long" + , startCount = Nothing + , owner = "vsc40035@gligar03.gligar.gent.vsc" + , session = 117962 + , times = TorqueJobTime + { ctime = 1406019524 + , qtime = 1406019524 + , etime = 1406019524 + , startTime = 1406019532 + , endTime = Just 1406019603 + } + , execHost = [TorqueExecHost {name = "node2142.delcatty.gent.vsc", cores = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]}] + , resourceRequest = TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , cputime = Just TorqueWalltime {days = 0, hours = 72, minutes = 0, seconds = 0} + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode {number = 1, ppn = Just 16} + , nice = Just 0 + , nodeCount = 1 + , nodes = TSN TorqueJobShortNode {number = 1, ppn = Just 16} + , select = Nothing + , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing + , pmem = Nothing + , vmem = Just 78367424512 + , pvmem = Nothing + , mppmem = Nothing + , walltime = TorqueWalltime {days = 0, hours = 72, minutes = 0, seconds = 0} + } + , resourceUsage = TorqueResourceUsage + { cputime = 25 + , energy = Nothing + , mem = 5443584 + , vmem = 80646144 + , walltime = TorqueWalltime {days = 0, hours = 0, minutes = 1, seconds = 14} + } + , totalExecutionSlots = 16 + , uniqueNodeCount = 1 + , exitStatus = 271 + , torqueEntryType = TorqueExitEntry + }) + + it "parse torque job exit with account field" $ do + let s = "torque: 08/03/2017 05:07:22;E;268279.master21.swalot.gent.vsc;user=vsc41771 group=vsc41771 account=lt1_2017-43 jobname=/user/scratch/gent/gvo000/gvo00003/vsc41771/amsterdam/restrained_md/test_withoutplumed queue=short ctime=1501686015 qtime=1501686015 etime=1501686015 start=1501686467 owner=vsc41771@gligar01.gligar.gent.vsc exec_host=node2612.swalot.gent.vsc/0-19+node2681.swalot.gent.vsc/0-19 Resource_List.neednodes=2:ppn=20 Resource_List.nice=0 Resource_List.nodect=2 Resource_List.nodes=2:ppn=20 Resource_List.vmem=143425316860b Resource_List.walltime=11:59:00 session=7473 total_execution_slots=40 unique_node_count=2 end=1501729642 Exit_status=-11 resources_used.cput=1725002 resources_used.energy_used=0 resources_used.mem=16209816kb resources_used.vmem=38821964kb resources_used.walltime=11:59:30" :: Text + s ~> parseTorqueExit `shouldParse` ("torque", TorqueExit TorqueJobExit + { torqueDatestamp = "08/03/2017 05:07:22" + , name = TorqueJobName {number = 268279, arrayId = Nothing, master = "master21", cluster = "swalot"} + , user = "vsc41771" + , group = "vsc41771" + , account = Just "lt1_2017-43" + , jobname = "/user/scratch/gent/gvo000/gvo00003/vsc41771/amsterdam/restrained_md/test_withoutplumed" + , queue = "short" + , startCount = Nothing + , owner = "vsc41771@gligar01.gligar.gent.vsc" + , session = 7473 + , times = TorqueJobTime {ctime = 1501686015, qtime = 1501686015, etime = 1501686015, startTime = 1501686467, endTime = Just 1501729642} + , execHost = + [ TorqueExecHost {name = "node2612.swalot.gent.vsc", cores = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]} + , TorqueExecHost {name = "node2681.swalot.gent.vsc", cores = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]} + ] + , resourceRequest = TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN TorqueJobShortNode {number = 2, ppn = Just 20} + , nice = Just 0 + , nodeCount = 2 + , nodes = TSN TorqueJobShortNode {number = 2, ppn = Just 20} + , select = Nothing + , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing + , pmem = Nothing + , vmem = Just 143425316860 + , pvmem = Nothing + , mppmem = Nothing + , walltime = TorqueWalltime {days = 0, hours = 11, minutes = 59, seconds = 0} + } + , resourceUsage = TorqueResourceUsage + { cputime = 1725002 + , energy = Just 0 + , mem = 16598851584 + , vmem = 39753691136 + , walltime = TorqueWalltime {days = 0, hours = 11, minutes = 59, seconds = 30} + } + , totalExecutionSlots = 40 + , uniqueNodeCount = 2 + , exitStatus = -11 + , torqueEntryType = TorqueExitEntry + }) + describe "parseTorqueQueue" $ do it "parse job queue entry" $ do - let s = "06/28/2017 14:31:09;Q;80.master23.banette.gent.vsc;queue=default" :: Text - s ~> parseTorqueQueue `shouldParse` ("torque", TorqueJobQueue - { name = TorqueJobName { number = 80, array_id = Nothing, master = "master23", cluster = "banette" } + let s = "torque: 06/28/2017 14:31:09;Q;80.mymaster.somecluster.somedomain;queue=default" :: Text + s ~> parseTorqueQueue `shouldParse` ("torque", TorqueQueue TorqueJobQueue + { torqueDatestamp = "06/28/2017 14:31:09" + , name = TorqueJobName { number = 80, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } , queue = "default" + , torqueEntryType = TorqueQueueEntry }) - describe "parseTorqueDelete" $ do + it "parse job queue entry master24 - torque 6.0" $ do + let s = "torque: 07/27/2017 14:17:41;Q;5.master24.somecluster.somedomain;queue=default" :: Text + s ~> parseTorqueQueue `shouldParse` ("torque", TorqueQueue TorqueJobQueue + { torqueDatestamp = "07/27/2017 14:17:41" + , name = TorqueJobName { number = 5, arrayId = Nothing, master = "master24" , cluster = "somecluster" } + , queue = "default" + , torqueEntryType = TorqueQueueEntry + }) + + it "parse job queue entry - torque 4.x" $ do + let s = "torque: 12/31/2014 15:51:48;Q;1166970[].somemaster.somecluster.gent.vsc;queue=long" :: Text + s ~> parseTorqueQueue `shouldParse`("torque", TorqueQueue TorqueJobQueue + { torqueDatestamp = "12/31/2014 15:51:48" + , name = TorqueJobName { number = 1166970, arrayId = Nothing, master = "somemaster", cluster = "somecluster"} + , queue = "long" + , torqueEntryType = TorqueQueueEntry + }) + + describe "parseTorqueDelete" $ it "parse job delete entry" $ do - let s = "06/28/2017 15:44:02;D;81.master23.banette.gent.vsc;requestor=vsc40075@gligar02.gligar.gent.vsc" :: Text - s ~> parseTorqueDelete `shouldParse` ("torque", TorqueJobDelete - { name = TorqueJobName { number = 81, array_id = Nothing, master = "master23", cluster = "banette" } - , requestor = TorqueRequestor { user = "vsc40075", whence = "gligar02.gligar.gent.vsc" } + let s = "torque: 06/28/2017 15:44:02;D;81.mymaster.somecluster.somedomain;requestor=vsc40075@submitnode02.submitnode.somedomain" :: Text + s ~> parseTorqueDelete `shouldParse` ("torque", TorqueDelete TorqueJobDelete + { torqueDatestamp = "06/28/2017 15:44:02" + , name = TorqueJobName { number = 81, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } + , requestor = TorqueRequestor { user = "vsc40075", whence = "submitnode02.submitnode.somedomain" } + , torqueEntryType = TorqueDeleteEntry }) + describe "parseTorqueAbort" $ + it "parse job abort entry" $ do + let s = "torque: 09/02/2013 17:34:26;A;34106.mymaster.somecluster.somedomain;" :: Text + s ~> parseTorqueAbort `shouldParse` ("torque", TorqueAbort TorqueJobAbort + { torqueDatestamp = "09/02/2013 17:34:26" + , name = TorqueJobName { number = 34106, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } + , torqueEntryType = TorqueAbortEntry + }) + + describe "parseTorqueRerun" $ + it "parse job rerun entry" $ do + let s = "torque: 09/02/2013 17:34:26;R;34106.mymaster.somecluster.somedomain;" :: Text + s ~> parseTorqueRerun `shouldParse` ("torque", TorqueRerun TorqueJobRerun + { torqueDatestamp = "09/02/2013 17:34:26" + , name = TorqueJobName { number = 34106, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } + , torqueEntryType = TorqueRerunEntry + }) + + describe "parseTorqueStart" $ do it "parse job start" $ do - let s = "06/20/2017 11:24:49;S;63.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1497950675 qtime=1497950675 etime=1497950675 start=1497950689 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.vmem=4224531456b Resource_List.nodes=1:ppn=1 Resource_List.walltime=00:10:00 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1 Resource_List.nice=0" :: Text - s ~> parseTorqueStart `shouldParse` ("torque", TorqueJobStart - { name = TorqueJobName { number = 63, array_id = Nothing, master = "master23", cluster = "banette" } + let s = "torque: 06/20/2017 11:24:49;S;63.mymaster.somecluster.somedomain;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1497950675 qtime=1497950675 etime=1497950675 start=1497950689 owner=vsc40075@submitnode01.submitnode.somedomain exec_host=node2801.somecluster.somedomain/0 Resource_List.vmem=4224531456b Resource_List.nodes=1:ppn=1 Resource_List.walltime=00:10:00 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1 Resource_List.nice=0" :: Text + s ~> parseTorqueStart `shouldParse` ("torque", TorqueStart TorqueJobStart + { torqueDatestamp = "06/20/2017 11:24:49" + , name = TorqueJobName { number = 63, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } , user = "vsc40075" , group = "vsc40075" + , account = Nothing , jobname = "STDIN" , queue = "short" - , owner = "vsc40075@gligar01.gligar.gent.vsc" + , owner = "vsc40075@submitnode01.submitnode.somedomain" , times = TorqueJobTime { ctime = 1497950675 , qtime = 1497950675 @@ -376,7 +754,7 @@ } , execHost = [ TorqueExecHost - { name = "node2801.banette.gent.vsc" + { name = "node2801.somecluster.somedomain" , cores = [0] } ] @@ -385,25 +763,75 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing - , neednodes = TSN - ( TorqueJobShortNode + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TSN + TorqueJobShortNode { number = 1 , ppn = Just 1 } - ) , nice = Just 0 , nodeCount = 1 , nodes = TSN - ( TorqueJobShortNode + TorqueJobShortNode { number = 1 , ppn = Just 1 } - ) , select = Nothing , qos = Nothing - , vmem = Just $ 4224531456 + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing + , vmem = Just 4224531456 , pmem = Nothing , pvmem = Nothing + , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 0, minutes = 10, seconds = 0} } + , torqueEntryType = TorqueStartEntry }) + it "parse job start - torque legacy 2009" $ do + let s = "torque: 02/23/2009 11:48:35;S;102355.master.cvos.cluster;user=vsc40014 group=vsc40014 jobname=MtChr5_9036000_rmwrap.sh queue=short_eth ctime=1235384686 qtime=1235384686 etime=1235384686 start=1235386115 owner=vsc40014@gengar1.cvos.cluster exec_host=node047.cvos.cluster/4 Resource_List.neednodes=node047.cvos.cluster Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.walltime=01:00:00" :: Text + s ~> parseTorqueStart `shouldParse` ("torque", TorqueStart TorqueJobStart + { torqueDatestamp = "02/23/2009 11:48:35" + , name = TorqueJobName {number = 102355, arrayId = Nothing, master = "master", cluster = "cvos"} + , user = "vsc40014" + , group = "vsc40014" + , account = Nothing + , jobname = "MtChr5_9036000_rmwrap.sh" + , queue = "short_eth" + , owner = "vsc40014@gengar1.cvos.cluster" + , times = TorqueJobTime {ctime = 1235384686, qtime = 1235384686, etime = 1235384686, startTime = 1235386115, endTime = Nothing} + , execHost = [TorqueExecHost {name = "node047.cvos.cluster", cores = [4]}] + , resourceRequest = TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , cputime = Nothing + , prologue = Nothing + , epilogue = Nothing + , neednodes = Just $ TFN [TorqueJobFQNode {name = "node047.cvos.cluster", ppn = Nothing}] + , nice = Just 0 + , nodeCount = 1 + , nodes = TSN (TorqueJobShortNode {number = 1, ppn = Nothing}) + , select = Nothing + , qos = Nothing + , other = Nothing + , feature = Nothing + , host = Nothing + , procs = Nothing + , nodeset = Nothing + , tpn = Nothing + , pmem = Nothing + , vmem = Nothing + , pvmem = Nothing + , mppmem = Nothing + , walltime = TorqueWalltime {days = 0, hours = 1, minutes = 0, seconds = 0} + } + , torqueEntryType = TorqueStartEntry + })