hnormalise 0.4.2.0 → 0.4.7.0
raw patch · 17 files changed
+986/−280 lines, 17 files
Files
- README.md +1/−1
- app/Main.hs +70/−53
- hnormalise.cabal +3/−2
- src/HNormalise.hs +3/−8
- src/HNormalise/Common/Internal.hs +0/−2
- src/HNormalise/Common/Parser.hs +5/−2
- src/HNormalise/Config.hs +30/−16
- src/HNormalise/Internal.hs +10/−10
- src/HNormalise/Json.hs +1/−1
- src/HNormalise/Parser.hs +24/−19
- src/HNormalise/Torque/Internal.hs +84/−28
- src/HNormalise/Torque/Json.hs +30/−7
- src/HNormalise/Torque/Parser.hs +167/−59
- src/HNormalise/Verbose.hs +54/−0
- test/Bench.hs +6/−6
- test/HNormalise/ParserSpec.hs +15/−11
- test/HNormalise/Torque/ParserSpec.hs +483/−55
README.md view
@@ -1,7 +1,7 @@ hNormalise ========== -[](http://travis-ci.org/itkovian/hnormalise)+[](http://travis-ci.org/hpcugent/hnormalise) `hNormalise` is a small tool and accompanying library for the conversion of regular [rsyslog](http://www.rsyslog.com) to structured log messages, i.e., turning the `msg` payload into (a nested) JSON object.
app/Main.hs view
@@ -1,12 +1,11 @@ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE RecordWildCards #-} module Main where -------------------------------------------------------------------------------- import Control.Applicative ((<$>), (<*>))-import Control.Concurrent.MVar (modifyMVar_, newMVar, newEmptyMVar, putMVar, readMVar, tryTakeMVar, withMVar, MVar)+import Control.Concurrent.MVar (modifyMVar_, newMVar, newEmptyMVar, putMVar, readMVar, tryTakeMVar, withMVar, takeMVar, MVar) import Control.Monad import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Loops (whileM_, untilM_)@@ -23,16 +22,17 @@ import Data.Conduit.Network import qualified Data.Conduit.Text as DCT import qualified Data.Conduit.ZMQ4 as ZMQC-import Data.Maybe (fromJust)+import Data.Maybe (fromJust, fromMaybe) import Data.Monoid ((<>)) import qualified Data.Text as T import qualified Data.Text.Encoding as TE+import Data.Time+import Data.Time.Format import Data.Version (showVersion) import qualified Options.Applicative as OA import qualified Paths_hnormalise import System.Exit (exitFailure, exitSuccess) import System.Posix.Signals (installHandler, Handler(CatchOnce), sigINT, sigTERM)-import qualified System.ZMQ4 as ZMQ (Receiver, Sender, Socket, Size, context, term, setIoThreads, Context) import qualified System.ZMQ4 as ZMQ (withContext, withSocket, bind, send, receive, connect, Push(..), Pull(..)) import Text.Printf (printf) @@ -42,15 +42,18 @@ import HNormalise.Config ( Config (..) , ConnectionType(..) , InputConfig(..)+ , LoggingConfig(..) , OutputConfig(..) , TcpOutputConfig(..) , TcpPortConfig(..) , ZeroMQOutputConfig(..) , ZeroMQPortConfig(..) , connectionType+ , defaultLoggingFrequency , loadConfig) import HNormalise.Internal (Rsyslog (..)) import HNormalise.Json+import HNormalise.Verbose -------------------------------------------------------------------------------- data Options = Options@@ -58,30 +61,35 @@ , oJsonInput :: !Bool , oVersion :: !Bool , oTestFilePath :: !(Maybe FilePath)+ , oVerbose :: !Bool } deriving (Show) -------------------------------------------------------------------------------- parserOptions :: OA.Parser Options parserOptions = Options- <$> (OA.optional $ OA.strOption $+ <$> OA.optional ( OA.strOption $ OA.long "configfile" <> OA.short 'c' <> OA.metavar "FILENAME" <> OA.help "configuration file location ")- <*> (OA.switch $+ <*> OA.switch ( OA.long "jsoninput" <> OA.short 'j' <> OA.help "Input will be delivered as JSON (slower)")- <*> (OA.switch $+ <*> OA.switch ( OA.long "version" <> OA.short 'v' <> OA.help "Display version and exit" <> OA.hidden)- <*> (OA.optional $ OA.strOption $+ <*> OA.optional ( OA.strOption $ OA.long "test" <> OA.short 't' <> OA.metavar "OUTPUT FILENAME" <> OA.help "run in test modus, sinking output to the given file")+ <*> (OA.switch $+ OA.long "verbose" <>+ OA.help "Verbose output" <>+ OA.hidden) --------------------------------------------------------------------------------@@ -103,7 +111,7 @@ -------------------------------------------------------------------------------- -- | 'messageSink' yields the parsed JSON downstream, or if parsing fails, yields the original message downstream-messageSink success failure = loop+messageSink success failure messageCount frequency = loop where loop = do v <- await@@ -111,13 +119,21 @@ Just (Transformed json) -> do yield json $$ success yield (SBS.pack "\n") $$ success+ increaseCount (1, 0) loop Just (Original l) -> do yield l $$ failure yield (SBS.pack "\n") $$ failure+ increaseCount (0, 1) loop Nothing -> return ()-+ increaseCount (s, f) =+ liftIO $ do+ (s', f') <- takeMVar messageCount+ when ((s' + f') `mod` frequency == 0) $ do+ epoch_int <- (read . formatTime defaultTimeLocale "%s" <$> getCurrentTime) :: IO Int+ printf "%ld - message count: %10d (success: %10d, fail: %10d)\n" epoch_int (s' + f') s' f'+ putMVar messageCount (s' + s, f' + f) -------------------------------------------------------------------------------- -- | 'mySink' yields the results downstream with the addition of a string mentioning success or failure@@ -140,9 +156,10 @@ Nothing -> return () ---------------------------------------------------------------------------------runTCPConnection options config = do+runTCPConnection options config messageCount = do let listenHost = fromJust $ input config >>= (\(InputConfig t _) -> t) >>= (\(TcpPortConfig h p) -> h) let listenPort = fromJust $ input config >>= (\(InputConfig t _) -> t) >>= (\(TcpPortConfig h p) -> p)+ let frequency = fromMaybe defaultLoggingFrequency (logging config >>= (\(LoggingConfig f) -> f)) runTCPServer (serverSettings listenPort "*") $ \appData -> do let fs = fields config case oTestFilePath options of@@ -154,20 +171,13 @@ runTCPClient (clientSettings successPort successHost) $ \successServer -> runTCPClient (clientSettings failurePort failureHost) $ \failServer ->- let normalisationConduit = case oJsonInput options of- True -> CB.lines $= C.map (normaliseJsonInput fs)- False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs)- in appSource appData- $= normalisationConduit- $$ messageSink (appSink successServer) (appSink failServer)- Just testSinkFileName ->- let normalisationConduit = case oJsonInput options of- True -> CB.lines $= C.map (normaliseJsonInput fs)- False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs)- in runResourceT $ appSource appData- $= normalisationConduit- $= mySink- $$ sinkFile testSinkFileName+ appSource appData+ $= normalisationConduit options config+ $$ messageSink (appSink successServer) (appSink failServer) messageCount frequency+ Just testSinkFileName -> runResourceT $ appSource appData+ $= normalisationConduit options config+ $= mySink+ $$ sinkFile testSinkFileName -- | 'zmqInterruptibleSource' converts a regular 0mq recieve operation on a socket into a conduit source -- The source is halted when something is put into the MVar, effectively stopping the program from checking@@ -183,17 +193,25 @@ liftIO $ putStrLn "Done!" ---------------------------------------------------------------------------------runZeroMQConnection options config s_interrupted = do+normalisationConduit options config = let fs = fields config+ in if oJsonInput options+ then CB.lines $= C.map (normaliseJsonInput fs)+ else DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs)+++--------------------------------------------------------------------------------+runZeroMQConnection options config s_interrupted messageCount verbose' = do+ let fs = fields config let listenHost = fromJust $ input config >>= (\(InputConfig _ z) -> z) >>= (\(ZeroMQPortConfig m h p) -> h) let listenPort = fromJust $ input config >>= (\(InputConfig _ z) -> z) >>= (\(ZeroMQPortConfig m h p) -> p)- liftIO $ putStrLn (printf "listening on tcp://%s:%d" listenHost listenPort)+ let frequency = fromMaybe defaultLoggingFrequency (logging config >>= (\(LoggingConfig f) -> f)) case oTestFilePath options of- Nothing -> do- ZMQ.withContext $ \ctx -> do+ Nothing -> ZMQ.withContext $ \ctx -> ZMQ.withSocket ctx ZMQ.Pull $ \s -> do ZMQ.bind s $ printf "tcp://%s:%d" listenHost listenPort+ verbose' $ printf "Listening on tcp://%s:%d" listenHost listenPort let successHost = fromJust $ output config >>= (\(OutputConfig _ z) -> z) >>= (\(ZeroMQOutputConfig s f) -> s) >>= (\(ZeroMQPortConfig m h p) -> h) let successPort = fromJust $ output config >>= (\(OutputConfig _ z) -> z) >>= (\(ZeroMQOutputConfig s f) -> s) >>= (\(ZeroMQPortConfig m h p) -> p)@@ -203,31 +221,24 @@ ZMQ.withSocket ctx ZMQ.Push $ \successSocket -> ZMQ.withSocket ctx ZMQ.Push $ \failureSocket -> do ZMQ.connect successSocket $ printf "tcp://%s:%d" successHost successPort- liftIO $ putStrLn (printf "connected to tcp://%s:%d" successHost successPort)- ZMQ.connect failureSocket $ printf "tcp://%s:%d" failureHost failurePort- liftIO $ putStrLn (printf "connected to tcp://%s:%d" failureHost failurePort)+ verbose' $ printf "Pushing successful parses on tcp://%s:%d" successHost successPort - let normalisationConduit = case oJsonInput options of- True -> CB.lines $= C.map (normaliseJsonInput fs)- False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs)+ ZMQ.connect failureSocket $ printf "tcp://%s:%d" failureHost failurePort+ verbose' $ printf "Pushing failed parses on tcp://%s:%d" failureHost failurePort liftIO $ zmqInterruptibleSource s_interrupted s- $= normalisationConduit- $$ messageSink (ZMQC.zmqSink successSocket []) (ZMQC.zmqSink failureSocket [])-- {-Just testSinkFileName ->- let normalisationConduit = case oJsonInput options of- True -> CB.lines $= C.map (normaliseJsonInput fs)- False -> DCT.decode DCT.utf8 $= DCT.lines $= C.map (normaliseText fs)+ $= normalisationConduit options config+ $$ messageSink (ZMQC.zmqSink successSocket []) (ZMQC.zmqSink failureSocket []) messageCount frequency - in ZMQ.withContext $ \ctx -> do- ZMQ.withSocket ctx ZMQ.Pull $ \s -> do- ZMQ.bind s $ printf "tcp://%s:%d" listenHost listenPort- liftIO $ zmqInterruptibleSource s_interrupted s- $= normalisationConduit- $= mySink- $$ sinkFile testSinkFileName+{- -- FIXME: This does not work, gives a type error+ Just testSinkFileName ->+ ZMQ.withContext $ \ctx ->+ ZMQ.withSocket ctx ZMQ.Pull $ \s -> do+ ZMQ.bind s $ printf "tcp://%s:%d" listenHost listenPort+ liftIO $ zmqInterruptibleSource s_interrupted s+ $= normalisationConduit options config+ $= mySink+ $$ sinkFile testSinkFileName -}- -------------------------------------------------------------------------------- -- | 'main' starts a TCP server, listening to incoming data and connecting to TCP servers downstream to -- for the pipeline.@@ -239,17 +250,23 @@ putStrLn $ showVersion Paths_hnormalise.version exitSuccess + let verbose' = makeVerbose (oVerbose options) config <- loadConfig (oConfigFilePath options) + verbose' $ "Config: " ++ show config+ -- For now, we only support input and output configurations of the same type, -- i.e., both TCP, both ZeroMQ, etc.+ messageCount <- newMVar ((0,0) :: (Int, Int)) case connectionType config of- TCP -> void $ runTCPConnection options config+ TCP -> void $ runTCPConnection options config messageCount ZeroMQ -> do- trace "zeromq" $ return ()+ verbose' "ZeroMQ connections" s_interrupted <- newEmptyMVar installHandler sigINT (CatchOnce $ handler s_interrupted) Nothing+ verbose' "SIGINT handler installed" installHandler sigTERM (CatchOnce $ handler s_interrupted) Nothing- runZeroMQConnection options config s_interrupted+ verbose' "SIGTERM handler installed"+ runZeroMQConnection options config s_interrupted messageCount verbose' exitSuccess
hnormalise.cabal view
@@ -1,5 +1,5 @@ name: hnormalise-version: 0.4.2.0+version: 0.4.7.0 synopsis: Log message normalisation tool producing structured JSON messages description: Log message normalisation tool producing structured JSON messages homepage: https://github.com/itkovian/hnormalise#readme@@ -7,7 +7,7 @@ license-file: LICENSE author: Andy Georges maintainer: itkovian@gmail.com-copyright: 2017 Andy Georges+copyright: 2017 Ghent University category: Command line build-type: Simple extra-source-files: README.md@@ -38,6 +38,7 @@ , HNormalise.Torque.Internal , HNormalise.Torque.Json , HNormalise.Torque.Parser+ , HNormalise.Verbose build-depends: base >= 4.7 && < 5 , aeson , aeson-pretty
src/HNormalise.hs view
@@ -32,10 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} --{-# LANGUAGE OverloadedStrings #-}-- module HNormalise ( normaliseRsyslog , normaliseJsonInput@@ -82,7 +78,7 @@ -> SBS.ByteString -- ^ Input representing an rsyslog message in JSON format -> Normalised -- ^ Transformed or Original result normaliseJsonInput fs logLine =- case (Aeson.decodeStrict logLine :: Maybe Rsyslog) >>= (normaliseRsyslog fs) of+ case (Aeson.decodeStrict logLine :: Maybe Rsyslog) >>= normaliseRsyslog fs of Just j -> Transformed j Nothing -> Original logLine @@ -100,7 +96,7 @@ _ -> original _ -> original where- original = Original $ encodeUtf8 $ logLine+ original = Original $ encodeUtf8 logLine -------------------------------------------------------------------------------- -- | The 'convertMessage' function transforms the actual message to a 'Maybe' 'ParseResult'. If parsing fails,@@ -124,5 +120,4 @@ normaliseRsyslog fs rsyslog = do cm <- convertMessage $ msg rsyslog return $ BS.toStrict- $ Aeson.encode- $ NRsyslog { rsyslog = rsyslog, normalised = cm, jsonkey = getJsonKey cm, fields = fs }+ $ Aeson.encode NRsyslog { rsyslog = rsyslog, normalised = cm, jsonkey = getJsonKey cm, fields = fs }
src/HNormalise/Common/Internal.hs view
@@ -32,10 +32,8 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE ExistentialQuantification #-}-{-# LANGUAGE OverloadedStrings #-} module HNormalise.Common.Internal ( Host (..)
src/HNormalise/Common/Parser.hs view
@@ -32,8 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-}-{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE OverloadedStrings #-} @@ -91,6 +89,11 @@ kvNumParser :: Integral a => Text -> Parser a kvNumParser key = keyParser key *> decimal {-# INLINE kvNumParser #-}++--------------------------------------------------------------------------------+kvSignedParser :: Integral a => Text -> Parser a+kvSignedParser key = keyParser key *> signed decimal+{-# INLINE kvSignedParser #-} -------------------------------------------------------------------------------- kvYesNoParser :: Text -> Parser Bool
src/HNormalise/Config.hs view
@@ -40,12 +40,14 @@ ( Config(..) , ConnectionType(..) , InputConfig(..)+ , LoggingConfig(..) , OutputConfig(..) , TcpOutputConfig(..) , TcpPortConfig(..) , ZeroMQOutputConfig(..) , ZeroMQPortConfig(..) , connectionType+ , defaultLoggingFrequency , loadConfig ) where @@ -60,6 +62,7 @@ import System.Directory import Debug.Trace+ -------------------------------------------------------------------------------- data ConnectionType = TCP | ZeroMQ@@ -72,9 +75,16 @@ _ -> ZeroMQ --------------------------------------------------------------------------------+defaultLoggingFrequency = 100000++data LoggingConfig = LoggingConfig+ { frequency :: !(Maybe Int) -- ^ How often should we write an update (# of messages processed)+ } deriving (Show)++-------------------------------------------------------------------------------- data TcpPortConfig = TcpPortConfig- { host :: !(Maybe Text)- , port :: !(Maybe Int)+ { host :: !(Maybe Text)+ , port :: !(Maybe Int) } deriving (Show) --------------------------------------------------------------------------------@@ -117,8 +127,8 @@ -------------------------------------------------------------------------------- data ZeroMQOutputConfig = ZeroMQOutputConfig- { success :: !(Maybe ZeroMQPortConfig)- , failure :: !(Maybe ZeroMQPortConfig)+ { success :: !(Maybe ZeroMQPortConfig)+ , failure :: !(Maybe ZeroMQPortConfig) } deriving (Show) --------------------------------------------------------------------------------@@ -131,8 +141,8 @@ -------------------------------------------------------------------------------- data InputConfig = InputConfig- { tcp :: !(Maybe TcpPortConfig)- , zeromq :: !(Maybe ZeroMQPortConfig)+ { tcp :: !(Maybe TcpPortConfig)+ , zeromq :: !(Maybe ZeroMQPortConfig) } deriving (Show) --------------------------------------------------------------------------------@@ -145,8 +155,8 @@ -------------------------------------------------------------------------------- data OutputConfig = OutputConfig- { tcp :: !(Maybe TcpOutputConfig)- , zeromq :: !(Maybe ZeroMQOutputConfig)+ { tcp :: !(Maybe TcpOutputConfig)+ , zeromq :: !(Maybe ZeroMQOutputConfig) } deriving (Show) --------------------------------------------------------------------------------@@ -165,8 +175,8 @@ } defaultOutputTcpConfig = TcpOutputConfig- { success = Just $ TcpPortConfig { host = Just "localhost", port = Just 26001 }- , failure = Just $ TcpPortConfig { host = Just "localhost", port = Just 26002 }+ { success = Just TcpPortConfig { host = Just "localhost", port = Just 26001 }+ , failure = Just TcpPortConfig { host = Just "localhost", port = Just 26002 } } defaultInputConfig = InputConfig@@ -181,22 +191,25 @@ -------------------------------------------------------------------------------- data Config = Config- { input :: !(Maybe InputConfig)- , output :: !(Maybe OutputConfig)- , fields :: !(Maybe [(Text, Text)])+ { logging :: !(Maybe LoggingConfig)+ , input :: !(Maybe InputConfig)+ , output :: !(Maybe OutputConfig)+ , fields :: !(Maybe [(Text, Text)]) } deriving (Show) -------------------------------------------------------------------------------- instance Monoid Config where- mempty = Config Nothing Nothing Nothing+ mempty = Config Nothing Nothing Nothing Nothing mappend l r = Config- { input = input l `mplus` input r+ { logging = logging l `mplus` logging r+ , input = input l `mplus` input r , output = output l `mplus` output r , fields = fields l `mplus` fields r } defaultConfig = Config- { input = Nothing+ { logging = Just LoggingConfig { frequency = Just defaultLoggingFrequency }+ , input = Nothing , output = Nothing , fields = Nothing }@@ -229,6 +242,7 @@ return $ userConfig <> systemConfig <> defaultConfig --------------------------------------------------------------------------------+$(deriveJSON defaultOptions ''LoggingConfig) $(deriveJSON defaultOptions ''TcpPortConfig) $(deriveJSON defaultOptions ''TcpOutputConfig) $(deriveJSON defaultOptions ''ZeroMQPortConfig)
src/HNormalise/Internal.hs view
@@ -32,9 +32,9 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE DuplicateRecordFields #-}-{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE DuplicateRecordFields #-}+{-# LANGUAGE OverloadedStrings #-} module HNormalise.Internal where @@ -54,7 +54,7 @@ import HNormalise.Shorewall.Json import HNormalise.Snoopy.Internal (Snoopy) import HNormalise.Snoopy.Json-import HNormalise.Torque.Internal (TorqueJobExit)+import HNormalise.Torque.Internal (TorqueParseResult) import HNormalise.Torque.Json --------------------------------------------------------------------------------@@ -67,17 +67,17 @@ | PR_Shorewall Shorewall -- | Represents a parsed 'Snoopy' message | PR_Snoopy Snoopy- -- | Represents a parsed 'TorqueJobExit' message- | PR_Torque TorqueJobExit+ -- | Represents a parsed 'Torque' message+ | PR_Torque TorqueParseResult deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- instance ToJSON ParseResult where- toEncoding (PR_Huppel v) = toEncoding v- toEncoding (PR_Lmod v) = toEncoding v+ toEncoding (PR_Huppel v) = toEncoding v+ toEncoding (PR_Lmod v) = toEncoding v toEncoding (PR_Shorewall v) = toEncoding v- toEncoding (PR_Snoopy v) = toEncoding v- toEncoding (PR_Torque v) = toEncoding v+ toEncoding (PR_Snoopy v) = toEncoding v+ toEncoding (PR_Torque v) = toEncoding v -------------------------------------------------------------------------------- data Rsyslog = Rsyslog
src/HNormalise/Json.hs view
@@ -41,7 +41,7 @@ -------------------------------------------------------------------------------- import Control.Monad import Data.Aeson-import qualified Data.HashMap.Lazy as M+import qualified Data.HashMap.Lazy as M import Data.Monoid --------------------------------------------------------------------------------
src/HNormalise/Parser.hs view
@@ -32,7 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE ExistentialQuantification #-} {-# LANGUAGE OverloadedStrings #-}@@ -40,16 +39,16 @@ module HNormalise.Parser where ---------------------------------------------------------------------------------import Control.Applicative ((<|>))-import Data.Aeson (encode)+import Control.Applicative ((<|>))+import Data.Aeson (encode) import Data.Attoparsec.Text import Data.Attoparsec.Time-import qualified Data.ByteString.Char8 as SBS-import qualified Data.ByteString.Lazy.Char8 as BS+import qualified Data.ByteString.Char8 as SBS+import qualified Data.ByteString.Lazy.Char8 as BS import Data.Char-import Data.Text (Text, empty)-import qualified Data.Text as T-import qualified Data.Text.Encoding as TE+import Data.Text (Text, empty)+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE -------------------------------------------------------------------------------- import HNormalise.Common.Parser import HNormalise.Huppel.Parser@@ -66,20 +65,26 @@ -- | The 'parseMessage' function will try and use each configured parser to normalise the input it's given parseMessage :: Parser (Text, ParseResult) parseMessage =- (parseLmodLoad >>= (\(a, v) -> return $ (a, PR_Lmod v)))- <|> (parseShorewall >>= (\(a, v) -> return $ (a, PR_Shorewall v)))- <|> (parseSnoopy >>= (\(a, v) -> return $ (a, PR_Snoopy v)))- <|> (parseTorqueExit >>= (\(a, v) -> return $ (a, PR_Torque v)))+ let pm parser target = (parser >>= (\(a, v) -> return (a, target v)))+ in pm parseLmodLoad PR_Lmod+ <|> pm parseShorewall PR_Shorewall+ <|> pm parseSnoopy PR_Snoopy+ <|> pm parseTorqueQueue PR_Torque+ <|> pm parseTorqueStart PR_Torque+ <|> pm parseTorqueDelete PR_Torque+ <|> pm parseTorqueExit PR_Torque+ <|> pm parseTorqueAbort PR_Torque+ <|> pm parseTorqueRerun PR_Torque -------------------------------------------------------------------------------- -- | The 'getJsonKey' function return the key under which the normalised message should appear when JSON is produced getJsonKey :: ParseResult -- ^ Wrapped result for which we need to get a key -> Text -- ^ Key for use in the JSON encoding of the result-getJsonKey (PR_Huppel _) = "huppel"-getJsonKey (PR_Lmod _) = "lmod"-getJsonKey (PR_Torque _) = "torque"+getJsonKey (PR_Huppel _) = "huppel"+getJsonKey (PR_Lmod _) = "lmod"+getJsonKey (PR_Torque _) = "torque" getJsonKey (PR_Shorewall _) = "shorewall"-getJsonKey (PR_Snoopy _) = "snoopy"+getJsonKey (PR_Snoopy _) = "snoopy" -------------------------------------------------------------------------------- -- | The 'parseRsyslogLogstashString' currently is a placeholder function that will convert the incoming rsyslog message@@ -92,7 +97,7 @@ char '<' p <- decimal char '>'- v <- maybeOption $ decimal+ v <- maybeOption decimal return (p, v) timereported <- skipSpace *> zonedTime hostname <- skipSpace *> takeTill isSpace@@ -100,7 +105,7 @@ skipSpace *> char '-' *> skipSpace (original, (appname, parsed)) <- match parseMessage return $ let jsonkey = getJsonKey parsed- in BS.toStrict $ encode $ NRsyslog+ in BS.toStrict $ encode NRsyslog { rsyslog = Rsyslog { msg = original , timereported = timereported@@ -110,7 +115,7 @@ , fromhost = T.empty , fromhost_ip = T.empty , pri = abspri >>= \(p, _) -> return p- , version = abspri >>= \(_, v) -> v+ , version = abspri >>= snd , syslogfacility = T.empty , syslogseverity = T.empty , timegenerated = Nothing
src/HNormalise/Torque/Internal.hs view
@@ -32,11 +32,8 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE BangPatterns #-}-{-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-}-{-# LANGUAGE OverloadedStrings #-} module HNormalise.Torque.Internal where @@ -45,6 +42,28 @@ import GHC.Generics (Generic) --------------------------------------------------------------------------------+-- | `TorqueParseResult` encapsulates the results we get from parsing torque lines+data TorqueParseResult+ = TorqueQueue TorqueJobQueue+ | TorqueStart TorqueJobStart+ | TorqueDelete TorqueJobDelete+ | TorqueExit TorqueJobExit+ | TorqueAbort TorqueJobAbort+ | TorqueRerun TorqueJobRerun+ deriving (Show, Eq, Generic)++--------------------------------------------------------------------------------+-- | `TorqueEntryType` distinguishes between accounting data entry Types+data TorqueEntryType+ = TorqueQueueEntry+ | TorqueStartEntry+ | TorqueDeleteEntry+ | TorqueExitEntry+ | TorqueAbortEntry+ | TorqueRerunEntry+ deriving (Show, Eq, Generic)++-------------------------------------------------------------------------------- data TorqueJobShortNode = TorqueJobShortNode { number :: !Int , ppn :: !(Maybe Int)@@ -53,7 +72,7 @@ -------------------------------------------------------------------------------- data TorqueJobFQNode = TorqueJobFQNode { name :: !Text- , ppn :: !Int+ , ppn :: !(Maybe Int) } deriving (Show, Eq, Generic) --------------------------------------------------------------------------------@@ -63,10 +82,13 @@ -------------------------------------------------------------------------------- data TorqueExecHost = TorqueExecHost- { name :: !Text- , cores :: ![Int]+ { name :: !Text+ , cores :: ![Int] } deriving (Show, Eq, Generic) +instance Ord TorqueExecHost where+ compare (TorqueExecHost t1 _) (TorqueExecHost t2 _) = compare t1 t2+ -------------------------------------------------------------------------------- data TorqueWalltime = TorqueWalltime { days :: !Int@@ -81,22 +103,32 @@ , advres :: !(Maybe Text) , naccesspolicy :: !(Maybe Text) , ncpus :: !(Maybe Int)- , neednodes :: !TorqueJobNode+ , cputime :: !(Maybe TorqueWalltime)+ , prologue :: !(Maybe Text)+ , epilogue :: !(Maybe Text)+ , neednodes :: !(Maybe TorqueJobNode) , nice :: !(Maybe Int) , nodeCount :: !Int , nodes :: !TorqueJobNode , select :: !(Maybe Text) , qos :: !(Maybe Text)+ , other :: !(Maybe Text)+ , feature :: !(Maybe Text)+ , host :: !(Maybe Text)+ , procs :: !(Maybe Text)+ , nodeset :: !(Maybe Text)+ , tpn :: !(Maybe Text) , pmem :: !(Maybe Integer) , vmem :: !(Maybe Integer) , pvmem :: !(Maybe Integer)+ , mppmem :: !(Maybe Integer) , walltime :: !TorqueWalltime } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueResourceUsage = TorqueResourceUsage { cputime :: !Integer- , energy :: !Integer+ , energy :: !(Maybe Integer) , mem :: !Integer , vmem :: !Integer , walltime :: !TorqueWalltime@@ -113,9 +145,11 @@ -------------------------------------------------------------------------------- data TorqueJobExit = TorqueJobExit- { name :: !TorqueJobName+ { torqueDatestamp :: !Text+ , name :: !TorqueJobName , user :: !Text , group :: !Text+ , account :: !(Maybe Text) , jobname :: !Text , queue :: !Text , startCount :: !(Maybe Int)@@ -128,43 +162,65 @@ , totalExecutionSlots :: !Int , uniqueNodeCount :: !Int , exitStatus :: !Int+ , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueJobName = TorqueJobName- { number :: !Integer- , array_id :: !(Maybe Integer)- , master :: !Text- , cluster :: !Text+ { number :: !Integer+ , arrayId :: !(Maybe Integer)+ , master :: !Text+ , cluster :: !Text } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueJobQueue = TorqueJobQueue- { name :: !TorqueJobName- , queue :: !Text+ { torqueDatestamp :: !Text+ , name :: !TorqueJobName+ , queue :: !Text+ , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueJobStart = TorqueJobStart- { name :: !TorqueJobName- , user :: !Text- , group :: !Text- , jobname :: !Text- , queue :: !Text- , owner :: !Text- , times :: !TorqueJobTime- , execHost :: ![TorqueExecHost]- , resourceRequest :: !TorqueResourceRequest+ { torqueDatestamp :: !Text+ , name :: !TorqueJobName+ , user :: !Text+ , group :: !Text+ , account :: !(Maybe Text)+ , jobname :: !Text+ , queue :: !Text+ , owner :: !Text+ , times :: !TorqueJobTime+ , execHost :: ![TorqueExecHost]+ , resourceRequest :: !TorqueResourceRequest+ , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueRequestor = TorqueRequestor- { user :: !Text- , whence :: !Text+ { user :: !Text+ , whence :: !Text } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- data TorqueJobDelete = TorqueJobDelete- { name :: !TorqueJobName- , requestor :: !TorqueRequestor+ { torqueDatestamp :: !Text+ , name :: !TorqueJobName+ , requestor :: !TorqueRequestor+ , torqueEntryType :: TorqueEntryType+ } deriving (Show, Eq, Generic)++--------------------------------------------------------------------------------+data TorqueJobAbort = TorqueJobAbort+ { torqueDatestamp :: !Text+ , name :: !TorqueJobName+ , torqueEntryType :: TorqueEntryType+ } deriving (Show, Eq, Generic)++--------------------------------------------------------------------------------+data TorqueJobRerun = TorqueJobRerun+ { torqueDatestamp :: !Text+ , name :: !TorqueJobName+ , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic)
src/HNormalise/Torque/Json.hs view
@@ -32,21 +32,18 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-}-{-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE OverloadedStrings #-}-- module HNormalise.Torque.Json where -------------------------------------------------------------------------------- import Data.Aeson-import Data.Monoid ((<>))+import Data.Monoid ((<>)) -------------------------------------------------------------------------------- import HNormalise.Torque.Internal --------------------------------------------------------------------------------+instance ToJSON TorqueEntryType where+ toEncoding = genericToEncoding defaultOptions instance ToJSON TorqueJobShortNode where toEncoding = genericToEncoding defaultOptions@@ -61,7 +58,7 @@ toEncoding (TorqueWalltime d h m s) = toEncoding $ (((d * 24 + h) * 60) + m) * 60 + s instance ToJSON TorqueJobNode where- toEncoding (TSN n) = toEncoding n+ toEncoding (TSN n) = toEncoding n toEncoding (TFN ns) = toEncoding ns instance ToJSON TorqueResourceRequest where@@ -78,3 +75,29 @@ instance ToJSON TorqueJobName where toEncoding = genericToEncoding defaultOptions++instance ToJSON TorqueRequestor where+ toEncoding = genericToEncoding defaultOptions++instance ToJSON TorqueJobStart where+ toEncoding = genericToEncoding defaultOptions++instance ToJSON TorqueJobQueue where+ toEncoding = genericToEncoding defaultOptions++instance ToJSON TorqueJobDelete where+ toEncoding = genericToEncoding defaultOptions++instance ToJSON TorqueJobAbort where+ toEncoding = genericToEncoding defaultOptions++instance ToJSON TorqueJobRerun where+ toEncoding = genericToEncoding defaultOptions++instance ToJSON TorqueParseResult where+ toEncoding (TorqueQueue ts) = toEncoding ts+ toEncoding (TorqueStart ts) = toEncoding ts+ toEncoding (TorqueDelete ts) = toEncoding ts+ toEncoding (TorqueExit ts) = toEncoding ts+ toEncoding (TorqueAbort ts) = toEncoding ts+ toEncoding (TorqueRerun ts) = toEncoding ts
src/HNormalise/Torque/Parser.hs view
@@ -32,8 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-}-{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE OverloadedStrings #-} @@ -41,13 +39,17 @@ -------------------------------------------------------------------------------- import Control.Applicative ((<|>))+import Control.Monad (join) import Data.Attoparsec.Combinator (lookAhead, manyTill) import Data.Attoparsec.Text import Data.Char (isDigit, isSpace)+import Data.List (concatMap, groupBy, sort) import qualified Data.Map as M+import Data.Maybe (fromMaybe) import Data.Text (Text) import qualified Data.Text as T-import Text.ParserCombinators.Perm ((<$$>), (<||>), (<$?>), (<|?>), permute)+import Text.ParserCombinators.Perm (permute, (<$$>), (<$?>), (<|?>),+ (<||>)) -------------------------------------------------------------------------------- import HNormalise.Common.Parser@@ -109,29 +111,34 @@ m <- char '.' *> takeTill (== '.') c <- char '.' *> takeTill (== '.') manyTill anyChar (lookAhead ";") *> char ';'- return $ TorqueJobName { number = n, array_id = a, master = m, cluster = c}+ return TorqueJobName { number = n, arrayId = a, master = m, cluster = c} where parseArrayId :: Parser (Maybe Integer)- parseArrayId = try $ maybeOption $ do- char '['- i <- decimal- char ']'- return i+ parseArrayId = try parseArrayIdBracket <|> parseArrayIdDash+ where parseArrayIdBracket = do+ char '['+ i <- maybeOption decimal+ char ']'+ return i+ parseArrayIdDash = maybeOption $ do+ char '-'+ decimal -------------------------------------------------------------------------------- -- | 'parseTorqueResourceNodeList' parses a list of FQDN nodes and their ppn or a nodecount and its ppn+-- FIXME: Add support for resource lists of the form Resource_List.neednodes=3:ppn=8+1:ppn=1 parseTorqueResourceNodeList :: Parser TorqueJobNode parseTorqueResourceNodeList = do c <- peekChar' if Data.Char.isDigit c then do number <- decimal ppn <- maybeOption $ char ':' *> string "ppn=" *> decimal- return $ TSN $ TorqueJobShortNode { number = number, ppn = ppn }- else TFN <$> (flip sepBy (char '+') $ do- fqdn <- Data.Attoparsec.Text.takeWhile (/= ':')- ppn <- char ':' *> kvNumParser "ppn"- return TorqueJobFQNode { name = fqdn, ppn = ppn})+ return $ TSN TorqueJobShortNode { number = number, ppn = ppn }+ else TFN <$> sepBy (do+ fqdn <- Data.Attoparsec.Text.takeWhile (\c -> c /= ':' && c /= ' ')+ ppn <- maybeOption $ char ':' *> kvNumParser "ppn"+ return TorqueJobFQNode { name = fqdn, ppn = ppn}) (char '+') -------------------------------------------------------------------------------- {- Examples found in the 2016 logs@@ -149,38 +156,61 @@ Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.qos Resource_List.vmem Resource_List.walltime Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.walltime++-- 2014 logs+Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime+Resource_List.cput Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime -} -- | 'parseTorqueResourceRequest' parses all key value pairs denoting resources requested.--- Most of these are not obligatory. Since the Torque documentation is vague on mentioning which entries occur, the last--- 1.5 years of data we have were used to make an educated guess as to which keys might appear and in what order+-- Most of these are not obligatory. Since the Torque documentation is vague on mentioning which entries occur,+-- the guesses as to the most common ordering and the mandatory fields is based on 5 years of log data from Torque+-- 4.x to 6.0 parseTorqueResourceRequest :: Parser TorqueResourceRequest-parseTorqueResourceRequest = do+parseTorqueResourceRequest = permute $ TorqueResourceRequest <$?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.mem=" *> parseTorqueMemory)) <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.advres")) <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.naccesspolicy")) <|?> (Nothing, Just `fmap` (skipSpace *> kvNumParser "Resource_List.ncpus"))- <||> skipSpace *> string "Resource_List.neednodes=" *> parseTorqueResourceNodeList+ <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.cput=" *> parseTorqueWalltime))+ <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.prologue"))+ <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.epilogue"))+ <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.neednodes=" *> parseTorqueResourceNodeList)) <|?> (Nothing, Just `fmap` (skipSpace *> kvNumParser "Resource_List.nice")) <||> skipSpace *> kvNumParser "Resource_List.nodect" <||> skipSpace *> string "Resource_List.nodes=" *> parseTorqueResourceNodeList <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.select")) <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.qos"))+ <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.other"))+ <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.feature"))+ <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.host"))+ <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.procs"))+ <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.nodeset"))+ <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.tpn")) <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.pmem=" *> parseTorqueMemory)) <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.vmem=" *> parseTorqueMemory)) <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.pvmem=" *> parseTorqueMemory))+ <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.mppmem=" *> parseTorqueMemory)) <||> skipSpace *> string "Resource_List.walltime=" *> parseTorqueWalltime --------------------------------------------------------------------------------+-- | 'parseTorqueCpuTime' parses the cpu time spent+-- This value is either given in seconds or in a torque timestamp format+-- We always convert the value to seconds if needed+parseTorqueCpuTime :: Parser Integer+parseTorqueCpuTime =+ try (parseTorqueWalltime >>= \(TorqueWalltime d h m s) -> return $ fromIntegral (((d*24+h)*60+m)*60+s)) <|> decimal++-------------------------------------------------------------------------------- -- | 'parseTorqueResourceUsage' parses all the key value pairs denoting used resources. parseTorqueResourceUsage :: Parser TorqueResourceUsage parseTorqueResourceUsage = do- cput <- skipSpace *> kvNumParser "resources_used.cput"- energy <- skipSpace *> kvNumParser "resources_used.energy_used"+ cput <- skipSpace *> string "resources_used.cput=" *> parseTorqueCpuTime+ energy <- skipSpace *> maybeOption (kvNumParser "resources_used.energy_used") mem <- skipSpace *> string "resources_used.mem=" *> parseTorqueMemory vmem <- skipSpace *> string "resources_used.vmem=" *> parseTorqueMemory walltime <- skipSpace *> string "resources_used.walltime=" *> parseTorqueWalltime- return $ TorqueResourceUsage+ return TorqueResourceUsage { cputime = cput , energy = energy , mem = mem@@ -189,16 +219,30 @@ } --------------------------------------------------------------------------------+-- | `aggregateHosts` take a list of TorqueExecHost and condenses them to a minimal form+-- There will be one entry for each different host, each time with the cores combined+aggregateHosts :: [TorqueExecHost] -> [TorqueExecHost]+aggregateHosts ths =+ let ths' = groupBy (\(TorqueExecHost n1 _) (TorqueExecHost n2 _) -> n1 == n2) $ sort ths+ in map aggCores ths'+ where aggCores :: [TorqueExecHost] -> TorqueExecHost+ aggCores ths@(TorqueExecHost n _:_) = TorqueExecHost+ { name = n+ , cores = sort . concatMap cores $ ths+ }++-------------------------------------------------------------------------------- -- | 'parseTorqueHostList' parses a '+' separated list of hostname/coreranges -- A core range can be of the form 1,3,5-7,9 parseTorqueHostList :: Parser [TorqueExecHost] parseTorqueHostList = do string "exec_host="- flip sepBy (char '+') $ do+ hosts <- flip sepBy (char '+') $ do fqdn <- Data.Attoparsec.Text.takeWhile (/= '/') char '/' cores <- parseCores- return $ TorqueExecHost { name = fqdn, cores = cores}+ return TorqueExecHost { name = fqdn, cores = cores}+ return $ aggregateHosts hosts where parseCores :: Parser [Int] parseCores = do cores <- flip sepBy1' (char ',') $ try parseRange <|> parseSingle@@ -225,36 +269,74 @@ { user = user , whence = whence }+--------------------------------------------------------------------------------+-- | `parseTorqueAccountingDatestamp` parses the datestamp and the given log line tag+parseTorqueAccountingDatestamp :: Text -> Parser Text+parseTorqueAccountingDatestamp tag = do+ string "torque: "+ torqueDatestamp <- takeTill (== ';')+ string tag -- drop the prefix+ return torqueDatestamp ----------------------------------------------------------------------------------- | 'parseTorqueExit' parses a complete log line denoting a job exit. Tested with Torque 6.1.x.-parseTorqueExit :: Parser (Text, TorqueJobExit)-parseTorqueExit = do- takeTill (== ';') *> string ";E;" -- drop the prefix+-- | 'parseCommonAccountingInfo' parser the initial part that is common between start and exit lines+parseCommonAccountingInfo :: Parser+ (TorqueJobName+ , Text+ , Text+ , Maybe Text+ , Text+ , Text+ , Integer+ , Integer+ , Integer)+parseCommonAccountingInfo = do name <- parseTorqueJobName user <- kvTextParser "user" group <- skipSpace *> kvTextParser "group"+ account <- skipSpace *> maybeOption (kvTextParser "account") jobname <- skipSpace *> kvTextParser "jobname" queue <- skipSpace *> kvTextParser "queue" ctime <- skipSpace *> kvNumParser "ctime" qtime <- skipSpace *> kvNumParser "qtime" etime <- skipSpace *> kvNumParser "etime"- start_count <- maybeOption $ skipSpace *> kvNumParser "start_count"+ return (name, user, group, account, jobname, queue, ctime, qtime, etime)++--------------------------------------------------------------------------------+-- | 'parseCommonStartInfo' parses the start information that is common between start and exit lines+parseCommonStartInfo :: Parser+ ( Integer+ , Text+ , [TorqueExecHost]+ , TorqueResourceRequest)+parseCommonStartInfo = do start <- skipSpace *> kvNumParser "start" owner <- skipSpace *> kvTextParser "owner" exec_host <- skipSpace *> parseTorqueHostList request <- parseTorqueResourceRequest+ return (start, owner, exec_host, request)++--------------------------------------------------------------------------------+-- | 'parseTorqueExit' parses a complete log line denoting a job exit. Tested with Torque 6.1.x.+parseTorqueExit :: Parser (Text, TorqueParseResult)+parseTorqueExit = do+ torqueDatestamp <- parseTorqueAccountingDatestamp ";E;"+ (name, user, group, account, jobname, queue, ctime, qtime, etime) <- parseCommonAccountingInfo+ start_count <- maybeOption $ skipSpace *> kvNumParser "start_count"+ (start, owner, exec_host, request) <- parseCommonStartInfo session <- skipSpace *> kvNumParser "session"- total_execution_slots <- skipSpace *> kvNumParser "total_execution_slots"- unique_node_count <- skipSpace *> kvNumParser "unique_node_count"+ total_execution_slots <- skipSpace *> maybeOption (kvNumParser "total_execution_slots")+ unique_node_count <- skipSpace *> maybeOption (kvNumParser "unique_node_count") end <- skipSpace *> kvNumParser "end"- exit_status <- skipSpace *> kvNumParser "Exit_status"+ exit_status <- skipSpace *> kvSignedParser "Exit_status" usage <- skipSpace *> parseTorqueResourceUsage - return $ ("torque", TorqueJobExit- { name = name+ return ("torque", TorqueExit TorqueJobExit+ { torqueDatestamp = torqueDatestamp+ , name = name , user = user , group = group+ , account = account , jobname = jobname , queue = queue , startCount = start_count@@ -270,59 +352,84 @@ , execHost = exec_host , resourceRequest = request , resourceUsage = usage- , totalExecutionSlots = total_execution_slots- , uniqueNodeCount = unique_node_count+ , totalExecutionSlots = fromMaybe (compute_total_execution_slots exec_host) total_execution_slots+ , uniqueNodeCount = fromMaybe (length exec_host) unique_node_count , exitStatus = exit_status+ , torqueEntryType = TorqueExitEntry })+ where compute_total_execution_slots = sum . map (\(TorqueExecHost _ cs) -> length cs) -------------------------------------------------------------------------------- -- | `parseTorqueDelete` parses a complete log line denoting a deleted job. Tested with Torue 6.1.x-parseTorqueDelete :: Parser (Text, TorqueJobDelete)+parseTorqueDelete :: Parser (Text, TorqueParseResult) parseTorqueDelete = do- takeTill (== ';') *> string ";D;" -- drop the prefix+ torqueDatestamp <- parseTorqueAccountingDatestamp ";D;" name <- parseTorqueJobName requestor <- parseTorqueRequestor - return ("torque", TorqueJobDelete- { name = name+ return ("torque", TorqueDelete TorqueJobDelete+ { torqueDatestamp = torqueDatestamp+ , name = name , requestor = requestor+ , torqueEntryType = TorqueDeleteEntry }) --------------------------------------------------------------------------------+-- | `parseTorqueAbort` parses a complete log line denoting an aborted job. Tested with Torque 4.x+parseTorqueAbort :: Parser (Text, TorqueParseResult)+parseTorqueAbort = do+ torqueDatestamp <- parseTorqueAccountingDatestamp ";A;"+ name <- parseTorqueJobName++ return ("torque", TorqueAbort TorqueJobAbort+ { torqueDatestamp = torqueDatestamp+ , name = name+ , torqueEntryType = TorqueAbortEntry+ })++--------------------------------------------------------------------------------+-- | `parseTorqueRerun` parses a complete log line denoting a rerun job. Tested with Torque 4.x+parseTorqueRerun :: Parser (Text, TorqueParseResult)+parseTorqueRerun = do+ torqueDatestamp <- parseTorqueAccountingDatestamp ";R;"+ name <- parseTorqueJobName++ return ("torque", TorqueRerun TorqueJobRerun+ { torqueDatestamp = torqueDatestamp+ , name = name+ , torqueEntryType = TorqueRerunEntry+ })+++-------------------------------------------------------------------------------- -- | `parseTorqueQueue` parses a complete log line denoting a queued job. Tested with Torue 6.1.x-parseTorqueQueue :: Parser (Text, TorqueJobQueue)+parseTorqueQueue :: Parser (Text, TorqueParseResult) parseTorqueQueue = do- takeTill (== ';') *> string ";Q;" -- drop the prefix+ torqueDatestamp <- parseTorqueAccountingDatestamp ";Q;" name <- parseTorqueJobName queue <- kvTextParser "queue" - return ("torque", TorqueJobQueue- { name = name+ return ("torque", TorqueQueue TorqueJobQueue+ { torqueDatestamp = torqueDatestamp+ , name = name , queue = queue+ , torqueEntryType = TorqueQueueEntry }) -------------------------------------------------------------------------------- -- | `parseTorqueStart` parses a complete log line denoting a started job. Tested with Torque 6.1.x-parseTorqueStart :: Parser (Text, TorqueJobStart)+parseTorqueStart :: Parser (Text, TorqueParseResult) parseTorqueStart = do- takeTill (== ';') *> string ";S;" -- drop the prefix- name <- parseTorqueJobName- user <- kvTextParser "user"- group <- skipSpace *> kvTextParser "group"- jobname <- skipSpace *> kvTextParser "jobname"- queue <- skipSpace *> kvTextParser "queue"- ctime <- skipSpace *> kvNumParser "ctime"- qtime <- skipSpace *> kvNumParser "qtime"- etime <- skipSpace *> kvNumParser "etime"- start <- skipSpace *> kvNumParser "start"- owner <- skipSpace *> kvTextParser "owner"- exec_host <- skipSpace *> parseTorqueHostList- request <- parseTorqueResourceRequest+ torqueDatestamp <- parseTorqueAccountingDatestamp ";S;"+ (name, user, group, account, jobname, queue, ctime, qtime, etime) <- parseCommonAccountingInfo+ (start, owner, exec_host, request) <- parseCommonStartInfo - return $ ("torque", TorqueJobStart- { name = name+ return ("torque", TorqueStart TorqueJobStart+ { torqueDatestamp = torqueDatestamp+ , name = name , user = user , group = group+ , account = account , jobname = jobname , queue = queue , owner = owner@@ -335,4 +442,5 @@ } , execHost = exec_host , resourceRequest = request+ , torqueEntryType = TorqueStartEntry })
+ src/HNormalise/Verbose.hs view
@@ -0,0 +1,54 @@+{- hnormalise - a log normalisation library+ -+ - Copyright Andy Georges (c) 2017+ -+ - All rights reserved.+ -+ - Redistribution and use in source and binary forms, with or without+ - modification, are permitted provided that the following conditions are met:+ -+ - * Redistributions of source code must retain the above copyright+ - notice, this list of conditions and the following disclaimer.+ -+ - * Redistributions in binary form must reproduce the above+ - copyright notice, this list of conditions and the following+ - disclaimer in the documentation and/or other materials provided+ - with the distribution.+ -+ - * Neither the name of Author name here nor the names of other+ - contributors may be used to endorse or promote products derived+ - from this software without specific prior written permission.+ -+ - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+-}++{- This code was taken from https://github.com/jaspervdj/stylish-haskell/blob/master/lib/Language/Haskell/Stylish/Verbose.hs -}+module HNormalise.Verbose+ ( Verbose+ , makeVerbose+ ) where+++--------------------------------------------------------------------------------+import System.IO (hPutStrLn, stderr)+++--------------------------------------------------------------------------------+type Verbose = String -> IO ()+++--------------------------------------------------------------------------------+makeVerbose :: Bool -> Verbose+makeVerbose verbose+ | verbose = hPutStrLn stderr+ | otherwise = const $ return ()
test/Bench.hs view
@@ -57,14 +57,14 @@ rnf (H.Original s) = rnf s ---------------------------------------------------------------------------------torqueJobExitInput1 = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text-torqueJobExitInput2 = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text+torqueJobExitInput1 = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text+torqueJobExitInput2 = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text -torqueJobExitFailInput1 = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short HUPPEL"+torqueJobExitFailInput1 = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short HUPPEL" -torqueJobQueueInput = "06/28/2017 14:31:09;Q;80.master23.banette.gent.vsc;queue=default"-torqueJobDeleteInput = "06/28/2017 15:44:02;D;81.master23.banette.gent.vsc;requestor=vsc40075@gligar02.gligar.gent.vsc"-torqueJobStartInput = "06/20/2017 11:24:49;S;63.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1497950675 qtime=1497950675 etime=1497950675 start=1497950689 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.vmem=4224531456b Resource_List.nodes=1:ppn=1 Resource_List.walltime=00:10:00 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1 Resource_List.nice=0"+torqueJobQueueInput = "torque: 06/28/2017 14:31:09;Q;80.master23.banette.gent.vsc;queue=default"+torqueJobDeleteInput = "torque: 06/28/2017 15:44:02;D;81.master23.banette.gent.vsc;requestor=vsc40075@gligar02.gligar.gent.vsc"+torqueJobStartInput = "torque: 06/20/2017 11:24:49;S;63.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1497950675 qtime=1497950675 etime=1497950675 start=1497950689 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.vmem=4224531456b Resource_List.nodes=1:ppn=1 Resource_List.walltime=00:10:00 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1 Resource_List.nice=0" lmodLoadInput1 = "lmod:: username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text fullLmodInput = "<13>1 2016-06-07T17:50:22.495571+02:00 node2159 lmod: - lmod:: username=vsc40307, cluster=delcatty, jobid=434.master16.delcatty.gent.vsc, userload=no, module=binutils/2.25-GCCcore-4.9.3, fn=/apps/gent/SL6/sandybridge/modules/all/binutils/2.25-GCCcore-4.9.3" :: Text
test/HNormalise/ParserSpec.hs view
@@ -38,16 +38,16 @@ module HNormalise.ParserSpec (main, spec) where ---------------------------------------------------------------------------------import Data.Text (Text)-import qualified Data.Text.Read as TR+import Data.Text (Text)+import qualified Data.Text.Read as TR+import qualified Net.IPv4 as NT import Test.Hspec import Test.Hspec.Attoparsec-import qualified Net.IPv4 as NT -------------------------------------------------------------------------------- import HNormalise.Common.Internal-import HNormalise.Lmod.Internal import HNormalise.Internal+import HNormalise.Lmod.Internal import HNormalise.Parser -------------------------------------------------------------------------------- main :: IO ()@@ -55,7 +55,7 @@ -------------------------------------------------------------------------------- spec :: Spec-spec = do+spec = describe "parse full rsyslog message" $ do it "lmod load" $ do let s = "<13>1 2016-06-07T17:50:22.658452+02:00 node2159 lmod: - lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016" :: Text@@ -63,13 +63,17 @@ "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}" it "imfile torque input" $ do- let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text- s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=vsc40075@gligar03.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"test2802\",\"torque\":{\"name\":{\"number\":0,\"array_id\":null,\"master\":\"master23\",\"cluster\":\"banette\"},\"user\":\"vsc40075\",\"group\":\"vsc40075\",\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"vsc40075@gligar03.gligar.gent.vsc\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"execHost\":[{\"name\":\"node2801.banette.gent.vsc\",\"cores\":[0]}],\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"neednodes\":{\"number\":1,\"ppn\":null},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"number\":1,\"ppn\":null},\"select\":null,\"qos\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"walltime\":3600},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":2644},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265}}"+ let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text+ s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"test2802\",\"torque\":{\"torqueDatestamp\":\"01/25/2017 15:04:10\",\"name\":{\"number\":0,\"arrayId\":null,\"master\":\"mymaster\",\"cluster\":\"somepokemon\"},\"user\":\"huppelde\",\"group\":\"huppelde\",\"account\":null,\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"huppelde@mymachine.mydomain.com\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"execHost\":[{\"name\":\"node2801.somepokemon.mydomain\",\"cores\":[0]}],\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"cputime\":null,\"prologue\":null,\"epilogue\":null,\"neednodes\":{\"number\":1,\"ppn\":null},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"number\":1,\"ppn\":null},\"select\":null,\"qos\":null,\"other\":null,\"feature\":null,\"host\":null,\"procs\":null,\"nodeset\":null,\"tpn\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"mppmem\":null,\"walltime\":3600},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":2644},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265,\"torqueEntryType\":\"TorqueExitEntry\"}}" + it "imfile torque abort job input" $ do+ let s = "<133>1 2017-09-14T21:13:09.181048+02:00 master15 torque - torque: 10/17/2013 02:46:14;A;86264.master15.delcatty.gent.vsc;\n" :: Text+ s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 10/17/2013 02:46:14;A;86264.master15.delcatty.gent.vsc;\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"master15\",\"torque\":{\"torqueDatestamp\":\"10/17/2013 02:46:14\",\"name\":{\"number\":86264,\"arrayId\":null,\"master\":\"master15\",\"cluster\":\"delcatty\"},\"torqueEntryType\":\"TorqueAbortEntry\"}}"+ it "snoopy with process ID" $ do- let s = "<86>1 2017-05-29T16:40:48.275334+02:00 master23 snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text- s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":86,\"syslog_version\":1,\"program\":\"snoopy\",\"@source_host\":\"master23\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}"+ let s = "<86>1 2017-05-29T16:40:48.275334+02:00 mymaster snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text+ s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":86,\"syslog_version\":1,\"program\":\"snoopy\",\"@source_host\":\"mymaster\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}" it "snoopy with process ID; only the @source_host" $ do- let s = "<86>1 2017-05-29T16:40:48.275334+02:00 master23 snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text- s ~> (parseRsyslogLogstashString $ Just [("@source_host", "hostname")]) `shouldParse` "{\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"},\"@source_host\":\"master23\"}"+ let s = "<86>1 2017-05-29T16:40:48.275334+02:00 mymaster snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text+ s ~> (parseRsyslogLogstashString $ Just [("@source_host", "hostname")]) `shouldParse` "{\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"},\"@source_host\":\"mymaster\"}"
test/HNormalise/Torque/ParserSpec.hs view
@@ -38,14 +38,14 @@ module HNormalise.Torque.ParserSpec (main, spec) where ---------------------------------------------------------------------------------import Data.Text (Text)-import qualified Data.Text.Read as TR+import Data.Text (Text)+import qualified Data.Text.Read as TR import Test.Hspec import Test.Hspec.Attoparsec ---------------------------------------------------------------------------------import HNormalise.Torque.Parser import HNormalise.Torque.Internal+import HNormalise.Torque.Parser -------------------------------------------------------------------------------- main :: IO ()@@ -56,20 +56,20 @@ spec = do describe "parseTorqueWalltime" $ do it "parse walltime given as SS" $ do- let s = ("1234567" :: Text)+ let s = "1234567" :: Text Right (s', _) = TR.decimal s s ~> parseTorqueWalltime `shouldParse` TorqueWalltime { days = 0, hours = 0, minutes = 0, seconds = s' } it "parse walltime given as MM:SS" $ do- let s = ("12:13") :: Text+ let s = "12:13" :: Text s ~> parseTorqueWalltime `shouldParse` TorqueWalltime { days = 0, hours = 0, minutes = 12, seconds = 13} it "parse walltime given as HH:MM:SS" $ do- let s = ("11:12:13") :: Text+ let s = "11:12:13" :: Text s ~> parseTorqueWalltime `shouldParse` TorqueWalltime { days = 0, hours = 11, minutes = 12, seconds = 13 } it "parse walltime given as MM:SS" $ do- let s = ("10:11:12:13") :: Text+ let s = "10:11:12:13" :: Text s ~> parseTorqueWalltime `shouldParse` TorqueWalltime { days = 10, hours = 11, minutes = 12, seconds = 13 } describe "parseTorqueMemory" $ do@@ -132,11 +132,11 @@ describe "parseTorqueJobName" $ do it "parse regular torque job name" $ do let s = "123456789.master.mycluster.mydomain;" :: Text- s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456789, array_id = Nothing, master = "master", cluster = "mycluster" }+ s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456789, arrayId = Nothing, master = "master", cluster = "mycluster" } it "parse array torque job name" $ do let s = "123456[789].master.mycluster.mydomain;" :: Text- s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456, array_id = Just 789, master = "master", cluster = "mycluster" }+ s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456, arrayId = Just 789, master = "master", cluster = "mycluster" } describe "parseTorqueResourceRequest" $ do it "parse mandatory fields in expected order" $ do@@ -146,15 +146,25 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing- , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1- , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing , pmem = Nothing , vmem = Nothing , pvmem = Nothing+ , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } @@ -165,15 +175,25 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing- , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1- , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing , pmem = Nothing , vmem = Nothing , pvmem = Nothing+ , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } @@ -184,15 +204,25 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing- , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1- , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing , pmem = Just $ 200 * 1024 , vmem = Just $ 1 * 1024 * 1024 , pvmem = Just $ 400 * 1024+ , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } @@ -203,15 +233,25 @@ , advres = Just "myreservation.1" , naccesspolicy = Nothing , ncpus = Nothing- , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1- , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing , pmem = Nothing , vmem = Nothing , pvmem = Nothing+ , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } @@ -222,18 +262,113 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing- , neednodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , nice = Nothing , nodeCount = 1- , nodes = TSN $ TorqueJobShortNode { number = 1, ppn = Just 1 }+ , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 1 } , select = Nothing , qos = Just "someqos"+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing , pmem = Nothing , vmem = Nothing , pvmem = Nothing+ , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } } + it "parse 2014 resource List" $ do+ let s = "Resource_List.neednodes=1:ppn=16 Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1:ppn=16 Resource_List.vmem=74737mb Resource_List.walltime=05:00:00" :: Text+ s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest+ { mem = Nothing+ , advres = Nothing+ , naccesspolicy = Nothing+ , ncpus = Nothing+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode { number = 1, ppn = Just 16 }+ , nice = Just 0+ , nodeCount = 1+ , nodes = TSN TorqueJobShortNode { number = 1, ppn = Just 16 }+ , select = Nothing+ , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing+ , pmem = Nothing+ , vmem = Just $ 74737 * 1024 * 1024+ , pvmem = Nothing+ , mppmem = Nothing+ , walltime = TorqueWalltime { days = 0, hours = 5, minutes = 0, seconds = 0 }+ }+ it "parse resource list with FQDN node and no ppn specified" $ do+ let s = "Resource_List.neednodes=somenode.somecluster.somedomain Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.walltime=01:00:00" :: Text+ s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest+ { mem = Nothing+ , advres = Nothing+ , naccesspolicy = Nothing+ , ncpus = Nothing+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TFN [TorqueJobFQNode {name = "somenode.somecluster.somedomain", ppn = Nothing}]+ , nice = Just 0+ , nodeCount = 1+ , nodes = TSN (TorqueJobShortNode {number = 1, ppn = Nothing})+ , select = Nothing+ , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing+ , pmem = Nothing+ , vmem = Nothing+ , pvmem = Nothing+ , mppmem = Nothing+ , walltime = TorqueWalltime {days = 0, hours = 1, minutes = 0, seconds = 0}+ }+ it "parse resource list without neednodes specified" $ do+ let s = "Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1:ppn=8 Resource_List.walltime=03:00:00" :: Text+ s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest+ { mem = Nothing+ , advres = Nothing+ , naccesspolicy = Nothing+ , ncpus = Nothing+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Nothing+ , nice = Just 0+ , nodeCount = 1+ , nodes = TSN (TorqueJobShortNode {number = 1, ppn = Just 8})+ , select = Nothing+ , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing+ , pmem = Nothing+ , vmem = Nothing+ , pvmem = Nothing+ , mppmem = Nothing+ , walltime = TorqueWalltime {days = 0, hours = 3, minutes = 0, seconds = 0}+ }+ describe "parseTorqueHostList" $ do it "parse comma separated list of single cores" $ do let s = "exec_host=node1001.mycluster.mydomain/1,3,5,7" :: Text@@ -267,16 +402,18 @@ ] describe "parseTorqueExit" $ do- it "parse job exit log line" $ do- let s = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text- s ~> parseTorqueExit `shouldParse` ("torque", TorqueJobExit- { name = TorqueJobName { number = 45, array_id = Nothing, master = "master23", cluster = "banette" }+ it "parse torque 6.0 job exit log line" $ do+ let s = "torque: 04/05/2017 13:06:53;E;45.mymaster.somecluster.somedomain;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@submitnode01.submitnode.somedomain exec_host=node2801.somecluster.somedomain/0-1+node2803.somecluster.somedomain/0-1 Resource_List.nodes=node2801.somecluster.somedomain:ppn=2+node2803.somecluster.somedomain:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.somecluster.somedomain:ppn=2+node2803.somecluster.somedomain:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text+ s ~> parseTorqueExit `shouldParse` ("torque", TorqueExit TorqueJobExit+ { torqueDatestamp = "04/05/2017 13:06:53"+ , name = TorqueJobName { number = 45, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } , user = "vsc40075" , group = "vsc40075"+ , account = Nothing , jobname = "STDIN" , queue = "short" , startCount = Nothing- , owner = "vsc40075@gligar01.gligar.gent.vsc"+ , owner = "vsc40075@submitnode01.submitnode.somedomain" , session = 15273 , times = TorqueJobTime { ctime = 1491390300@@ -287,11 +424,11 @@ } , execHost = [ TorqueExecHost- { name = "node2801.banette.gent.vsc"+ { name = "node2801.somecluster.somedomain" , cores = [0,1] } , TorqueExecHost- { name = "node2803.banette.gent.vsc"+ { name = "node2803.somecluster.somedomain" , cores = [0,1] } ]@@ -300,38 +437,48 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing- , neednodes = TFN+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TFN [ TorqueJobFQNode- { name = "node2801.banette.gent.vsc"- , ppn = 2+ { name = "node2801.somecluster.somedomain"+ , ppn = Just 2 } , TorqueJobFQNode- { name = "node2803.banette.gent.vsc"- , ppn = 2+ { name = "node2803.somecluster.somedomain"+ , ppn = Just 2 } ] , nice = Just 0 , nodeCount = 2 , nodes = TFN [ TorqueJobFQNode- { name = "node2801.banette.gent.vsc"- , ppn = 2+ { name = "node2801.somecluster.somedomain"+ , ppn = Just 2 } , TorqueJobFQNode- { name = "node2803.banette.gent.vsc"- , ppn = 2+ { name = "node2803.somecluster.somedomain"+ , ppn = Just 2 } ] , select = Nothing , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing , vmem = Just $ 1 * 1024 * 1024 * 1024 , pmem = Nothing , pvmem = Nothing+ , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0} } , resourceUsage = TorqueResourceUsage { cputime = 0- , energy = 0+ , energy = Just 0 , mem = 55048 * 1024 , vmem = 92488 * 1024 , walltime = TorqueWalltime { days = 0, hours = 0, minutes = 1, seconds = 44 }@@ -339,34 +486,265 @@ , totalExecutionSlots = 4 , uniqueNodeCount = 2 , exitStatus = 0+ , torqueEntryType = TorqueExitEntry })+ it "parse 2014 torque job exit line" $ do+ let s = "torque: 01/12/2014 23:57:07;E;161299[389].mymaster.somecluster.somedomain;user=vsc40909 group=vsc40909 jobname=30by40XconChoicesResults-389 queue=short ctime=1389546423 qtime=1389546423 etime=1389546423 start=1389567229 owner=vsc40909@submitnode02.submitnode.somedomain exec_host=node2135.somecluster.somedomain/0+node2135.somecluster.somedomain/1+node2135.somecluster.somedomain/2+node2135.somecluster.somedomain/3+node2135.somecluster.somedomain/4+node2135.somecluster.somedomain/5+node2135.somecluster.somedomain/6+node2135.somecluster.somedomain/7+node2135.somecluster.somedomain/8+node2135.somecluster.somedomain/9+node2135.somecluster.somedomain/10+node2135.somecluster.somedomain/11+node2135.somecluster.somedomain/12+node2135.somecluster.somedomain/13+node2135.somecluster.somedomain/14+node2135.somecluster.somedomain/15 Resource_List.neednodes=1:ppn=16 Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1:ppn=16 Resource_List.vmem=74737mb Resource_List.walltime=05:00:00 session=32698 end=1389567427 Exit_status=0 resources_used.cput=00:48:40 resources_used.mem=307504kb resources_used.vmem=1985904kb resources_used.walltime=00:03:21" :: Text+ s ~> parseTorqueExit `shouldParse` ("torque", TorqueExit TorqueJobExit+ { torqueDatestamp = "01/12/2014 23:57:07"+ , name = TorqueJobName { number = 161299, arrayId = Just 389, master = "mymaster", cluster = "somecluster" }+ , user = "vsc40909"+ , group = "vsc40909"+ , account = Nothing+ , jobname = "30by40XconChoicesResults-389"+ , queue = "short"+ , startCount = Nothing+ , owner = "vsc40909@submitnode02.submitnode.somedomain"+ , session = 32698+ , times = TorqueJobTime+ { ctime = 1389546423+ , qtime = 1389546423+ , etime = 1389546423+ , startTime = 1389567229+ , endTime = Just 1389567427+ }+ , execHost =+ [ TorqueExecHost+ { name = "node2135.somecluster.somedomain"+ , cores = [0 .. 15]+ }+ ]+ , resourceRequest = TorqueResourceRequest+ { mem = Nothing+ , advres = Nothing+ , naccesspolicy = Nothing+ , ncpus = Nothing+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode+ { number = 1+ , ppn = Just 16+ }+ , nice = Just 0+ , nodeCount = 1+ , nodes = TSN TorqueJobShortNode+ { number = 1+ , ppn = Just 16+ }+ , select = Nothing+ , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing+ , pmem = Nothing+ , vmem = Just 78367424512+ , pvmem = Nothing+ , mppmem = Nothing+ , walltime = TorqueWalltime { days = 0, hours = 5, minutes = 0, seconds = 0}+ }+ , resourceUsage = TorqueResourceUsage+ { cputime = 2920+ , energy = Nothing+ , mem = 314884096+ , vmem = 2033565696+ , walltime = TorqueWalltime { days = 0, hours = 0, minutes = 3, seconds = 21}+ }+ , totalExecutionSlots = 16+ , uniqueNodeCount = 1+ , exitStatus = 0+ , torqueEntryType = TorqueExitEntry+ }) + it "parse torque job exit line with cput resource request" $ do+ let s = "torque: 07/22/2014 11:00:03;E;621344.master15.delcatty.gent.vsc;user=vsc40035 group=vsc40035 jobname=NB03N queue=long ctime=1406019524 qtime=1406019524 etime=1406019524 start=1406019532 owner=vsc40035@gligar03.gligar.gent.vsc exec_host=node2142.delcatty.gent.vsc/0+node2142.delcatty.gent.vsc/1+node2142.delcatty.gent.vsc/2+node2142.delcatty.gent.vsc/3+node2142.delcatty.gent.vsc/4+node2142.delcatty.gent.vsc/5+node2142.delcatty.gent.vsc/6+node2142.delcatty.gent.vsc/7+node2142.delcatty.gent.vsc/8+node2142.delcatty.gent.vsc/9+node2142.delcatty.gent.vsc/10+node2142.delcatty.gent.vsc/11+node2142.delcatty.gent.vsc/12+node2142.delcatty.gent.vsc/13+node2142.delcatty.gent.vsc/14+node2142.delcatty.gent.vsc/15 Resource_List.cput=72:00:00 Resource_List.neednodes=1:ppn=16 Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1:ppn=16 Resource_List.vmem=74737mb Resource_List.walltime=72:00:00 session=117962 end=1406019603 Exit_status=271 resources_used.cput=00:00:25 resources_used.mem=5316kb resources_used.vmem=78756kb resources_used.walltime=00:01:14" :: Text+ s ~> parseTorqueExit `shouldParse` ("torque", TorqueExit TorqueJobExit+ { torqueDatestamp = "07/22/2014 11:00:03"+ , name = TorqueJobName {number = 621344, arrayId = Nothing, master = "master15", cluster = "delcatty"}+ , user = "vsc40035"+ , group = "vsc40035"+ , account = Nothing+ , jobname = "NB03N"+ , queue = "long"+ , startCount = Nothing+ , owner = "vsc40035@gligar03.gligar.gent.vsc"+ , session = 117962+ , times = TorqueJobTime+ { ctime = 1406019524+ , qtime = 1406019524+ , etime = 1406019524+ , startTime = 1406019532+ , endTime = Just 1406019603+ }+ , execHost = [TorqueExecHost {name = "node2142.delcatty.gent.vsc", cores = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15]}]+ , resourceRequest = TorqueResourceRequest+ { mem = Nothing+ , advres = Nothing+ , naccesspolicy = Nothing+ , ncpus = Nothing+ , cputime = Just TorqueWalltime {days = 0, hours = 72, minutes = 0, seconds = 0}+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode {number = 1, ppn = Just 16}+ , nice = Just 0+ , nodeCount = 1+ , nodes = TSN TorqueJobShortNode {number = 1, ppn = Just 16}+ , select = Nothing+ , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing+ , pmem = Nothing+ , vmem = Just 78367424512+ , pvmem = Nothing+ , mppmem = Nothing+ , walltime = TorqueWalltime {days = 0, hours = 72, minutes = 0, seconds = 0}+ }+ , resourceUsage = TorqueResourceUsage+ { cputime = 25+ , energy = Nothing+ , mem = 5443584+ , vmem = 80646144+ , walltime = TorqueWalltime {days = 0, hours = 0, minutes = 1, seconds = 14}+ }+ , totalExecutionSlots = 16+ , uniqueNodeCount = 1+ , exitStatus = 271+ , torqueEntryType = TorqueExitEntry+ })++ it "parse torque job exit with account field" $ do+ let s = "torque: 08/03/2017 05:07:22;E;268279.master21.swalot.gent.vsc;user=vsc41771 group=vsc41771 account=lt1_2017-43 jobname=/user/scratch/gent/gvo000/gvo00003/vsc41771/amsterdam/restrained_md/test_withoutplumed queue=short ctime=1501686015 qtime=1501686015 etime=1501686015 start=1501686467 owner=vsc41771@gligar01.gligar.gent.vsc exec_host=node2612.swalot.gent.vsc/0-19+node2681.swalot.gent.vsc/0-19 Resource_List.neednodes=2:ppn=20 Resource_List.nice=0 Resource_List.nodect=2 Resource_List.nodes=2:ppn=20 Resource_List.vmem=143425316860b Resource_List.walltime=11:59:00 session=7473 total_execution_slots=40 unique_node_count=2 end=1501729642 Exit_status=-11 resources_used.cput=1725002 resources_used.energy_used=0 resources_used.mem=16209816kb resources_used.vmem=38821964kb resources_used.walltime=11:59:30" :: Text+ s ~> parseTorqueExit `shouldParse` ("torque", TorqueExit TorqueJobExit+ { torqueDatestamp = "08/03/2017 05:07:22"+ , name = TorqueJobName {number = 268279, arrayId = Nothing, master = "master21", cluster = "swalot"}+ , user = "vsc41771"+ , group = "vsc41771"+ , account = Just "lt1_2017-43"+ , jobname = "/user/scratch/gent/gvo000/gvo00003/vsc41771/amsterdam/restrained_md/test_withoutplumed"+ , queue = "short"+ , startCount = Nothing+ , owner = "vsc41771@gligar01.gligar.gent.vsc"+ , session = 7473+ , times = TorqueJobTime {ctime = 1501686015, qtime = 1501686015, etime = 1501686015, startTime = 1501686467, endTime = Just 1501729642}+ , execHost =+ [ TorqueExecHost {name = "node2612.swalot.gent.vsc", cores = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]}+ , TorqueExecHost {name = "node2681.swalot.gent.vsc", cores = [0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19]}+ ]+ , resourceRequest = TorqueResourceRequest+ { mem = Nothing+ , advres = Nothing+ , naccesspolicy = Nothing+ , ncpus = Nothing+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN TorqueJobShortNode {number = 2, ppn = Just 20}+ , nice = Just 0+ , nodeCount = 2+ , nodes = TSN TorqueJobShortNode {number = 2, ppn = Just 20}+ , select = Nothing+ , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing+ , pmem = Nothing+ , vmem = Just 143425316860+ , pvmem = Nothing+ , mppmem = Nothing+ , walltime = TorqueWalltime {days = 0, hours = 11, minutes = 59, seconds = 0}+ }+ , resourceUsage = TorqueResourceUsage+ { cputime = 1725002+ , energy = Just 0+ , mem = 16598851584+ , vmem = 39753691136+ , walltime = TorqueWalltime {days = 0, hours = 11, minutes = 59, seconds = 30}+ }+ , totalExecutionSlots = 40+ , uniqueNodeCount = 2+ , exitStatus = -11+ , torqueEntryType = TorqueExitEntry+ })+ describe "parseTorqueQueue" $ do it "parse job queue entry" $ do- let s = "06/28/2017 14:31:09;Q;80.master23.banette.gent.vsc;queue=default" :: Text- s ~> parseTorqueQueue `shouldParse` ("torque", TorqueJobQueue- { name = TorqueJobName { number = 80, array_id = Nothing, master = "master23", cluster = "banette" }+ let s = "torque: 06/28/2017 14:31:09;Q;80.mymaster.somecluster.somedomain;queue=default" :: Text+ s ~> parseTorqueQueue `shouldParse` ("torque", TorqueQueue TorqueJobQueue+ { torqueDatestamp = "06/28/2017 14:31:09"+ , name = TorqueJobName { number = 80, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } , queue = "default"+ , torqueEntryType = TorqueQueueEntry }) - describe "parseTorqueDelete" $ do+ it "parse job queue entry master24 - torque 6.0" $ do+ let s = "torque: 07/27/2017 14:17:41;Q;5.master24.somecluster.somedomain;queue=default" :: Text+ s ~> parseTorqueQueue `shouldParse` ("torque", TorqueQueue TorqueJobQueue+ { torqueDatestamp = "07/27/2017 14:17:41"+ , name = TorqueJobName { number = 5, arrayId = Nothing, master = "master24" , cluster = "somecluster" }+ , queue = "default"+ , torqueEntryType = TorqueQueueEntry+ })++ it "parse job queue entry - torque 4.x" $ do+ let s = "torque: 12/31/2014 15:51:48;Q;1166970[].somemaster.somecluster.gent.vsc;queue=long" :: Text+ s ~> parseTorqueQueue `shouldParse`("torque", TorqueQueue TorqueJobQueue+ { torqueDatestamp = "12/31/2014 15:51:48"+ , name = TorqueJobName { number = 1166970, arrayId = Nothing, master = "somemaster", cluster = "somecluster"}+ , queue = "long"+ , torqueEntryType = TorqueQueueEntry+ })++ describe "parseTorqueDelete" $ it "parse job delete entry" $ do- let s = "06/28/2017 15:44:02;D;81.master23.banette.gent.vsc;requestor=vsc40075@gligar02.gligar.gent.vsc" :: Text- s ~> parseTorqueDelete `shouldParse` ("torque", TorqueJobDelete- { name = TorqueJobName { number = 81, array_id = Nothing, master = "master23", cluster = "banette" }- , requestor = TorqueRequestor { user = "vsc40075", whence = "gligar02.gligar.gent.vsc" }+ let s = "torque: 06/28/2017 15:44:02;D;81.mymaster.somecluster.somedomain;requestor=vsc40075@submitnode02.submitnode.somedomain" :: Text+ s ~> parseTorqueDelete `shouldParse` ("torque", TorqueDelete TorqueJobDelete+ { torqueDatestamp = "06/28/2017 15:44:02"+ , name = TorqueJobName { number = 81, arrayId = Nothing, master = "mymaster", cluster = "somecluster" }+ , requestor = TorqueRequestor { user = "vsc40075", whence = "submitnode02.submitnode.somedomain" }+ , torqueEntryType = TorqueDeleteEntry }) + describe "parseTorqueAbort" $+ it "parse job abort entry" $ do+ let s = "torque: 09/02/2013 17:34:26;A;34106.mymaster.somecluster.somedomain;" :: Text+ s ~> parseTorqueAbort `shouldParse` ("torque", TorqueAbort TorqueJobAbort+ { torqueDatestamp = "09/02/2013 17:34:26"+ , name = TorqueJobName { number = 34106, arrayId = Nothing, master = "mymaster", cluster = "somecluster" }+ , torqueEntryType = TorqueAbortEntry+ })++ describe "parseTorqueRerun" $+ it "parse job rerun entry" $ do+ let s = "torque: 09/02/2013 17:34:26;R;34106.mymaster.somecluster.somedomain;" :: Text+ s ~> parseTorqueRerun `shouldParse` ("torque", TorqueRerun TorqueJobRerun+ { torqueDatestamp = "09/02/2013 17:34:26"+ , name = TorqueJobName { number = 34106, arrayId = Nothing, master = "mymaster", cluster = "somecluster" }+ , torqueEntryType = TorqueRerunEntry+ })++ describe "parseTorqueStart" $ do it "parse job start" $ do- let s = "06/20/2017 11:24:49;S;63.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1497950675 qtime=1497950675 etime=1497950675 start=1497950689 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0 Resource_List.vmem=4224531456b Resource_List.nodes=1:ppn=1 Resource_List.walltime=00:10:00 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1 Resource_List.nice=0" :: Text- s ~> parseTorqueStart `shouldParse` ("torque", TorqueJobStart- { name = TorqueJobName { number = 63, array_id = Nothing, master = "master23", cluster = "banette" }+ let s = "torque: 06/20/2017 11:24:49;S;63.mymaster.somecluster.somedomain;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1497950675 qtime=1497950675 etime=1497950675 start=1497950689 owner=vsc40075@submitnode01.submitnode.somedomain exec_host=node2801.somecluster.somedomain/0 Resource_List.vmem=4224531456b Resource_List.nodes=1:ppn=1 Resource_List.walltime=00:10:00 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1 Resource_List.nice=0" :: Text+ s ~> parseTorqueStart `shouldParse` ("torque", TorqueStart TorqueJobStart+ { torqueDatestamp = "06/20/2017 11:24:49"+ , name = TorqueJobName { number = 63, arrayId = Nothing, master = "mymaster", cluster = "somecluster" } , user = "vsc40075" , group = "vsc40075"+ , account = Nothing , jobname = "STDIN" , queue = "short"- , owner = "vsc40075@gligar01.gligar.gent.vsc"+ , owner = "vsc40075@submitnode01.submitnode.somedomain" , times = TorqueJobTime { ctime = 1497950675 , qtime = 1497950675@@ -376,7 +754,7 @@ } , execHost = [ TorqueExecHost- { name = "node2801.banette.gent.vsc"+ { name = "node2801.somecluster.somedomain" , cores = [0] } ]@@ -385,25 +763,75 @@ , advres = Nothing , naccesspolicy = Nothing , ncpus = Nothing- , neednodes = TSN- ( TorqueJobShortNode+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TSN+ TorqueJobShortNode { number = 1 , ppn = Just 1 }- ) , nice = Just 0 , nodeCount = 1 , nodes = TSN- ( TorqueJobShortNode+ TorqueJobShortNode { number = 1 , ppn = Just 1 }- ) , select = Nothing , qos = Nothing- , vmem = Just $ 4224531456+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing+ , vmem = Just 4224531456 , pmem = Nothing , pvmem = Nothing+ , mppmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 0, minutes = 10, seconds = 0} }+ , torqueEntryType = TorqueStartEntry })+ it "parse job start - torque legacy 2009" $ do+ let s = "torque: 02/23/2009 11:48:35;S;102355.master.cvos.cluster;user=vsc40014 group=vsc40014 jobname=MtChr5_9036000_rmwrap.sh queue=short_eth ctime=1235384686 qtime=1235384686 etime=1235384686 start=1235386115 owner=vsc40014@gengar1.cvos.cluster exec_host=node047.cvos.cluster/4 Resource_List.neednodes=node047.cvos.cluster Resource_List.nice=0 Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.walltime=01:00:00" :: Text+ s ~> parseTorqueStart `shouldParse` ("torque", TorqueStart TorqueJobStart+ { torqueDatestamp = "02/23/2009 11:48:35"+ , name = TorqueJobName {number = 102355, arrayId = Nothing, master = "master", cluster = "cvos"}+ , user = "vsc40014"+ , group = "vsc40014"+ , account = Nothing+ , jobname = "MtChr5_9036000_rmwrap.sh"+ , queue = "short_eth"+ , owner = "vsc40014@gengar1.cvos.cluster"+ , times = TorqueJobTime {ctime = 1235384686, qtime = 1235384686, etime = 1235384686, startTime = 1235386115, endTime = Nothing}+ , execHost = [TorqueExecHost {name = "node047.cvos.cluster", cores = [4]}]+ , resourceRequest = TorqueResourceRequest+ { mem = Nothing+ , advres = Nothing+ , naccesspolicy = Nothing+ , ncpus = Nothing+ , cputime = Nothing+ , prologue = Nothing+ , epilogue = Nothing+ , neednodes = Just $ TFN [TorqueJobFQNode {name = "node047.cvos.cluster", ppn = Nothing}]+ , nice = Just 0+ , nodeCount = 1+ , nodes = TSN (TorqueJobShortNode {number = 1, ppn = Nothing})+ , select = Nothing+ , qos = Nothing+ , other = Nothing+ , feature = Nothing+ , host = Nothing+ , procs = Nothing+ , nodeset = Nothing+ , tpn = Nothing+ , pmem = Nothing+ , vmem = Nothing+ , pvmem = Nothing+ , mppmem = Nothing+ , walltime = TorqueWalltime {days = 0, hours = 1, minutes = 0, seconds = 0}+ }+ , torqueEntryType = TorqueStartEntry+ })