hnormalise 0.4.8.0 → 0.5.0.0
raw patch · 38 files changed
+456/−200 lines, 38 filesdep +deepseqdep +extradep +stm-conduitsetup-changed
Dependencies added: deepseq, extra, stm-conduit
Files
- 0mq/Main.hs +17/−1
- Setup.hs +1/−1
- app/Main.hs +97/−78
- hnormalise.cabal +21/−11
- src/HNormalise.hs +11/−12
- src/HNormalise/Common/Internal.hs +13/−6
- src/HNormalise/Common/Json.hs +1/−9
- src/HNormalise/Common/Parser.hs +3/−3
- src/HNormalise/Communication/Input/ZeroMQ.hs +63/−0
- src/HNormalise/Communication/Output/File.hs +65/−0
- src/HNormalise/Config.hs +1/−1
- src/HNormalise/Huppel/Internal.hs +1/−1
- src/HNormalise/Huppel/Json.hs +1/−1
- src/HNormalise/Huppel/Parser.hs +1/−1
- src/HNormalise/Internal.hs +16/−14
- src/HNormalise/Json.hs +8/−4
- src/HNormalise/Lmod/Internal.hs +21/−4
- src/HNormalise/Lmod/Json.hs +8/−1
- src/HNormalise/Lmod/Parser.hs +15/−3
- src/HNormalise/Parser.hs +19/−18
- src/HNormalise/Shorewall/Internal.hs +8/−4
- src/HNormalise/Shorewall/Json.hs +1/−1
- src/HNormalise/Shorewall/Parser.hs +1/−1
- src/HNormalise/Snoopy/Internal.hs +6/−5
- src/HNormalise/Snoopy/Json.hs +1/−1
- src/HNormalise/Snoopy/Parser.hs +1/−1
- src/HNormalise/Torque/Internal.hs +21/−1
- src/HNormalise/Torque/Json.hs +1/−1
- src/HNormalise/Torque/Parser.hs +1/−1
- src/HNormalise/Verbose.hs +1/−1
- test/Bench.hs +1/−1
- test/HNormalise/Common/ParserSpec.hs +1/−1
- test/HNormalise/Lmod/ParserSpec.hs +14/−2
- test/HNormalise/ParserSpec.hs +11/−6
- test/HNormalise/Shorewall/ParserSpec.hs +1/−1
- test/HNormalise/Snoopy/ParserSpec.hs +1/−1
- test/HNormalise/Torque/ParserSpec.hs +1/−1
- test/Spec.hs +1/−1
0mq/Main.hs view
@@ -195,13 +195,29 @@ (liftIO (ZMQ.receive s) >>= yield) liftIO $ putStrLn "Done!" ++encodingConduit = loop+ where+ loop = do+ v <- await+ case v of+ Just v -> do+ Data.Conduit.yield (enc v)+ loop+ Nothing -> return ()++enc v = case v of+ Left l -> Original l+ Right n -> Transformed $ encodeNormalisedRsyslog n++ -------------------------------------------------------------------------------- normalisationConduit options config = let fs = fields config in if oJsonInput options then CB.lines $= C.map (normaliseJsonInput fs) --else CB.lines $= conduitPooledMapBuffered 20 (normaliseText fs)- else CB.lines $= C.map (normaliseText fs)+ else CB.lines $= C.map (normaliseText fs) $= encodingConduit -------------------------------------------------------------------------------- {- runZeroMQConnection :: Main.Options
Setup.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
app/Main.hs view
@@ -1,16 +1,57 @@+{- hnormalise - a log normalisation library+ -+ - Copyright Ghent University (c) 2017+ -+ - All rights reserved.+ -+ - Redistribution and use in source and binary forms, with or without+ - modification, are permitted provided that the following conditions are met:+ -+ - * Redistributions of source code must retain the above copyright+ - notice, this list of conditions and the following disclaimer.+ -+ - * Redistributions in binary form must reproduce the above+ - copyright notice, this list of conditions and the following+ - disclaimer in the documentation and/or other materials provided+ - with the distribution.+ -+ - * Neither the name of Author name here nor the names of other+ - contributors may be used to endorse or promote products derived+ - from this software without specific prior written permission.+ -+ - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+-}+ {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RankNTypes #-} module Main where -------------------------------------------------------------------------------- import Control.Applicative ((<$>), (<*>))+import Control.Concurrent.Chan+import Control.Concurrent.Extra import Control.Concurrent.MVar (modifyMVar_, newMVar, newEmptyMVar, putMVar, readMVar, tryTakeMVar, withMVar, takeMVar, MVar)+import Control.Concurrent.QSem+import Control.DeepSeq+import Control.Exception (evaluate) import Control.Monad+import Control.Monad.Extra import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Loops (whileM_, untilM_) import Control.Monad.Trans (lift)-import Control.Monad.Trans.Resource (runResourceT)+import Control.Monad.Trans.Resource (runResourceT, ResourceT) import Data.Aeson import Data.Attoparsec.Text import qualified Data.ByteString.Char8 as SBS@@ -22,7 +63,7 @@ import Data.Conduit.Network import qualified Data.Conduit.Text as DCT import qualified Data.Conduit.ZMQ4 as ZMQC-import Data.Maybe (fromJust, fromMaybe)+import Data.Maybe (fromJust, isJust, fromMaybe) import Data.Monoid ((<>)) import qualified Data.Text as T import qualified Data.Text.Encoding as TE@@ -36,9 +77,9 @@ import qualified System.ZMQ4 as ZMQ (withContext, withSocket, bind, send, receive, connect, Push(..), Pull(..)) import Text.Printf (printf) -import Debug.Trace -------------------------------------------------------------------------------- import HNormalise+import HNormalise.Communication.Input.ZeroMQ (zmqInterruptibleSource) import HNormalise.Config ( Config (..) , ConnectionType(..) , InputConfig(..)@@ -116,18 +157,20 @@ loop = do v <- await case v of- Just (Transformed json) -> do- yield json $$ success- yield (SBS.pack "\n") $$ success- liftIO $ increaseCount (1, 0) messageCount frequency- loop- Just (Original l) -> do- yield l $$ failure- yield (SBS.pack "\n") $$ failure- liftIO $ increaseCount (0, 1) messageCount frequency- loop+ Just n -> case n of+ Right json -> do+ Data.Conduit.yield (encodeNormalisedRsyslog json) $$ success+ Data.Conduit.yield (SBS.pack "\n") $$ success+ liftIO $ increaseCount (1, 0) messageCount frequency+ loop+ Left l -> do+ Data.Conduit.yield l $$ failure+ Data.Conduit.yield (SBS.pack "\n") $$ failure+ liftIO $ increaseCount (0, 1) messageCount frequency+ loop Nothing -> return () +-------------------------------------------------------------------------------- increaseCount (s, f) messageCount frequency = do (s', f') <- takeMVar messageCount when ((s' + f') `mod` frequency == 0) $ do@@ -136,70 +179,44 @@ putMVar messageCount (s' + s, f' + f) ----------------------------------------------------------------------------------- | 'mySink' yields the results downstream with the addition of a string mentioning success or failure--- for testing purposes-mySink messageCount frequency = loop- where- loop = do- v <- await- case v of- Just (Transformed json) -> do- yield (SBS.pack "success: ")- yield json- yield (SBS.pack "\n")- -- liftIO $ increaseCount (1, 0) messageCount frequency- loop- Just (Original l) -> do- yield (SBS.pack "fail - original: ")- yield l- yield (SBS.pack "\n")- --liftIO $ increaseCount (0, 1) messageCount frequency- loop- Nothing -> return ()-----------------------------------------------------------------------------------runTCPConnection options config messageCount = do- let listenHost = fromJust $ input config >>= (\(InputConfig t _) -> t) >>= (\(TcpPortConfig h p) -> h)- let listenPort = fromJust $ input config >>= (\(InputConfig t _) -> t) >>= (\(TcpPortConfig h p) -> p)- let frequency = fromMaybe defaultLoggingFrequency (logging config >>= (\(LoggingConfig f) -> f))- runTCPServer (serverSettings listenPort "*") $ \appData -> do- let fs = fields config- case oTestFilePath options of- Nothing -> do- let successHost = TE.encodeUtf8 $ fromJust $ output config >>= (\(OutputConfig t _) -> t) >>= (\(TcpOutputConfig s f) -> s) >>= (\(TcpPortConfig h p) -> h)- let successPort = fromJust $ output config >>= (\(OutputConfig t _) -> t) >>= (\(TcpOutputConfig s f) -> s) >>= (\(TcpPortConfig h p) -> p)- let failureHost = TE.encodeUtf8 $ fromJust $ output config >>= (\(OutputConfig t _) -> t) >>= (\(TcpOutputConfig s f) -> f) >>= (\(TcpPortConfig h p) -> h)- let failurePort = fromJust $ output config >>= (\(OutputConfig t _) -> t) >>= (\(TcpOutputConfig s f) -> f) >>= (\(TcpPortConfig h p) -> p)-- runTCPClient (clientSettings successPort successHost) $ \successServer ->- runTCPClient (clientSettings failurePort failureHost) $ \failServer ->- runResourceT $ appSource appData- $= normalisationConduit options config- $$ messageSink (appSink successServer) (appSink failServer) messageCount frequency- Just testSinkFileName -> runResourceT $ appSource appData- $= normalisationConduit options config- $= mySink messageCount frequency- $$ sinkFile testSinkFileName---- | 'zmqInterruptibleSource' converts a regular 0mq recieve operation on a socket into a conduit source--- The source is halted when something is put into the MVar, effectively stopping the program from checking--- for new incoming messages.-zmqInterruptibleSource m s = do- whileM_- (liftIO $ do- val <- tryTakeMVar m- case val of- Just _ -> return False- Nothing -> return True)- (liftIO (ZMQ.receive s) >>= yield)- liftIO $ putStrLn "Done!"+-- The following code was adapted from https://github.com/ndmitchell/hoogle+pipelineC :: Int -> Consumer o (ResourceT IO) r -> Consumer o (ResourceT IO) r+pipelineC buffer sink = do+ sem <- liftIO $ newQSem buffer -- how many are in flow, to avoid memory leaks+ chan <- liftIO newChan -- the items in flow (type o)+ bar <- liftIO newBarrier -- the result type (type r)+ me <- liftIO myThreadId+ liftIO $ printf "normalising thread has tid %s\n" (show me)+ liftIO $ flip forkFinally (either (throwTo me) (signalBarrier bar)) $ do+ t <- myThreadId+ printf "sinking thread has tid %s\n" (show t)+ runResourceT . runConduit $+ whileM (do+ x <- liftIO $ readChan chan+ liftIO $ signalQSem sem+ whenJust x Data.Conduit.yield+ return $ isJust x) =$=+ sink+ awaitForever $ \x -> liftIO $ do+ waitQSem sem+ writeChan chan $ Just x+ liftIO $ writeChan chan Nothing+ liftIO $ waitBarrier bar -------------------------------------------------------------------------------- normalisationConduit options config = let fs = fields config in if oJsonInput options- then CB.lines $= C.map (normaliseJsonInput fs)- else CB.lines $= C.map (normaliseText fs)+ then undefined -- CB.lines $= C.map (normaliseJsonInput fs)+ else CB.lines $= normalise fs -- $= C.map (normaliseText fs)+ where normalise fs = do+ m <- await+ case m of+ Just m' -> do+ m'' <- liftIO $ evaluate $ force (normaliseText fs) m'+ Data.Conduit.yield m''+ normalise fs+ Nothing -> return () -------------------------------------------------------------------------------- runZeroMQConnection :: Main.Options@@ -217,7 +234,7 @@ case oTestFilePath options of Nothing -> ZMQ.withContext $ \ctx -> ZMQ.withSocket ctx ZMQ.Pull $ \s -> do- ZMQ.bind s $ printf "tcp://%s:%d" listenHost listenPort+ ZMQ.connect s $ printf "tcp://%s:%d" listenHost listenPort verbose' $ printf "Listening on tcp://%s:%d" listenHost listenPort let successHost = fromJust $ output config >>= (\(OutputConfig _ z) -> z) >>= (\(ZeroMQOutputConfig s f) -> s) >>= (\(ZeroMQPortConfig m h p) -> h)@@ -232,19 +249,21 @@ ZMQ.connect failureSocket $ printf "tcp://%s:%d" failureHost failurePort verbose' $ printf "Pushing failed parses on tcp://%s:%d" failureHost failurePort++ runResourceT $ zmqInterruptibleSource s_interrupted s $= normalisationConduit options config- $$ messageSink (ZMQC.zmqSink successSocket []) (ZMQC.zmqSink failureSocket []) messageCount frequency+ $$ pipelineC 100 (messageSink (ZMQC.zmqSink successSocket []) (ZMQC.zmqSink failureSocket []) messageCount frequency) Just testSinkFileName -> ZMQ.withContext $ \ctx -> ZMQ.withSocket ctx ZMQ.Pull $ \s -> do- ZMQ.bind s $ printf "tcp://%s:%d" listenHost listenPort+ ZMQ.connect s $ printf "tcp://%s:%d" listenHost listenPort verbose' $ printf "Listening on tcp://%s:%d" listenHost listenPort runResourceT $ zmqInterruptibleSource s_interrupted s $= normalisationConduit options config- $= mySink messageCount frequency- $$ sinkFile testSinkFileName+ $$ pipelineC 100 (let sf = sinkFile testSinkFileName in messageSink sf sf messageCount frequency)+ -------------------------------------------------------------------------------- -- | 'main' starts a TCP server, listening to incoming data and connecting to TCP servers downstream to -- for the pipeline.@@ -265,7 +284,7 @@ -- i.e., both TCP, both ZeroMQ, etc. messageCount <- newMVar ((0,0) :: (Int, Int)) case connectionType config of- TCP -> void $ runTCPConnection options config messageCount+ -- TCP -> void $ runTCPConnection options config messageCount ZeroMQ -> do verbose' "ZeroMQ connections" s_interrupted <- newEmptyMVar
hnormalise.cabal view
@@ -1,5 +1,5 @@ name: hnormalise-version: 0.4.8.0+version: 0.5.0.0 synopsis: Log message normalisation tool producing structured JSON messages description: Log message normalisation tool producing structured JSON messages homepage: https://github.com/itkovian/hnormalise#readme@@ -23,6 +23,8 @@ , HNormalise.Common.Json , HNormalise.Common.Parser , HNormalise.Common.Internal+ , HNormalise.Communication.Input.ZeroMQ+ , HNormalise.Communication.Output.File , HNormalise.Huppel.Internal , HNormalise.Huppel.Json , HNormalise.Huppel.Parser@@ -46,20 +48,24 @@ , attoparsec , attoparsec-iso8601 , bytestring+ , conduit , containers+ , deepseq , directory , ip+ , monad-loops , permute , resourcet , text , time , unordered-containers , yaml+ , zeromq4-haskell+ , zeromq4-conduit default-language: Haskell2010 --executable hnormalise-0mq- hs-source-dirs: 0mq+executable hnormalise+ hs-source-dirs: app main-is: Main.hs ghc-options: -threaded -rtsopts -with-rtsopts=-N -funbox-strict-fields -optc-O2 build-depends: base@@ -73,14 +79,15 @@ , conduit-combinators , conduit-extra , containers+ , deepseq , directory+ , extra , ip- , lifted-base , monad-loops- , monad-par , mtl , optparse-applicative , resourcet+ , stm-conduit , text , time , unix@@ -88,10 +95,14 @@ , yaml , zeromq4-haskell , zeromq4-conduit+ , lifted-base+ , monad-control+ , transformers-base default-language: Haskell2010 -executable hnormalise- hs-source-dirs: app++executable hnormalise-0mq+ hs-source-dirs: 0mq main-is: Main.hs ghc-options: -threaded -rtsopts -with-rtsopts=-N -funbox-strict-fields -optc-O2 build-depends: base@@ -107,7 +118,9 @@ , containers , directory , ip+ , lifted-base , monad-loops+ , monad-par , mtl , optparse-applicative , resourcet@@ -118,9 +131,6 @@ , yaml , zeromq4-haskell , zeromq4-conduit- , lifted-base- , monad-control- , transformers-base default-language: Haskell2010
src/HNormalise.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -42,6 +42,7 @@ -------------------------------------------------------------------------------- import Control.Applicative ((<|>))+import Control.DeepSeq import Data.Aeson (ToJSON) import qualified Data.Aeson as Aeson import Data.Attoparsec.Combinator (lookAhead, manyTill)@@ -84,19 +85,17 @@ -------------------------------------------------------------------------------- -- | The 'normaliseText' function converts a 'Text' to a normalised message or keeps the original (in 'ByteString') -- format if the conversion fails-normaliseText :: Maybe [(Text, Text)] -- ^ Output fields- -> SBS.ByteString -- ^ Input- -> Normalised -- ^ Transformed or Original result+normaliseText :: Maybe [(Text, Text)] -- ^ Output fields+ -> SBS.ByteString -- ^ Input+ -> Either SBS.ByteString NormalisedRsyslog -- ^ Transformed or Original result normaliseText fs logLine =- let !p = parse (parseRsyslogLogstashString fs) $ decodeUtf8 logLine- in case p of- Done _ r -> Transformed r+ case parse (parseRsyslogLogstashString fs) $ decodeUtf8 logLine of+ Done _ r -> Right r Partial c -> case c empty of- Done _ r -> Transformed r- _ -> original- _ -> original- where- original = Original logLine+ Done _ r -> Right r+ _ -> Left logLine+ _ -> Left logLine+ -------------------------------------------------------------------------------- -- | The 'convertMessage' function transforms the actual message to a 'Maybe' 'ParseResult'. If parsing fails,
src/HNormalise/Common/Internal.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -32,6 +32,8 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} +{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE ExistentialQuantification #-} @@ -40,14 +42,19 @@ ) where ---------------------------------------------------------------------------------import Data.Aeson (FromJSON, ToJSON, toEncoding,- toJSON)+import Control.DeepSeq (NFData, rnf)+import Data.Aeson (FromJSON, ToJSON, toEncoding, toJSON) import Data.Text-import GHC.Generics (Generic)-import qualified Net.Types as Net+import GHC.Generics (Generic)+import qualified Net.Types as Net -------------------------------------------------------------------------------- data Host = Hostname Text -- hostname | IPv4 Net.IPv4 | IPv6 Net.IPv6- deriving (Show, Eq)+ deriving (Show, Eq, Generic)++instance NFData Host+instance NFData Net.IPv4+instance NFData Net.IPv6 where+ rnf !_ = ()
src/HNormalise/Common/Json.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -32,11 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-}-{-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE OverloadedStrings #-}-- module HNormalise.Common.Json where --------------------------------------------------------------------------------@@ -47,9 +42,6 @@ import qualified Net.Types as NT -------------------------------------------------------------------------------- import HNormalise.Common.Internal--instance ToJSON NT.IPv6 where- toJSON = String . pack . show -------------------------------------------------------------------------------- instance ToJSON Host where
src/HNormalise/Common/Parser.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -43,8 +43,8 @@ import Data.Char (isSpace) import Data.Text (Text) import qualified Data.Text as T-import qualified Net.IPv4.Text as IPv4-import qualified Net.IPv6.Text as IPv6+import qualified Net.IPv4 as IPv4+import qualified Net.IPv6 as IPv6 -------------------------------------------------------------------------------- import HNormalise.Common.Internal
+ src/HNormalise/Communication/Input/ZeroMQ.hs view
@@ -0,0 +1,63 @@+{- hnormalise - a log normalisation library+ -+ - Copyright Ghent University (c) 2017+ -+ - All rights reserved.+ -+ - Redistribution and use in source and binary forms, with or without+ - modification, are permitted provided that the following conditions are met:+ -+ - * Redistributions of source code must retain the above copyright+ - notice, this list of conditions and the following disclaimer.+ -+ - * Redistributions in binary form must reproduce the above+ - copyright notice, this list of conditions and the following+ - disclaimer in the documentation and/or other materials provided+ - with the distribution.+ -+ - * Neither the name of Author name here nor the names of other+ - contributors may be used to endorse or promote products derived+ - from this software without specific prior written permission.+ -+ - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+-}++{-# LANGUAGE OverloadedStrings #-}++module HNormalise.Communication.Input.ZeroMQ+ ( zmqInterruptibleSource+ ) where+++import Control.Concurrent.MVar (modifyMVar_, newMVar, newEmptyMVar, putMVar, readMVar, tryTakeMVar, withMVar, takeMVar, MVar)+import Control.Monad.IO.Class (MonadIO, liftIO)+import Control.Monad.Loops (whileM_, untilM_)+import Data.Conduit+import qualified System.ZMQ4 as ZMQ (withContext, withSocket, bind, send, receive, connect, Push(..), Pull(..))++--------------------------------------------------------------------------------+import HNormalise.Config++--------------------------------------------------------------------------------+-- | 'zmqInterruptibleSource' converts a regular 0mq recieve operation on a socket into a conduit source+-- The source is halted when something is put into the MVar, effectively stopping the program from checking+-- for new incoming messages.+zmqInterruptibleSource m s = do+ whileM_+ (liftIO $ do+ val <- tryTakeMVar m+ case val of+ Just _ -> return False+ Nothing -> return True)+ (liftIO (ZMQ.receive s) >>= yield)+ liftIO $ putStrLn "Done!"
+ src/HNormalise/Communication/Output/File.hs view
@@ -0,0 +1,65 @@+{- hnormalise - a log normalisation library+ -+ - Copyright Ghent University (c) 2017+ -+ - All rights reserved.+ -+ - Redistribution and use in source and binary forms, with or without+ - modification, are permitted provided that the following conditions are met:+ -+ - * Redistributions of source code must retain the above copyright+ - notice, this list of conditions and the following disclaimer.+ -+ - * Redistributions in binary form must reproduce the above+ - copyright notice, this list of conditions and the following+ - disclaimer in the documentation and/or other materials provided+ - with the distribution.+ -+ - * Neither the name of Author name here nor the names of other+ - contributors may be used to endorse or promote products derived+ - from this software without specific prior written permission.+ -+ - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+ - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+ - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+ - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+ - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+ - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+ - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+ - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+ - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+ - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+-}+{-# LANGUAGE OverloadedStrings #-}++module HNormalise.Communication.Output.File where++import qualified Data.ByteString.Char8 as SBS+import qualified Data.ByteString.Lazy.Char8 as BS+import Data.Conduit++--------------------------------------------------------------------------------+import HNormalise.Json (encodeNormalisedRsyslog)++--------------------------------------------------------------------------------+-- | 'mySink' yields the results downstream with the addition of a string mentioning success or failure+-- for testing purposes+mySink messageCount frequency = loop+ where+ loop = do+ v <- await+ case v of+ Just (Right json) -> do+ yield (SBS.pack "success: ")+ yield $ encodeNormalisedRsyslog json+ yield (SBS.pack "\n")+ -- liftIO $ increaseCount (1, 0) messageCount frequency+ loop+ Just (Left l) -> do+ yield (SBS.pack "fail - original: ")+ yield l+ yield (SBS.pack "\n")+ --liftIO $ increaseCount (0, 1) messageCount frequency+ loop+ Nothing -> return ()
src/HNormalise/Config.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Huppel/Internal.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Huppel/Json.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Huppel/Parser.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Internal.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -33,12 +33,13 @@ -} {-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DuplicateRecordFields #-}-{-# LANGUAGE OverloadedStrings #-} module HNormalise.Internal where --------------------------------------------------------------------------------+import Control.DeepSeq (NFData) import Data.Aeson (FromJSON, ToJSON, toEncoding, toJSON) import Data.Text (Text)@@ -48,7 +49,7 @@ -------------------------------------------------------------------------------- import HNormalise.Huppel.Internal (Huppel) import HNormalise.Huppel.Json-import HNormalise.Lmod.Internal (LmodLoad)+import HNormalise.Lmod.Internal (LmodParseResult) import HNormalise.Lmod.Json import HNormalise.Shorewall.Internal (Shorewall) import HNormalise.Shorewall.Json@@ -59,25 +60,22 @@ -------------------------------------------------------------------------------- data ParseResult- -- | 'Huppel' Result for testing purposes, should you want to check the pipeline works without pushing in actual data- = PR_Huppel Huppel -- | Represents a parsed 'LmodLoad' message- | PR_Lmod LmodLoad+ = PRLmod LmodParseResult -- | Represents a parsed 'Shorewall' message- | PR_Shorewall Shorewall+ | PRShorewall Shorewall -- | Represents a parsed 'Snoopy' message- | PR_Snoopy Snoopy+ | PRSnoopy Snoopy -- | Represents a parsed 'Torque' message- | PR_Torque TorqueParseResult+ | PRTorque TorqueParseResult deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- instance ToJSON ParseResult where- toEncoding (PR_Huppel v) = toEncoding v- toEncoding (PR_Lmod v) = toEncoding v- toEncoding (PR_Shorewall v) = toEncoding v- toEncoding (PR_Snoopy v) = toEncoding v- toEncoding (PR_Torque v) = toEncoding v+ toEncoding (PRLmod v) = toEncoding v+ toEncoding (PRShorewall v) = toEncoding v+ toEncoding (PRSnoopy v) = toEncoding v+ toEncoding (PRTorque v) = toEncoding v -------------------------------------------------------------------------------- data Rsyslog = Rsyslog@@ -111,3 +109,7 @@ , jsonkey :: Text -- ^ The key under which the normalised info will appear in the JSON result , fields :: Maybe [(Text, Text)] -- ^ The fields we need to output when creating the JSON encoding as (key, fieldname) } deriving (Show, Generic)++instance NFData ParseResult+instance NFData Rsyslog+instance NFData NormalisedRsyslog
src/HNormalise/Json.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -32,8 +32,6 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-}-{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE OverloadedStrings #-} module HNormalise.Json where@@ -41,6 +39,8 @@ -------------------------------------------------------------------------------- import Control.Monad import Data.Aeson+import qualified Data.ByteString.Char8 as SBS+import qualified Data.ByteString.Lazy.Char8 as BS import qualified Data.HashMap.Lazy as M import Data.Monoid @@ -92,5 +92,9 @@ <> k .= n ) Just fs' -> case toJSON r of- Object syslog -> pairs $ foldl (\v (key, fieldname) -> v <> key .= (M.lookupDefault Null fieldname syslog)) (k .= n) fs'+ Object syslog -> pairs $ foldl (\v (key, fieldname) -> v <> key .= M.lookupDefault Null fieldname syslog) (k .= n) fs' _ -> pairs (k .= n) -- FIXME: this should never happen++encodeNormalisedRsyslog :: NormalisedRsyslog+ -> SBS.ByteString+encodeNormalisedRsyslog = BS.toStrict . encode
src/HNormalise/Lmod/Internal.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -32,18 +32,23 @@ - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -} -{-# LANGUAGE DeriveDataTypeable #-} {-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DuplicateRecordFields #-}-{-# LANGUAGE OverloadedStrings #-} module HNormalise.Lmod.Internal where --------------------------------------------------------------------------------+import Control.DeepSeq (NFData) import Data.Text-import GHC.Generics (Generic)+import GHC.Generics (Generic) -------------------------------------------------------------------------------- +data LmodParseResult+ = LmodLoadParse LmodLoad+ | LmodCommandParse LmodCommand+ deriving (Eq, Show, Generic)+ data LmodModule = LmodModule { name :: !Text , version :: !Text@@ -61,3 +66,15 @@ , modul :: !LmodModule , filename :: !Text } deriving (Show, Eq, Generic)++data LmodCommand = LmodCommand+ { info :: !LmodInfo+ , command :: !Text+ , arguments :: !Text+ } deriving (Eq, Show, Generic)++instance NFData LmodModule+instance NFData LmodInfo+instance NFData LmodLoad+instance NFData LmodCommand+instance NFData LmodParseResult
src/HNormalise/Lmod/Json.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -74,3 +74,10 @@ ( "name" .= name <> "version" .= version )++instance ToJSON LmodCommand where+ toEncoding = genericToEncoding defaultOptions++instance ToJSON LmodParseResult where+ toEncoding (LmodLoadParse l) = toEncoding l+ toEncoding (LmodCommandParse l) = toEncoding l
src/HNormalise/Lmod/Parser.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -70,16 +70,28 @@ , version = version } -parseLmodLoad :: Parser (Text, LmodLoad)+parseLmodLoad :: Parser (Text, LmodParseResult) parseLmodLoad = do string "lmod::" info <- skipSpace *> parseLmodInfo userload <- char ',' *> skipSpace *> kvYesNoParser "userload" m <- char ',' *> skipSpace *> parseLmodModule filename <- char ',' *> skipSpace *> kvTextParser "fn"- return ("lmod", LmodLoad+ return ("lmod", LmodLoadParse LmodLoad { info = info , userload = userload , modul = m , filename = filename+ })++parseLmodCommand :: Parser (Text, LmodParseResult)+parseLmodCommand = do+ string "lmod::"+ info <- skipSpace *> parseLmodInfo+ command <- char ',' *> skipSpace *> kvTextDelimParser "cmd" ","+ arguments <- char ',' *> skipSpace *> kvTextParser "args"+ return ("lmod", LmodCommandParse LmodCommand+ { info = info+ , command = command+ , arguments = arguments })
src/HNormalise/Parser.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -66,32 +66,33 @@ parseMessage :: Parser (Text, ParseResult) parseMessage = let pm parser target = (parser >>= (\(a, v) -> return (a, target v)))- in pm parseLmodLoad PR_Lmod- <|> pm parseShorewall PR_Shorewall- <|> pm parseSnoopy PR_Snoopy- <|> pm parseTorqueQueue PR_Torque- <|> pm parseTorqueStart PR_Torque- <|> pm parseTorqueDelete PR_Torque- <|> pm parseTorqueExit PR_Torque- <|> pm parseTorqueAbort PR_Torque- <|> pm parseTorqueRerun PR_Torque+ in pm parseLmodLoad PRLmod+ <|> pm parseLmodCommand PRLmod+ <|> pm parseShorewall PRShorewall+ <|> pm parseSnoopy PRSnoopy+ <|> pm parseTorqueQueue PRTorque+ <|> pm parseTorqueStart PRTorque+ <|> pm parseTorqueDelete PRTorque+ <|> pm parseTorqueExit PRTorque+ <|> pm parseTorqueAbort PRTorque+ <|> pm parseTorqueRerun PRTorque -------------------------------------------------------------------------------- -- | The 'getJsonKey' function return the key under which the normalised message should appear when JSON is produced getJsonKey :: ParseResult -- ^ Wrapped result for which we need to get a key -> Text -- ^ Key for use in the JSON encoding of the result-getJsonKey (PR_Huppel _) = "huppel"-getJsonKey (PR_Lmod _) = "lmod"-getJsonKey (PR_Torque _) = "torque"-getJsonKey (PR_Shorewall _) = "shorewall"-getJsonKey (PR_Snoopy _) = "snoopy"+--getJsonKey (PR_Huppel _) = "huppel"+getJsonKey (PRLmod _) = "lmod"+getJsonKey (PRTorque _) = "torque"+getJsonKey (PRShorewall _) = "shorewall"+getJsonKey (PRSnoopy _) = "snoopy" -------------------------------------------------------------------------------- -- | The 'parseRsyslogLogstashString' currently is a placeholder function that will convert the incoming rsyslog message -- if it is encoded as expected in a plain string format -- <%PRI%>1 %timegenerated:::date-rfc3339% %HOSTNAME% %syslogtag% - %APP-NAME%: %msg%-parseRsyslogLogstashString :: Maybe [(Text, Text)] -- ^ Output fields- -> Parser SBS.ByteString -- ^ Resulting encoded JSON representation+parseRsyslogLogstashString :: Maybe [(Text, Text)] -- ^ Output fields+ -> Parser NormalisedRsyslog -- SBS.ByteString -- ^ Resulting encoded JSON representation parseRsyslogLogstashString fs = do abspri <- maybeOption $ do char '<'@@ -105,7 +106,7 @@ skipSpace *> char '-' *> skipSpace (original, (appname, parsed)) <- match parseMessage return $ let jsonkey = getJsonKey parsed- in BS.toStrict $ encode NRsyslog+ in NRsyslog { rsyslog = Rsyslog { msg = original , timereported = timereported
src/HNormalise/Shorewall/Internal.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -33,6 +33,7 @@ -} {-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE ExistentialQuantification #-} {-# LANGUAGE OverloadedStrings #-}@@ -51,10 +52,10 @@ "raw" : "2016-03-29T16:10:49.386951+02:00 gligar03 kernel: - kernel:: Shorewall:ext2fw:REJECT:IN=em3 OUT= MAC=aa:aa:bb:ff:88:bc:bc:15:80:8b:f8:f8:80:00 SRC=78.0.0.1 DST=150.0.0.1 LEN=52 TOS=0x00 PREC=0x00 TTL=117 ID=7564 DF PROTO=TCP SPT=60048 DPT=22 WINDOW=65535 RES=0x00 SYN URGP=0", -} ---------------------------------------------------------------------------------import Data.Aeson (FromJSON, ToJSON, toEncoding,- toJSON)+import Control.DeepSeq (NFData)+import Data.Aeson (FromJSON, ToJSON, toEncoding, toJSON) import Data.Text-import GHC.Generics (Generic)+import GHC.Generics (Generic) -------------------------------------------------------------------------------- import HNormalise.Common.Internal@@ -75,3 +76,6 @@ , fwspt :: !(Maybe Integer) , fwdpt :: !(Maybe Integer) } deriving (Show, Eq, Generic)++instance NFData ShorewallProtocol+instance NFData Shorewall
src/HNormalise/Shorewall/Json.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Shorewall/Parser.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Snoopy/Internal.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -35,16 +35,15 @@ {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DuplicateRecordFields #-} {-# LANGUAGE ExistentialQuantification #-}-{-# LANGUAGE OverloadedStrings #-} module HNormalise.Snoopy.Internal where ---------------------------------------------------------------------------------import Data.Aeson (FromJSON, ToJSON, toEncoding,- toJSON)+import Control.DeepSeq (NFData)+import Data.Aeson (FromJSON, ToJSON, toEncoding, toJSON) import Data.Text-import GHC.Generics (Generic)+import GHC.Generics (Generic) -------------------------------------------------------------------------------- import HNormalise.Common.Internal@@ -59,3 +58,5 @@ , executable :: !Text , command :: !Text } deriving (Show, Eq, Generic)++instance NFData Snoopy
src/HNormalise/Snoopy/Json.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Snoopy/Parser.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Torque/Internal.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -38,6 +38,7 @@ module HNormalise.Torque.Internal where --------------------------------------------------------------------------------+import Control.DeepSeq (NFData) import Data.Text import GHC.Generics (Generic) @@ -224,3 +225,22 @@ , name :: !TorqueJobName , torqueEntryType :: TorqueEntryType } deriving (Show, Eq, Generic)++instance NFData TorqueParseResult+instance NFData TorqueEntryType+instance NFData TorqueJobShortNode+instance NFData TorqueJobFQNode+instance NFData TorqueJobNode+instance NFData TorqueExecHost+instance NFData TorqueWalltime+instance NFData TorqueResourceRequest+instance NFData TorqueResourceUsage+instance NFData TorqueJobTime+instance NFData TorqueJobExit+instance NFData TorqueJobName+instance NFData TorqueJobQueue+instance NFData TorqueJobStart+instance NFData TorqueRequestor+instance NFData TorqueJobDelete+instance NFData TorqueJobAbort+instance NFData TorqueJobRerun
src/HNormalise/Torque/Json.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Torque/Parser.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
src/HNormalise/Verbose.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
test/Bench.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
test/HNormalise/Common/ParserSpec.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
test/HNormalise/Lmod/ParserSpec.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -65,7 +65,7 @@ it "parse module load" $ do let s = "lmod:: username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text- s ~> parseLmodLoad `shouldParse` ("lmod", LmodLoad+ s ~> parseLmodLoad `shouldParse` ("lmod", LmodLoadParse LmodLoad { info = LmodInfo { username = "myuser" , cluster = "mycluster"@@ -77,4 +77,16 @@ , version = "2.3-intel-2016b" } , filename = "/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b"+ })++ it "parse command" $ do+ let s = "lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster" :: Text+ s ~> parseLmodCommand `shouldParse` ("lmod", LmodCommandParse LmodCommand+ { info = LmodInfo+ { username = "myuser"+ , cluster = "mycluster"+ , jobid = "132.mymaster.mycluster.mydomain"+ }+ , command = "load"+ , arguments = "cluster/othercluster" })
test/HNormalise/ParserSpec.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -@@ -49,6 +49,7 @@ import HNormalise.Internal import HNormalise.Lmod.Internal import HNormalise.Parser+import HNormalise.Json -------------------------------------------------------------------------------- main :: IO () main = hspec spec@@ -59,21 +60,25 @@ describe "parse full rsyslog message" $ do it "lmod load" $ do let s = "<13>1 2016-06-07T17:50:22.658452+02:00 node2159 lmod: - lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016" :: Text- s ~> (parseRsyslogLogstashString Nothing) `shouldParse`+ s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}" + it "lmod command" $ do+ let s = "<13>1 2017-10-19T21:38:22.533439+02:00 node2801 lmod: - lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster" :: Text+ s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2801\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"mycluster\",\"jobid\":\"132.mymaster.mycluster.mydomain\"},\"command\":\"load\",\"arguments\":\"cluster/othercluster\"}}"+ it "imfile torque input" $ do let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text- s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"test2802\",\"torque\":{\"torqueDatestamp\":\"01/25/2017 15:04:10\",\"name\":{\"number\":0,\"arrayId\":null,\"master\":\"mymaster\",\"cluster\":\"somepokemon\"},\"user\":\"huppelde\",\"group\":\"huppelde\",\"account\":null,\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"huppelde@mymachine.mydomain.com\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"execHost\":[{\"name\":\"node2801.somepokemon.mydomain\",\"cores\":[0]}],\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"cputime\":null,\"prologue\":null,\"epilogue\":null,\"neednodes\":{\"number\":1,\"ppn\":null},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"number\":1,\"ppn\":null},\"select\":null,\"qos\":null,\"other\":null,\"feature\":null,\"host\":null,\"procs\":null,\"nodeset\":null,\"tpn\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"mppmem\":null,\"walltime\":3600},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":2644},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265,\"torqueEntryType\":\"TorqueExitEntry\"}}"+ s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"test2802\",\"torque\":{\"torqueDatestamp\":\"01/25/2017 15:04:10\",\"name\":{\"number\":0,\"arrayId\":null,\"master\":\"mymaster\",\"cluster\":\"somepokemon\"},\"user\":\"huppelde\",\"group\":\"huppelde\",\"account\":null,\"jobname\":\"STDIN\",\"queue\":\"short\",\"startCount\":null,\"owner\":\"huppelde@mymachine.mydomain.com\",\"session\":22598,\"times\":{\"ctime\":1485350399,\"qtime\":1485350399,\"etime\":1485350399,\"startTime\":1485350407,\"endTime\":1485353050},\"execHost\":[{\"name\":\"node2801.somepokemon.mydomain\",\"cores\":[0]}],\"resourceRequest\":{\"mem\":null,\"advres\":null,\"naccesspolicy\":null,\"ncpus\":null,\"cputime\":null,\"prologue\":null,\"epilogue\":null,\"neednodes\":{\"number\":1,\"ppn\":null},\"nice\":0,\"nodeCount\":1,\"nodes\":{\"number\":1,\"ppn\":null},\"select\":null,\"qos\":null,\"other\":null,\"feature\":null,\"host\":null,\"procs\":null,\"nodeset\":null,\"tpn\":null,\"pmem\":null,\"vmem\":4224531456,\"pvmem\":null,\"mppmem\":null,\"walltime\":3600},\"resourceUsage\":{\"cputime\":0,\"energy\":0,\"mem\":31776768,\"vmem\":1578610688,\"walltime\":2644},\"totalExecutionSlots\":1,\"uniqueNodeCount\":1,\"exitStatus\":265,\"torqueEntryType\":\"TorqueExitEntry\"}}" it "imfile torque abort job input" $ do let s = "<133>1 2017-09-14T21:13:09.181048+02:00 master15 torque - torque: 10/17/2013 02:46:14;A;86264.master15.delcatty.gent.vsc;\n" :: Text- s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 10/17/2013 02:46:14;A;86264.master15.delcatty.gent.vsc;\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"master15\",\"torque\":{\"torqueDatestamp\":\"10/17/2013 02:46:14\",\"name\":{\"number\":86264,\"arrayId\":null,\"master\":\"master15\",\"cluster\":\"delcatty\"},\"torqueEntryType\":\"TorqueAbortEntry\"}}"+ s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"torque: 10/17/2013 02:46:14;A;86264.master15.delcatty.gent.vsc;\",\"syslog_abspri\":133,\"syslog_version\":1,\"program\":\"torque\",\"@source_host\":\"master15\",\"torque\":{\"torqueDatestamp\":\"10/17/2013 02:46:14\",\"name\":{\"number\":86264,\"arrayId\":null,\"master\":\"master15\",\"cluster\":\"delcatty\"},\"torqueEntryType\":\"TorqueAbortEntry\"}}" it "snoopy with process ID" $ do let s = "<86>1 2017-05-29T16:40:48.275334+02:00 mymaster snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text- s ~> (parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":86,\"syslog_version\":1,\"program\":\"snoopy\",\"@source_host\":\"mymaster\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}"+ s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python\",\"syslog_abspri\":86,\"syslog_version\":1,\"program\":\"snoopy\",\"@source_host\":\"mymaster\",\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"}}" it "snoopy with process ID; only the @source_host" $ do let s = "<86>1 2017-05-29T16:40:48.275334+02:00 mymaster snoopy[28949]: - snoopy[28949]:: [uid:992 username:nrpe sid:11542 tty:(none) cwd:/ filename:/usr/bin/which]: which python" :: Text- s ~> (parseRsyslogLogstashString $ Just [("@source_host", "hostname")]) `shouldParse` "{\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"},\"@source_host\":\"mymaster\"}"+ s ~> (encodeNormalisedRsyslog `fmap` (parseRsyslogLogstashString $ Just [("@source_host", "hostname")])) `shouldParse` "{\"snoopy\":{\"pid\":28949,\"uid\":992,\"username\":\"nrpe\",\"sid\":11542,\"tty\":\"(none)\",\"cwd\":\"/\",\"executable\":\"/usr/bin/which\",\"command\":\"which python\"},\"@source_host\":\"mymaster\"}"
test/HNormalise/Shorewall/ParserSpec.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
test/HNormalise/Snoopy/ParserSpec.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
test/HNormalise/Torque/ParserSpec.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -
test/Spec.hs view
@@ -1,6 +1,6 @@ {- hnormalise - a log normalisation library -- - Copyright Andy Georges (c) 2017+ - Copyright Ghent University (c) 2017 - - All rights reserved. -