diff --git a/hnormalise.cabal b/hnormalise.cabal --- a/hnormalise.cabal +++ b/hnormalise.cabal @@ -1,5 +1,5 @@ name: hnormalise -version: 0.5.0.0 +version: 0.5.1.0 synopsis: Log message normalisation tool producing structured JSON messages description: Log message normalisation tool producing structured JSON messages homepage: https://github.com/itkovian/hnormalise#readme diff --git a/src/HNormalise/Lmod/Internal.hs b/src/HNormalise/Lmod/Internal.hs --- a/src/HNormalise/Lmod/Internal.hs +++ b/src/HNormalise/Lmod/Internal.hs @@ -44,6 +44,8 @@ import GHC.Generics (Generic) -------------------------------------------------------------------------------- +import HNormalise.Torque.Internal (TorqueJobName) + data LmodParseResult = LmodLoadParse LmodLoad | LmodCommandParse LmodCommand @@ -57,7 +59,7 @@ data LmodInfo = LmodInfo { username :: !Text , cluster :: !Text - , jobid :: !Text + , jobid :: !(Maybe TorqueJobName) } deriving (Show, Eq, Generic) data LmodLoad = LmodLoad diff --git a/src/HNormalise/Lmod/Json.hs b/src/HNormalise/Lmod/Json.hs --- a/src/HNormalise/Lmod/Json.hs +++ b/src/HNormalise/Lmod/Json.hs @@ -45,20 +45,14 @@ -------------------------------------------------------------------------------- import HNormalise.Lmod.Internal +import HNormalise.Torque.Json -------------------------------------------------------------------------------- instance ToJSON LmodInfo where - --toEncoding = genericToEncoding defaultOptions - toEncoding (LmodInfo username cluster jobid) = - pairs - ( "username" .= username - <> "cluster" .= cluster - <> "jobid" .= jobid - ) + toEncoding = genericToEncoding defaultOptions instance ToJSON LmodLoad where - --toEncoding = genericToEncoding defaultOptions toEncoding (LmodLoad info userload modul filename) = pairs ( "info" .= info @@ -68,12 +62,7 @@ ) instance ToJSON LmodModule where - --toEncoding = genericToEncoding defaultOptions - toEncoding (LmodModule name version) = - pairs - ( "name" .= name - <> "version" .= version - ) + toEncoding = genericToEncoding defaultOptions instance ToJSON LmodCommand where toEncoding = genericToEncoding defaultOptions diff --git a/src/HNormalise/Lmod/Parser.hs b/src/HNormalise/Lmod/Parser.hs --- a/src/HNormalise/Lmod/Parser.hs +++ b/src/HNormalise/Lmod/Parser.hs @@ -48,13 +48,14 @@ import HNormalise.Common.Parser import HNormalise.Lmod.Internal +import HNormalise.Torque.Parser (parseTorqueJobName) -------------------------------------------------------------------------------- parseLmodInfo :: Parser LmodInfo parseLmodInfo = do username <- kvTextDelimParser "username" "," cluster <- char ',' *> skipSpace *> kvTextDelimParser "cluster" "," - jobid <- char ',' *> skipSpace *> kvTextDelimParser "jobid" "," + jobid <- char ',' *> skipSpace *> string "jobid=" *> maybeOption (parseTorqueJobName ",") return LmodInfo { username = username , cluster = cluster diff --git a/src/HNormalise/Torque/Parser.hs b/src/HNormalise/Torque/Parser.hs --- a/src/HNormalise/Torque/Parser.hs +++ b/src/HNormalise/Torque/Parser.hs @@ -47,6 +47,7 @@ import Data.List (concatMap, groupBy, sort) import qualified Data.Map as M import Data.Maybe (fromMaybe) +import Data.String import Data.Text (Text) import qualified Data.Text as T import Text.ParserCombinators.Perm (permute, (<$$>), (<$?>), (<|?>), @@ -105,13 +106,13 @@ -------------------------------------------------------------------------------- -- | 'parseTorqueJobName' splits the job name in its components, i.e., ID, [ array ID,] master and cluster -parseTorqueJobName :: Parser TorqueJobName -parseTorqueJobName = do +parseTorqueJobName :: String -> Parser TorqueJobName +parseTorqueJobName sep = do n <- decimal a <- parseArrayId m <- char '.' *> takeTill (== '.') c <- char '.' *> takeTill (== '.') - manyTill anyChar (lookAhead ";") *> char ';' + takeTill (`elem` sep) return TorqueJobName { number = n, arrayId = a, master = m, cluster = c} where parseArrayId :: Parser (Maybe Integer) @@ -344,7 +345,8 @@ , Integer , Integer) parseCommonAccountingInfo = do - name <- parseTorqueJobName + name <- parseTorqueJobName ";" + char ';' user <- kvTextParser "user" group <- skipSpace *> kvTextParser "group" account <- skipSpace *> maybeOption (kvTextParser "account") @@ -417,7 +419,8 @@ parseTorqueDelete :: Parser (Text, TorqueParseResult) parseTorqueDelete = do torqueDatestamp <- parseTorqueAccountingDatestamp ";D;" - name <- parseTorqueJobName + name <- parseTorqueJobName ";" + char ';' requestor <- parseTorqueRequestor return ("torque", TorqueDelete TorqueJobDelete @@ -432,7 +435,8 @@ parseTorqueAbort :: Parser (Text, TorqueParseResult) parseTorqueAbort = do torqueDatestamp <- parseTorqueAccountingDatestamp ";A;" - name <- parseTorqueJobName + name <- parseTorqueJobName ";" + char ';' return ("torque", TorqueAbort TorqueJobAbort { torqueDatestamp = torqueDatestamp @@ -445,7 +449,8 @@ parseTorqueRerun :: Parser (Text, TorqueParseResult) parseTorqueRerun = do torqueDatestamp <- parseTorqueAccountingDatestamp ";R;" - name <- parseTorqueJobName + name <- parseTorqueJobName ";" + char ';' return ("torque", TorqueRerun TorqueJobRerun { torqueDatestamp = torqueDatestamp @@ -459,7 +464,8 @@ parseTorqueQueue :: Parser (Text, TorqueParseResult) parseTorqueQueue = do torqueDatestamp <- parseTorqueAccountingDatestamp ";Q;" - name <- parseTorqueJobName + name <- parseTorqueJobName ";" + char ';' queue <- kvTextParser "queue" return ("torque", TorqueQueue TorqueJobQueue diff --git a/test/HNormalise/Lmod/ParserSpec.hs b/test/HNormalise/Lmod/ParserSpec.hs --- a/test/HNormalise/Lmod/ParserSpec.hs +++ b/test/HNormalise/Lmod/ParserSpec.hs @@ -46,6 +46,7 @@ -------------------------------------------------------------------------------- import HNormalise.Lmod.Parser import HNormalise.Lmod.Internal +import HNormalise.Torque.Internal -------------------------------------------------------------------------------- main :: IO () @@ -53,23 +54,54 @@ -------------------------------------------------------------------------------- spec :: Spec -spec = do +spec = describe "parseLmodInfo" $ do it "parse regular info" $ do - let s = "username=someuser, cluster=myspace, jobid=myjobid" :: Text - s ~> parseLmodInfo `shouldParse` LmodInfo { username = "someuser" , cluster = "myspace" , jobid = "myjobid" } + let s = "username=someuser, cluster=myspace, jobid=11[1].mymaster.mycluster.mydomain" :: Text + s ~> parseLmodInfo `shouldParse` LmodInfo + { username = "someuser" + , cluster = "myspace" + , jobid = Just TorqueJobName + { number = 11 + , arrayId = Just 1 + , master = "mymaster" + , cluster = "mycluster" + } + } it "parse module" $ do let s = "module=HNormalise/0.2.0.0-ghc-8.0.2," :: Text s ~> parseLmodModule `shouldParse` LmodModule { name = "HNormalise", version = "0.2.0.0-ghc-8.0.2" } + it "parse module load with empty jobid" $ do + let s = "lmod:: username=myuser, cluster=mycluster, jobid=, userload=yes, module=cluster/.mycluster, fn=/etc/modulefiles/vsc/cluster/.banette.lua" :: Text + s ~> parseLmodLoad `shouldParse` ("lmod", LmodLoadParse LmodLoad + { info = LmodInfo + { username = "myuser" + , cluster = "mycluster" + , jobid = Nothing + } + , userload = True + , modul = LmodModule + { name = "cluster" + , version = ".mycluster" + } + , filename = "/etc/modulefiles/vsc/cluster/.banette.lua" + }) + + it "parse module load" $ do let s = "lmod:: username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text s ~> parseLmodLoad `shouldParse` ("lmod", LmodLoadParse LmodLoad { info = LmodInfo { username = "myuser" , cluster = "mycluster" - , jobid = "3230905.master.mycluster.mydomain" + , jobid = Just TorqueJobName + { number = 3230905 + , arrayId = Nothing + , master = "master" + , cluster = "mycluster" + } } , userload = True , modul = LmodModule @@ -85,7 +117,12 @@ { info = LmodInfo { username = "myuser" , cluster = "mycluster" - , jobid = "132.mymaster.mycluster.mydomain" + , jobid = Just TorqueJobName + { number = 132 + , arrayId = Nothing + , master = "mymaster" + , cluster = "mycluster" + } } , command = "load" , arguments = "cluster/othercluster" diff --git a/test/HNormalise/ParserSpec.hs b/test/HNormalise/ParserSpec.hs --- a/test/HNormalise/ParserSpec.hs +++ b/test/HNormalise/ParserSpec.hs @@ -61,11 +61,12 @@ it "lmod load" $ do let s = "<13>1 2016-06-07T17:50:22.658452+02:00 node2159 lmod: - lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016" :: Text s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` - "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}" + "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":{\"number\":434,\"arrayId\":null,\"master\":\"master\",\"cluster\":\"mycluster\"}},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}" it "lmod command" $ do let s = "<13>1 2017-10-19T21:38:22.533439+02:00 node2801 lmod: - lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster" :: Text - s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2801\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"mycluster\",\"jobid\":\"132.mymaster.mycluster.mydomain\"},\"command\":\"load\",\"arguments\":\"cluster/othercluster\"}}" + s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` + "{\"message\":\"lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2801\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"mycluster\",\"jobid\":{\"number\":132,\"arrayId\":null,\"master\":\"mymaster\",\"cluster\":\"mycluster\"}},\"command\":\"load\",\"arguments\":\"cluster/othercluster\"}}" it "imfile torque input" $ do let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text diff --git a/test/HNormalise/Torque/ParserSpec.hs b/test/HNormalise/Torque/ParserSpec.hs --- a/test/HNormalise/Torque/ParserSpec.hs +++ b/test/HNormalise/Torque/ParserSpec.hs @@ -132,11 +132,11 @@ describe "parseTorqueJobName" $ do it "parse regular torque job name" $ do let s = "123456789.master.mycluster.mydomain;" :: Text - s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456789, arrayId = Nothing, master = "master", cluster = "mycluster" } + s ~> parseTorqueJobName ";" `shouldParse` TorqueJobName { number = 123456789, arrayId = Nothing, master = "master", cluster = "mycluster" } it "parse array torque job name" $ do let s = "123456[789].master.mycluster.mydomain;" :: Text - s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456, arrayId = Just 789, master = "master", cluster = "mycluster" } + s ~> parseTorqueJobName ";" `shouldParse` TorqueJobName { number = 123456, arrayId = Just 789, master = "master", cluster = "mycluster" } describe "parseTorqueResourceRequest" $ do it "parse mandatory fields in expected order" $ do