diff --git a/hnormalise.cabal b/hnormalise.cabal --- a/hnormalise.cabal +++ b/hnormalise.cabal @@ -1,5 +1,5 @@ name: hnormalise -version: 0.3.0.0 +version: 0.3.1.0 synopsis: Log message normalisation tool producing structured JSON messages description: Log message normalisation tool producing structured JSON messages homepage: https://github.com/itkovian/hnormalise#readme diff --git a/src/HNormalise/Torque/Internal.hs b/src/HNormalise/Torque/Internal.hs --- a/src/HNormalise/Torque/Internal.hs +++ b/src/HNormalise/Torque/Internal.hs @@ -11,8 +11,6 @@ import GHC.Generics (Generic) -------------------------------------------------------------------------------- - - data TorqueJobShortNode = TorqueJobShortNode { number :: !Int , ppn :: !(Maybe Int) @@ -41,12 +39,20 @@ -------------------------------------------------------------------------------- data TorqueResourceRequest = TorqueResourceRequest - { nodes :: !(Either TorqueJobShortNode [TorqueJobFQNode]) - , vmem :: !Integer - , nodeCount :: !Int - , neednodes :: !(Either TorqueJobShortNode [TorqueJobFQNode]) - , nice :: !(Maybe Int) - , walltime :: !TorqueWalltime + { mem :: !(Maybe Integer) + , advres :: !(Maybe Text) + , naccesspolicy :: !(Maybe Text) + , ncpus :: !(Maybe Int) + , neednodes :: !(Either TorqueJobShortNode [TorqueJobFQNode]) + , nice :: !(Maybe Int) + , nodeCount :: !Int + , nodes :: !(Either TorqueJobShortNode [TorqueJobFQNode]) + , select :: !(Maybe Text) + , qos :: !(Maybe Text) + , pmem :: !(Maybe Integer) + , vmem :: !(Maybe Integer) + , pvmem :: !(Maybe Integer) + , walltime :: !TorqueWalltime } deriving (Show, Eq, Generic) -------------------------------------------------------------------------------- diff --git a/src/HNormalise/Torque/Parser.hs b/src/HNormalise/Torque/Parser.hs --- a/src/HNormalise/Torque/Parser.hs +++ b/src/HNormalise/Torque/Parser.hs @@ -13,7 +13,7 @@ import qualified Data.Map as M import Data.Text (Text) import qualified Data.Text as T -import Text.ParserCombinators.Perm ((<$$>), (<||>), permute) +import Text.ParserCombinators.Perm ((<$$>), (<||>), (<$?>), (<|?>), permute) -------------------------------------------------------------------------------- import HNormalise.Common.Parser @@ -96,14 +96,38 @@ return TorqueJobFQNode { name = fqdn, ppn = ppn}) -------------------------------------------------------------------------------- +{- Examples found in the 2016 logs +Resource_List.advres Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime +Resource_List.mem Resource_List.ncpus Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime +Resource_List.mem Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.qos Resource_List.vmem Resource_List.walltime +Resource_List.mem Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime +Resource_List.mem Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.walltime +Resource_List.naccesspolicy Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime +Resource_List.ncpus Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime +Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.nodeset Resource_List.vmem Resource_List.walltime +Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.pmem Resource_List.vmem Resource_List.walltime +Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.pmem Resource_List.walltime +Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.pvmem Resource_List.vmem Resource_List.walltime +Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.qos Resource_List.vmem Resource_List.walltime +Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime +Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.walltime +-} parseTorqueResourceRequest :: Parser TorqueResourceRequest parseTorqueResourceRequest = do permute $ TorqueResourceRequest - <$$> skipSpace *> string "Resource_List.nodes=" *> parseTorqueResourceNodeList - <||> skipSpace *> string "Resource_List.vmem=" *> parseTorqueMemory - <||> skipSpace *> kvNumParser "Resource_List.nodect" + <$?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.mem=" *> parseTorqueMemory)) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.advres")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.naccesspolicy")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvNumParser "Resource_List.ncpus")) <||> skipSpace *> string "Resource_List.neednodes=" *> parseTorqueResourceNodeList - <||> maybeOption (skipSpace *> kvNumParser "Resource_List.nice") + <|?> (Nothing, Just `fmap` (skipSpace *> kvNumParser "Resource_List.nice")) + <||> skipSpace *> kvNumParser "Resource_List.nodect" + <||> skipSpace *> string "Resource_List.nodes=" *> parseTorqueResourceNodeList + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.select")) + <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.qos")) + <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.pmem=" *> parseTorqueMemory)) + <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.vmem=" *> parseTorqueMemory)) + <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.pvmem=" *> parseTorqueMemory)) <||> skipSpace *> string "Resource_List.walltime=" *> parseTorqueWalltime -------------------------------------------------------------------------------- @@ -149,10 +173,10 @@ group <- skipSpace *> kvTextParser "group" jobname <- skipSpace *> kvTextParser "jobname" queue <- skipSpace *> kvTextParser "queue" - start_count <- maybeOption $ skipSpace *> kvNumParser "start_count" ctime <- skipSpace *> kvNumParser "ctime" qtime <- skipSpace *> kvNumParser "qtime" etime <- skipSpace *> kvNumParser "etime" + start_count <- maybeOption $ skipSpace *> kvNumParser "start_count" start <- skipSpace *> kvNumParser "start" owner <- skipSpace *> kvTextParser "owner" exec_host <- skipSpace *> parseTorqueHostList diff --git a/test/HNormalise/Torque/ParserSpec.hs b/test/HNormalise/Torque/ParserSpec.hs --- a/test/HNormalise/Torque/ParserSpec.hs +++ b/test/HNormalise/Torque/ParserSpec.hs @@ -104,9 +104,105 @@ let s = "123456[789].master.mycluster.mydomain;" :: Text s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456, array_id = Just 789, master = "master", cluster = "mycluster" } + describe "parseTorqueResourceRequest" $ do + it "parse mandatory fields in expected order" $ do + let s = "Resource_List.neednodes=1:ppn=1 Resource_List.nodect=1 Resource_List.nodes=1:ppn=1 Resource_List.walltime=01:00:00" :: Text + s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , neednodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , nice = Nothing + , nodeCount = 1 + , nodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , select = Nothing + , qos = Nothing + , pmem = Nothing + , vmem = Nothing + , pvmem = Nothing + , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } + } + + it "parse mandatory fields in reverse order" $ do + let s = "Resource_List.walltime=01:00:00 Resource_List.nodes=1:ppn=1 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1" :: Text + s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , neednodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , nice = Nothing + , nodeCount = 1 + , nodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , select = Nothing + , qos = Nothing + , pmem = Nothing + , vmem = Nothing + , pvmem = Nothing + , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } + } + + it "parse mandatory fields plus mem fields" $ do + let s = "Resource_List.vmem=1mb Resource_List.walltime=01:00:00 Resource_List.pvmem=400kb Resource_List.pmem=200kB Resource_List.nodes=1:ppn=1 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1" :: Text + s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , neednodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , nice = Nothing + , nodeCount = 1 + , nodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , select = Nothing + , qos = Nothing + , pmem = Just $ 200 * 1024 + , vmem = Just $ 1 * 1024 * 1024 + , pvmem = Just $ 400 * 1024 + , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } + } + + it "parse mandatory fields plus reservation field" $ do + let s = "Resource_List.walltime=01:00:00 Resource_List.advres=myreservation.1 Resource_List.nodes=1:ppn=1 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1" :: Text + s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest + { mem = Nothing + , advres = Just "myreservation.1" + , naccesspolicy = Nothing + , ncpus = Nothing + , neednodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , nice = Nothing + , nodeCount = 1 + , nodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , select = Nothing + , qos = Nothing + , pmem = Nothing + , vmem = Nothing + , pvmem = Nothing + , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } + } + + it "parse mandatory fields plus qos field" $ do + let s = "Resource_List.walltime=01:00:00 Resource_List.nodes=1:ppn=1 Resource_List.qos=someqos Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1" :: Text + s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , neednodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , nice = Nothing + , nodeCount = 1 + , nodes = Left TorqueJobShortNode { number = 1, ppn = Just 1 } + , select = Nothing + , qos = Just "someqos" + , pmem = Nothing + , vmem = Nothing + , pvmem = Nothing + , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 } + } + describe "parseTorqueExit" $ do it "parse job exit log line" $ do - let s = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text + let s = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text s ~> parseTorqueExit `shouldParse` ("torque", TorqueJobExit { name = TorqueJobName { number = 45, array_id = Nothing, master = "master23", cluster = "banette" } , user = "vsc40075" @@ -124,7 +220,11 @@ , endTime = 1491390413 } , resourceRequest = TorqueResourceRequest - { nodes = Right + { mem = Nothing + , advres = Nothing + , naccesspolicy = Nothing + , ncpus = Nothing + , neednodes = Right [ TorqueJobFQNode { name = "node2801.banette.gent.vsc" , ppn = 2 @@ -134,9 +234,9 @@ , ppn = 2 } ] - , vmem = 1 * 1024 * 1024 * 1024 + , nice = Just 0 , nodeCount = 2 - , neednodes = Right + , nodes = Right [ TorqueJobFQNode { name = "node2801.banette.gent.vsc" , ppn = 2 @@ -146,7 +246,11 @@ , ppn = 2 } ] - , nice = Just 0 + , select = Nothing + , qos = Nothing + , vmem = Just $ 1 * 1024 * 1024 * 1024 + , pmem = Nothing + , pvmem = Nothing , walltime = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0} } , resourceUsage = TorqueResourceUsage