hnormalise 0.3.0.0 → 0.3.1.0

raw patch · 4 files changed

+154/−20 lines, 4 filesPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

API changes (from Hackage documentation)

Files

• hnormalise.cabal +1/−1
• src/HNormalise/Torque/Internal.hs +14/−8
• src/HNormalise/Torque/Parser.hs +30/−6
• test/HNormalise/Torque/ParserSpec.hs +109/−5

@@ -1,5 +1,5 @@ name:                hnormalise-version:             0.3.0.0+version:             0.3.1.0 synopsis:            Log message normalisation tool producing structured JSON messages description:         Log message normalisation tool producing structured JSON messages homepage:            https://github.com/itkovian/hnormalise#readme

@@ -11,8 +11,6 @@ import           GHC.Generics (Generic)  ---------------------------------------------------------------------------------- data TorqueJobShortNode = TorqueJobShortNode     { number :: !Int     , ppn    :: !(Maybe Int)@@ -41,12 +39,20 @@  -------------------------------------------------------------------------------- data TorqueResourceRequest = TorqueResourceRequest-    { nodes     :: !(Either TorqueJobShortNode [TorqueJobFQNode])-    , vmem      :: !Integer-    , nodeCount :: !Int-    , neednodes :: !(Either TorqueJobShortNode [TorqueJobFQNode])-    , nice      :: !(Maybe Int)-    , walltime  :: !TorqueWalltime+    { mem           :: !(Maybe Integer)+    , advres        :: !(Maybe Text)+    , naccesspolicy :: !(Maybe Text)+    , ncpus         :: !(Maybe Int)+    , neednodes     :: !(Either TorqueJobShortNode [TorqueJobFQNode])+    , nice          :: !(Maybe Int)+    , nodeCount     :: !Int+    , nodes         :: !(Either TorqueJobShortNode [TorqueJobFQNode])+    , select        :: !(Maybe Text)+    , qos           :: !(Maybe Text)+    , pmem          :: !(Maybe Integer)+    , vmem          :: !(Maybe Integer)+    , pvmem         :: !(Maybe Integer)+    , walltime      :: !TorqueWalltime     } deriving (Show, Eq, Generic)  --------------------------------------------------------------------------------

@@ -13,7 +13,7 @@ import qualified Data.Map                    as M import           Data.Text                   (Text) import qualified Data.Text                   as T-import           Text.ParserCombinators.Perm ((<$$>), (<||>), permute)+import           Text.ParserCombinators.Perm ((<$$>), (<||>), (<$?>), (<|?>), permute)  -------------------------------------------------------------------------------- import           HNormalise.Common.Parser@@ -96,14 +96,38 @@         return TorqueJobFQNode { name = fqdn, ppn = ppn})  --------------------------------------------------------------------------------+{- Examples found in the 2016 logs+Resource_List.advres Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime+Resource_List.mem Resource_List.ncpus Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime+Resource_List.mem Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.qos Resource_List.vmem Resource_List.walltime+Resource_List.mem Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime+Resource_List.mem Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.walltime+Resource_List.naccesspolicy Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime+Resource_List.ncpus Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime+Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.nodeset Resource_List.vmem Resource_List.walltime+Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.pmem Resource_List.vmem Resource_List.walltime+Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.pmem Resource_List.walltime+Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.pvmem Resource_List.vmem Resource_List.walltime+Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.qos Resource_List.vmem Resource_List.walltime+Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.vmem Resource_List.walltime+Resource_List.neednodes Resource_List.nice Resource_List.nodect Resource_List.nodes Resource_List.walltime+-} parseTorqueResourceRequest :: Parser TorqueResourceRequest parseTorqueResourceRequest = do     permute $ TorqueResourceRequest-        <$$> skipSpace *> string "Resource_List.nodes=" *> parseTorqueResourceNodeList-        <||> skipSpace *> string "Resource_List.vmem=" *> parseTorqueMemory-        <||> skipSpace *> kvNumParser "Resource_List.nodect"+        <$?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.mem=" *> parseTorqueMemory))+        <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.advres"))+        <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.naccesspolicy"))+        <|?> (Nothing, Just `fmap` (skipSpace *> kvNumParser "Resource_List.ncpus"))         <||> skipSpace *> string "Resource_List.neednodes=" *> parseTorqueResourceNodeList-        <||> maybeOption (skipSpace *> kvNumParser "Resource_List.nice")+        <|?> (Nothing, Just `fmap` (skipSpace *> kvNumParser "Resource_List.nice"))+        <||> skipSpace *> kvNumParser "Resource_List.nodect"+        <||> skipSpace *> string "Resource_List.nodes=" *> parseTorqueResourceNodeList+        <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.select"))+        <|?> (Nothing, Just `fmap` (skipSpace *> kvTextParser "Resource_List.qos"))+        <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.pmem=" *> parseTorqueMemory))+        <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.vmem=" *> parseTorqueMemory))+        <|?> (Nothing, Just `fmap` (skipSpace *> string "Resource_List.pvmem=" *> parseTorqueMemory))         <||> skipSpace *> string "Resource_List.walltime=" *> parseTorqueWalltime  --------------------------------------------------------------------------------@@ -149,10 +173,10 @@     group <- skipSpace *> kvTextParser "group"     jobname <- skipSpace *> kvTextParser "jobname"     queue <- skipSpace *> kvTextParser "queue"-    start_count <- maybeOption $ skipSpace *> kvNumParser "start_count"     ctime <- skipSpace *> kvNumParser "ctime"     qtime <- skipSpace *> kvNumParser "qtime"     etime <- skipSpace *> kvNumParser "etime"+    start_count <- maybeOption $ skipSpace *> kvNumParser "start_count"     start <- skipSpace *> kvNumParser "start"     owner <- skipSpace *> kvTextParser "owner"     exec_host <- skipSpace *> parseTorqueHostList

@@ -104,9 +104,105 @@             let s = "123456[789].master.mycluster.mydomain;" :: Text             s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456, array_id = Just 789, master = "master", cluster = "mycluster" } +    describe "parseTorqueResourceRequest" $ do+        it "parse mandatory fields in expected order" $ do+            let s = "Resource_List.neednodes=1:ppn=1 Resource_List.nodect=1 Resource_List.nodes=1:ppn=1 Resource_List.walltime=01:00:00" :: Text+            s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest+                { mem           = Nothing+                , advres        = Nothing+                , naccesspolicy = Nothing+                , ncpus         = Nothing+                , neednodes     = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , nice          = Nothing+                , nodeCount     = 1+                , nodes         = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , select        = Nothing+                , qos           = Nothing+                , pmem          = Nothing+                , vmem          = Nothing+                , pvmem         = Nothing+                , walltime      = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 }+                }++        it "parse mandatory fields in reverse order" $ do+            let s = "Resource_List.walltime=01:00:00 Resource_List.nodes=1:ppn=1 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1" :: Text+            s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest+                { mem           = Nothing+                , advres        = Nothing+                , naccesspolicy = Nothing+                , ncpus         = Nothing+                , neednodes     = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , nice          = Nothing+                , nodeCount     = 1+                , nodes         = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , select        = Nothing+                , qos           = Nothing+                , pmem          = Nothing+                , vmem          = Nothing+                , pvmem         = Nothing+                , walltime      = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 }+                }++        it "parse mandatory fields plus mem fields" $ do+            let s = "Resource_List.vmem=1mb Resource_List.walltime=01:00:00 Resource_List.pvmem=400kb Resource_List.pmem=200kB Resource_List.nodes=1:ppn=1 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1" :: Text+            s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest+                { mem           = Nothing+                , advres        = Nothing+                , naccesspolicy = Nothing+                , ncpus         = Nothing+                , neednodes     = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , nice          = Nothing+                , nodeCount     = 1+                , nodes         = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , select        = Nothing+                , qos           = Nothing+                , pmem          = Just $ 200 * 1024+                , vmem          = Just $ 1 * 1024 * 1024+                , pvmem         = Just $ 400 * 1024+                , walltime      = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 }+                }++        it "parse mandatory fields plus reservation field" $ do+            let s = "Resource_List.walltime=01:00:00 Resource_List.advres=myreservation.1 Resource_List.nodes=1:ppn=1 Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1" :: Text+            s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest+                { mem           = Nothing+                , advres        = Just "myreservation.1"+                , naccesspolicy = Nothing+                , ncpus         = Nothing+                , neednodes     = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , nice          = Nothing+                , nodeCount     = 1+                , nodes         = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , select        = Nothing+                , qos           = Nothing+                , pmem          = Nothing+                , vmem          = Nothing+                , pvmem         = Nothing+                , walltime      = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 }+                }++        it "parse mandatory fields plus qos field" $ do+            let s = "Resource_List.walltime=01:00:00 Resource_List.nodes=1:ppn=1 Resource_List.qos=someqos Resource_List.nodect=1 Resource_List.neednodes=1:ppn=1" :: Text+            s ~> parseTorqueResourceRequest `shouldParse` TorqueResourceRequest+                { mem           = Nothing+                , advres        = Nothing+                , naccesspolicy = Nothing+                , ncpus         = Nothing+                , neednodes     = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , nice          = Nothing+                , nodeCount     = 1+                , nodes         = Left TorqueJobShortNode { number = 1, ppn = Just 1 }+                , select        = Nothing+                , qos           = Just "someqos"+                , pmem          = Nothing+                , vmem          = Nothing+                , pvmem         = Nothing+                , walltime      = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0 }+                }+     describe "parseTorqueExit" $ do         it "parse job exit log line" $ do-            let s = "04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text+            let s = "torque: 04/05/2017 13:06:53;E;45.master23.banette.gent.vsc;user=vsc40075 group=vsc40075 jobname=STDIN queue=short ctime=1491390300 qtime=1491390300 etime=1491390300 start=1491390307 owner=vsc40075@gligar01.gligar.gent.vsc exec_host=node2801.banette.gent.vsc/0-1+node2803.banette.gent.vsc/0-1 Resource_List.nodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.vmem=1gb Resource_List.nodect=2 Resource_List.neednodes=node2801.banette.gent.vsc:ppn=2+node2803.banette.gent.vsc:ppn=2 Resource_List.nice=0 Resource_List.walltime=01:00:00 session=15273 total_execution_slots=4 unique_node_count=2 end=1491390413 Exit_status=0 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=55048kb resources_used.vmem=92488kb resources_used.walltime=00:01:44" :: Text             s ~> parseTorqueExit `shouldParse` ("torque", TorqueJobExit                 { name = TorqueJobName { number = 45, array_id = Nothing, master = "master23", cluster = "banette" }                 , user = "vsc40075"@@ -124,7 +220,11 @@                     , endTime = 1491390413                     }                 , resourceRequest = TorqueResourceRequest-                    { nodes = Right+                    { mem           = Nothing+                    , advres        = Nothing+                    , naccesspolicy = Nothing+                    , ncpus         = Nothing+                    , neednodes = Right                         [ TorqueJobFQNode                             { name = "node2801.banette.gent.vsc"                             , ppn  = 2@@ -134,9 +234,9 @@                             , ppn  = 2                             }                         ]-                    , vmem = 1 * 1024 * 1024 * 1024+                    , nice      = Just 0                     , nodeCount = 2-                    , neednodes = Right+                    , nodes = Right                         [ TorqueJobFQNode                             { name = "node2801.banette.gent.vsc"                             , ppn  = 2@@ -146,7 +246,11 @@                             , ppn  = 2                             }                         ]-                    , nice      = Just 0+                    , select        = Nothing+                    , qos           = Nothing+                    , vmem = Just $ 1 * 1024 * 1024 * 1024+                    , pmem = Nothing+                    , pvmem = Nothing                     , walltime  = TorqueWalltime { days = 0, hours = 1, minutes = 0, seconds = 0}                     }                 , resourceUsage = TorqueResourceUsage