hnormalise 0.5.0.0 → 0.5.1.0
raw patch · 8 files changed
+70/−34 lines, 8 files
Files
- hnormalise.cabal +1/−1
- src/HNormalise/Lmod/Internal.hs +3/−1
- src/HNormalise/Lmod/Json.hs +3/−14
- src/HNormalise/Lmod/Parser.hs +2/−1
- src/HNormalise/Torque/Parser.hs +14/−8
- test/HNormalise/Lmod/ParserSpec.hs +42/−5
- test/HNormalise/ParserSpec.hs +3/−2
- test/HNormalise/Torque/ParserSpec.hs +2/−2
hnormalise.cabal view
@@ -1,5 +1,5 @@ name: hnormalise-version: 0.5.0.0+version: 0.5.1.0 synopsis: Log message normalisation tool producing structured JSON messages description: Log message normalisation tool producing structured JSON messages homepage: https://github.com/itkovian/hnormalise#readme
src/HNormalise/Lmod/Internal.hs view
@@ -44,6 +44,8 @@ import GHC.Generics (Generic) -------------------------------------------------------------------------------- +import HNormalise.Torque.Internal (TorqueJobName)+ data LmodParseResult = LmodLoadParse LmodLoad | LmodCommandParse LmodCommand@@ -57,7 +59,7 @@ data LmodInfo = LmodInfo { username :: !Text , cluster :: !Text- , jobid :: !Text+ , jobid :: !(Maybe TorqueJobName) } deriving (Show, Eq, Generic) data LmodLoad = LmodLoad
src/HNormalise/Lmod/Json.hs view
@@ -45,20 +45,14 @@ -------------------------------------------------------------------------------- import HNormalise.Lmod.Internal+import HNormalise.Torque.Json -------------------------------------------------------------------------------- instance ToJSON LmodInfo where- --toEncoding = genericToEncoding defaultOptions- toEncoding (LmodInfo username cluster jobid) =- pairs- ( "username" .= username- <> "cluster" .= cluster- <> "jobid" .= jobid- )+ toEncoding = genericToEncoding defaultOptions instance ToJSON LmodLoad where- --toEncoding = genericToEncoding defaultOptions toEncoding (LmodLoad info userload modul filename) = pairs ( "info" .= info@@ -68,12 +62,7 @@ ) instance ToJSON LmodModule where- --toEncoding = genericToEncoding defaultOptions- toEncoding (LmodModule name version) =- pairs- ( "name" .= name- <> "version" .= version- )+ toEncoding = genericToEncoding defaultOptions instance ToJSON LmodCommand where toEncoding = genericToEncoding defaultOptions
src/HNormalise/Lmod/Parser.hs view
@@ -48,13 +48,14 @@ import HNormalise.Common.Parser import HNormalise.Lmod.Internal+import HNormalise.Torque.Parser (parseTorqueJobName) -------------------------------------------------------------------------------- parseLmodInfo :: Parser LmodInfo parseLmodInfo = do username <- kvTextDelimParser "username" "," cluster <- char ',' *> skipSpace *> kvTextDelimParser "cluster" ","- jobid <- char ',' *> skipSpace *> kvTextDelimParser "jobid" ","+ jobid <- char ',' *> skipSpace *> string "jobid=" *> maybeOption (parseTorqueJobName ",") return LmodInfo { username = username , cluster = cluster
src/HNormalise/Torque/Parser.hs view
@@ -47,6 +47,7 @@ import Data.List (concatMap, groupBy, sort) import qualified Data.Map as M import Data.Maybe (fromMaybe)+import Data.String import Data.Text (Text) import qualified Data.Text as T import Text.ParserCombinators.Perm (permute, (<$$>), (<$?>), (<|?>),@@ -105,13 +106,13 @@ -------------------------------------------------------------------------------- -- | 'parseTorqueJobName' splits the job name in its components, i.e., ID, [ array ID,] master and cluster-parseTorqueJobName :: Parser TorqueJobName-parseTorqueJobName = do+parseTorqueJobName :: String -> Parser TorqueJobName+parseTorqueJobName sep = do n <- decimal a <- parseArrayId m <- char '.' *> takeTill (== '.') c <- char '.' *> takeTill (== '.')- manyTill anyChar (lookAhead ";") *> char ';'+ takeTill (`elem` sep) return TorqueJobName { number = n, arrayId = a, master = m, cluster = c} where parseArrayId :: Parser (Maybe Integer)@@ -344,7 +345,8 @@ , Integer , Integer) parseCommonAccountingInfo = do- name <- parseTorqueJobName+ name <- parseTorqueJobName ";"+ char ';' user <- kvTextParser "user" group <- skipSpace *> kvTextParser "group" account <- skipSpace *> maybeOption (kvTextParser "account")@@ -417,7 +419,8 @@ parseTorqueDelete :: Parser (Text, TorqueParseResult) parseTorqueDelete = do torqueDatestamp <- parseTorqueAccountingDatestamp ";D;"- name <- parseTorqueJobName+ name <- parseTorqueJobName ";"+ char ';' requestor <- parseTorqueRequestor return ("torque", TorqueDelete TorqueJobDelete@@ -432,7 +435,8 @@ parseTorqueAbort :: Parser (Text, TorqueParseResult) parseTorqueAbort = do torqueDatestamp <- parseTorqueAccountingDatestamp ";A;"- name <- parseTorqueJobName+ name <- parseTorqueJobName ";"+ char ';' return ("torque", TorqueAbort TorqueJobAbort { torqueDatestamp = torqueDatestamp@@ -445,7 +449,8 @@ parseTorqueRerun :: Parser (Text, TorqueParseResult) parseTorqueRerun = do torqueDatestamp <- parseTorqueAccountingDatestamp ";R;"- name <- parseTorqueJobName+ name <- parseTorqueJobName ";"+ char ';' return ("torque", TorqueRerun TorqueJobRerun { torqueDatestamp = torqueDatestamp@@ -459,7 +464,8 @@ parseTorqueQueue :: Parser (Text, TorqueParseResult) parseTorqueQueue = do torqueDatestamp <- parseTorqueAccountingDatestamp ";Q;"- name <- parseTorqueJobName+ name <- parseTorqueJobName ";"+ char ';' queue <- kvTextParser "queue" return ("torque", TorqueQueue TorqueJobQueue
test/HNormalise/Lmod/ParserSpec.hs view
@@ -46,6 +46,7 @@ -------------------------------------------------------------------------------- import HNormalise.Lmod.Parser import HNormalise.Lmod.Internal+import HNormalise.Torque.Internal -------------------------------------------------------------------------------- main :: IO ()@@ -53,23 +54,54 @@ -------------------------------------------------------------------------------- spec :: Spec-spec = do+spec = describe "parseLmodInfo" $ do it "parse regular info" $ do- let s = "username=someuser, cluster=myspace, jobid=myjobid" :: Text- s ~> parseLmodInfo `shouldParse` LmodInfo { username = "someuser" , cluster = "myspace" , jobid = "myjobid" }+ let s = "username=someuser, cluster=myspace, jobid=11[1].mymaster.mycluster.mydomain" :: Text+ s ~> parseLmodInfo `shouldParse` LmodInfo+ { username = "someuser"+ , cluster = "myspace"+ , jobid = Just TorqueJobName+ { number = 11+ , arrayId = Just 1+ , master = "mymaster"+ , cluster = "mycluster"+ }+ } it "parse module" $ do let s = "module=HNormalise/0.2.0.0-ghc-8.0.2," :: Text s ~> parseLmodModule `shouldParse` LmodModule { name = "HNormalise", version = "0.2.0.0-ghc-8.0.2" } + it "parse module load with empty jobid" $ do+ let s = "lmod:: username=myuser, cluster=mycluster, jobid=, userload=yes, module=cluster/.mycluster, fn=/etc/modulefiles/vsc/cluster/.banette.lua" :: Text+ s ~> parseLmodLoad `shouldParse` ("lmod", LmodLoadParse LmodLoad+ { info = LmodInfo+ { username = "myuser"+ , cluster = "mycluster"+ , jobid = Nothing+ }+ , userload = True+ , modul = LmodModule+ { name = "cluster"+ , version = ".mycluster"+ }+ , filename = "/etc/modulefiles/vsc/cluster/.banette.lua"+ })++ it "parse module load" $ do let s = "lmod:: username=myuser, cluster=mycluster, jobid=3230905.master.mycluster.mydomain, userload=yes, module=GSL/2.3-intel-2016b, fn=/apps/gent/CO7/sandybridge/modules/all/GSL/2.3-intel-2016b" :: Text s ~> parseLmodLoad `shouldParse` ("lmod", LmodLoadParse LmodLoad { info = LmodInfo { username = "myuser" , cluster = "mycluster"- , jobid = "3230905.master.mycluster.mydomain"+ , jobid = Just TorqueJobName+ { number = 3230905+ , arrayId = Nothing+ , master = "master"+ , cluster = "mycluster"+ } } , userload = True , modul = LmodModule@@ -85,7 +117,12 @@ { info = LmodInfo { username = "myuser" , cluster = "mycluster"- , jobid = "132.mymaster.mycluster.mydomain"+ , jobid = Just TorqueJobName+ { number = 132+ , arrayId = Nothing+ , master = "mymaster"+ , cluster = "mycluster"+ } } , command = "load" , arguments = "cluster/othercluster"
test/HNormalise/ParserSpec.hs view
@@ -61,11 +61,12 @@ it "lmod load" $ do let s = "<13>1 2016-06-07T17:50:22.658452+02:00 node2159 lmod: - lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016" :: Text s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse`- "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":\"434.master.mycluster.mydomain\"},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}"+ "{\"message\":\"lmod:: username=myuser, cluster=dmycluster, jobid=434.master.mycluster.mydomain, userload=yes, module=intel/2016a, fn=/apps/gent/SL6/sandybridge/modules/all/intel/2016\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2159\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"dmycluster\",\"jobid\":{\"number\":434,\"arrayId\":null,\"master\":\"master\",\"cluster\":\"mycluster\"}},\"userload\":true,\"module\":{\"name\":\"intel\",\"version\":\"2016a\"},\"filename\":\"/apps/gent/SL6/sandybridge/modules/all/intel/2016\"}}" it "lmod command" $ do let s = "<13>1 2017-10-19T21:38:22.533439+02:00 node2801 lmod: - lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster" :: Text- s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` "{\"message\":\"lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2801\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"mycluster\",\"jobid\":\"132.mymaster.mycluster.mydomain\"},\"command\":\"load\",\"arguments\":\"cluster/othercluster\"}}"+ s ~> (encodeNormalisedRsyslog `fmap` parseRsyslogLogstashString Nothing) `shouldParse` + "{\"message\":\"lmod:: username=myuser, cluster=mycluster, jobid=132.mymaster.mycluster.mydomain, cmd=load, args=cluster/othercluster\",\"syslog_abspri\":13,\"syslog_version\":1,\"program\":\"lmod\",\"@source_host\":\"node2801\",\"lmod\":{\"info\":{\"username\":\"myuser\",\"cluster\":\"mycluster\",\"jobid\":{\"number\":132,\"arrayId\":null,\"master\":\"mymaster\",\"cluster\":\"mycluster\"}},\"command\":\"load\",\"arguments\":\"cluster/othercluster\"}}" it "imfile torque input" $ do let s = "<133>1 2017-05-24T18:01:53.367275+02:00 test2802 torque - torque: 01/25/2017 15:04:10;E;0.mymaster.somepokemon.mydomain;user=huppelde group=huppelde jobname=STDIN queue=short ctime=1485350399 qtime=1485350399 etime=1485350399 start=1485350407 owner=huppelde@mymachine.mydomain.com exec_host=node2801.somepokemon.mydomain/0 Resource_List.walltime=01:00:00 Resource_List.vmem=4224531456b Resource_List.nodect=1 Resource_List.nodes=1 Resource_List.neednodes=1 Resource_List.nice=0 session=22598 total_execution_slots=1 unique_node_count=1 end=1485353050 Exit_status=265 resources_used.cput=0 resources_used.energy_used=0 resources_used.mem=31032kb resources_used.vmem=1541612kb resources_used.walltime=00:44:04" :: Text
test/HNormalise/Torque/ParserSpec.hs view
@@ -132,11 +132,11 @@ describe "parseTorqueJobName" $ do it "parse regular torque job name" $ do let s = "123456789.master.mycluster.mydomain;" :: Text- s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456789, arrayId = Nothing, master = "master", cluster = "mycluster" }+ s ~> parseTorqueJobName ";" `shouldParse` TorqueJobName { number = 123456789, arrayId = Nothing, master = "master", cluster = "mycluster" } it "parse array torque job name" $ do let s = "123456[789].master.mycluster.mydomain;" :: Text- s ~> parseTorqueJobName `shouldParse` TorqueJobName { number = 123456, arrayId = Just 789, master = "master", cluster = "mycluster" }+ s ~> parseTorqueJobName ";" `shouldParse` TorqueJobName { number = 123456, arrayId = Just 789, master = "master", cluster = "mycluster" } describe "parseTorqueResourceRequest" $ do it "parse mandatory fields in expected order" $ do