diff --git a/app/Main.hs b/app/Main.hs new file mode 100644 --- /dev/null +++ b/app/Main.hs @@ -0,0 +1,458 @@ +{-# language BangPatterns #-} +{-# language DeriveGeneric #-} +{-# language DuplicateRecordFields #-} +{-# language LambdaCase #-} +{-# language NamedFieldPuns #-} +{-# language NumericUnderscores #-} +{-# language OverloadedRecordDot #-} +{-# language OverloadedStrings #-} +{-# language PatternSynonyms #-} +{-# language ScopedTypeVariables #-} + +import Control.Exception (catchJust) +import Control.Monad.ST.Run (runByteArrayST) +import Control.Monad.Trans.State.Strict (State) +import Data.Bits ((.&.)) +import Data.Bool (bool) +import Data.ByteString (ByteString) +import Data.Bytes (Bytes) +import Data.Bytes.Builder (Builder) +import Data.Bytes.Chunks (Chunks(ChunksNil,ChunksCons)) +import Data.Char (toUpper) +import Data.Foldable (for_) +import Data.Int (Int64) +import Data.Map.Strict (Map) +import Data.Primitive (ByteArray,SmallMutableArray) +import Data.Primitive (SmallArray) +import Data.Primitive.PrimVar (PrimVar) +import Data.Primitive.PrimVar (readPrimVar,writePrimVar,newPrimVar) +import Data.Text (Text) +import Data.Text.Short (ShortText) +import Data.WideWord (Word128(Word128)) +import Foreign.C.Types (CChar) +import GHC.Exts (RealWorld) +import GHC.Generics (Generic) +import Json (pattern (:->)) +import Net.Types (IP(IP),IPv6(IPv6)) +import Panos.Syslog (Log(LogTraffic),decode) +import Panos.Syslog (Traffic) +import System.Directory (createDirectoryIfMissing) +import System.IO (Handle) +import System.IO.Error (isEOFError) + +import qualified Data.Primitive.ByteArray.BigEndian as BigEndian +import qualified Codec.Compression.Zlib.Raw as Deflate +import qualified Data.Primitive as PM +import qualified Data.Primitive.Contiguous as Contiguous +import qualified Data.Map.Strict as Map +import qualified Panos.Syslog.Traffic as Traffic +import qualified Data.Primitive.Ptr as PM +import qualified Data.Bytes.Text.Ascii as Ascii +import qualified Data.ByteString as ByteString +import qualified Data.Bytes as Bytes +import qualified Chronos +import qualified Data.Bytes.Chunks as Chunks +import qualified Data.Bytes.Builder as Builder +import qualified Data.Bytes.Builder.Avro as Avro +import qualified Data.Text as T +import qualified Data.Text.Short as TS +import qualified Control.Monad.Trans.State.Strict as State +import qualified System.IO as IO +import qualified Json +import qualified Data.ByteString.Lazy as LBS +import qualified GHC.Exts as Exts +import qualified Net.IP as IP +import qualified Options.Generic as Opts + +data Nullability = Nullable | NonNullable + +data Patchedness = Patched | Unpatched + +data Atom + = Signed32 + | Signed64 + | Ip + | String + | Timestamp + +data Ty + = Record -- We do not allow records to be nullable + [Field] + | Atomic Nullability Atom + +data Field = Field + { name :: !ShortText + -- ^ Field name must not be the empty string + , ty :: !Ty + } + +-- x fieldToJson :: Field -> Json.Value +-- x fieldToJson s = case s.ty of +-- x Record snext -> Json.object3 +-- x ("name" :-> s.name) +-- x ("type" :-> Json.shortText "record") +-- x ("fields" :-> schemaToJson snext) + +makeRecordType :: ShortText -> SmallArray Json.Value -> Json.Value +makeRecordType tyName fields = Json.object3 + ("type" :-> Json.shortText "record") + ("name" :-> Json.shortText tyName) + ("fields" :-> Json.Array fields) + +encodeAtom :: Atom -> Json.Value +encodeAtom = \case + Timestamp -> Json.object2 + ("type" :-> Json.shortText "long") + ("logicalType" :-> Json.shortText "timestamp-millis") + Signed64 -> Json.shortText "long" + Signed32 -> Json.shortText "int" + String -> Json.shortText "string" + Ip -> Json.shortText "exts.IP" + +encodeAtomWithNullability :: Atom -> Nullability -> Json.Value +encodeAtomWithNullability a n = case n of + NonNullable -> enc + Nullable -> Json.Array (Contiguous.doubleton (Json.shortText "null") enc) + where + enc = encodeAtom a + +-- In this, the prefix is only used to name record components. +fieldToJson :: ShortText -> Field -> Json.Value +fieldToJson prefix x = case x.ty of + Record children -> + let titleName = TS.fromText (uppercaseHead (TS.toText x.name)) + fullName = prefix <> titleName + convertedFields = Exts.fromList (fmap (fieldToJson fullName) children) + in Json.object2 + ("type" :-> makeRecordType fullName convertedFields) + ("name" :-> Json.shortText x.name) + Atomic nullability atom -> + let memberName = "name" :-> Json.shortText x.name + memberType = "type" :-> encodeAtomWithNullability atom nullability + in Json.object2 memberName memberType + +-- This converts the nesting source.nat.ip to SourceNatIp +fieldToFlatJson :: Text -> Field -> [Json.Value] +fieldToFlatJson prefix x = case x.ty of + Record children -> fieldToFlatJson (prefix <> uppercaseHead (TS.toText x.name)) =<< children + Atomic nullability atom -> + let memberName = "name" :-> Json.text (prefix <> uppercaseHead (TS.toText x.name)) + memberType = "type" :-> encodeAtomWithNullability atom nullability + in pure (Json.object2 memberName memberType) + +uppercaseHead :: Text -> Text +uppercaseHead t = case T.uncons t of + Just (c,cs) -> T.cons (toUpper c) cs + Nothing -> T.empty + +replaceIpWithS64 :: [Field] -> [Field] +replaceIpWithS64 = map replaceIpWithS64Single + +replaceIpWithS64Single :: Field -> Field +replaceIpWithS64Single x = case x.ty of + Record children -> Field x.name (Record (replaceIpWithS64 children)) + Atomic nullability atom -> case atom of + Ip -> Field x.name (Atomic nullability Signed64) + _ -> Field x.name (Atomic nullability atom) + +theSchema :: [Field] +theSchema = + [ Field "timestamp" $ Atomic NonNullable Timestamp + , Field "chronopartition" $ Atomic NonNullable Signed32 + , Field "network" $ Record + [ Field "application" $ Atomic Nullable String + , Field "iana_number" $ Atomic NonNullable Signed32 + ] + , Field "source" $ Record + [ Field "ip" $ Atomic NonNullable Ip + , Field "port" $ Atomic NonNullable Signed32 + , Field "packets" $ Atomic NonNullable Signed64 + , Field "bytes" $ Atomic NonNullable Signed64 + , Field "user" $ Record + [ Field "original" $ Atomic Nullable String + ] + ] + , Field "destination" $ Record + [ Field "ip" $ Atomic NonNullable Ip + , Field "port" $ Atomic NonNullable Signed32 + , Field "packets" $ Atomic NonNullable Signed64 + , Field "bytes" $ Atomic NonNullable Signed64 + , Field "user" $ Record + [ Field "original" $ Atomic Nullable String + ] + ] + ] + +-- Avro is weird about how users have to use the "fixed" type. You have +-- to provide a "name" for it so that a Java code generation tool can +-- use that as the class name. You cannot reuse the same name more than +-- once. And you cannot define all the names in advance. You have to define +-- them inline as you are defining the fields for a document. So, this +-- function is a hack to work around this strange limitation. Here, we +-- find the first occurrence of "exts.IP" and replace it with a definition +-- that names that type. All future occurrences then refer to the original. +patchFirstExtsIP :: Json.Value -> Json.Value +patchFirstExtsIP v0 = State.evalState (go v0) Unpatched + where + go :: Json.Value -> State Patchedness Json.Value + go v = State.get >>= \case + Patched -> pure v + Unpatched -> case v of + Json.Array xs -> Json.Array <$> traverse go xs + Json.Object mbrs -> Json.Object + <$> traverse (\Json.Member{key,value} -> Json.Member key <$> go value) mbrs + Json.String "exts.IP" -> do + State.put Patched + pure $ Json.object4 + ("name" :-> Json.shortText "IP") + ("namespace" :-> Json.shortText "exts") + ("type" :-> Json.shortText "fixed") + ("size" :-> Json.int 16) + x -> pure x + +data Settings = Settings + { nested :: !Bool + , ipAsSigned64 :: !Bool + , timeBucket :: !(Maybe Text) + , compress :: !Bool + } + deriving (Generic, Show) + +instance Opts.ParseRecord Settings + +data Compression = CompressionNone | CompressionDeflate + +main :: IO () +main = do + Settings{nested,ipAsSigned64,timeBucket=timeBucketStr,compress} <- Opts.getRecord "pan-os-syslog-to-avro" + timeBucket :: Int64 <- case timeBucketStr of + Just "1d" -> pure 86400 + Just "24h" -> pure 86400 + Just "1h" -> pure 3600 + Just "60m" -> pure 3600 + Just "5m" -> pure 300 + Just "none" -> pure 0 + Nothing -> pure 0 + _ -> fail "Unsupported timeBucket. Try: 5m, 60m, 1d" + let compression = case compress of + True -> CompressionDeflate + False -> CompressionNone + let ipEncoding = case ipAsSigned64 of + True -> IpEncodingS64 + False -> IpEncodingU128 + let theSchema' = case ipAsSigned64 of + True -> replaceIpWithS64 theSchema + False -> theSchema + let encodedSchema = Builder.run 512 $ Json.encode $ patchFirstExtsIP $ case nested of + False -> Json.object3 + ("type" :-> Json.shortText "record") + ("name" :-> Json.shortText "docroot") + ("fields" :-> Json.Array (Exts.fromList (fieldToFlatJson T.empty =<< theSchema'))) + True -> Json.object3 + ("type" :-> Json.shortText "record") + ("name" :-> Json.shortText "docroot") + ("fields" :-> Json.Array (Exts.fromList (map (fieldToJson "") theSchema'))) + let syncMarker@(Word128 syncA syncB) = 0xabcd_0123_9876_fedc_4567_4321_3456_cdef + putStrLn "Schema" + putStrLn "======" + Chunks.hPut IO.stdout encodedSchema + putStrLn "" + putStrLn "Sync Marker" + putStrLn "===========" + Chunks.hPut IO.stdout + ( Builder.run 16 + (Builder.word64PaddedUpperHex syncA <> Builder.word64PaddedUpperHex syncB) + ) + putStrLn "" + sinks <- handleUntilEof timeBucket compression encodedSchema syncMarker ipEncoding + for_ sinks $ \Sink{handle=h,buffer,position} -> do + ix <- readPrimVar position + case ix of + 0 -> pure () + _ -> do + dst' <- PM.freezeSmallArray buffer 0 ix + Chunks.hPut h (encodeTrafficLogBatch compression ipEncoding syncMarker dst') + IO.hClose h + pure () + +pushAvroHeader :: + Compression + -> Word128 -- sync marker + -> Chunks -- encoded schema + -> Handle -- sink + -> IO () +pushAvroHeader !compression !syncMarker !encSchema !sink = Chunks.hPut sink $ Builder.run 1024 $ + Builder.ascii4 'O' 'b' 'j' '\x01' + <> + Avro.map2 + "avro.schema" + (Avro.chunks encSchema) + "avro.codec" + (Avro.text (case compression of {CompressionNone -> "null"; CompressionDeflate -> "deflate"})) + <> + Avro.word128 syncMarker + +data Sink = Sink + { buffer :: !(SmallMutableArray RealWorld Traffic) + , position :: !(PrimVar RealWorld Int) + , handle :: !Handle + } + +initializeSinkIfNotExists :: Compression -> Word128 -> Chunks -> Map Int64 Sink -> Int64 -> IO (Sink, Map Int64 Sink) +initializeSinkIfNotExists !compression !syncMarker !encSchema !sinks !k = case Map.lookup k sinks of + Nothing -> do + buffer <- PM.newSmallArray batchSize errorThunk + position <- newPrimVar 0 + createDirectoryIfMissing False "output" + let partitionDir = ("output/chronopartition=" ++ show k) + createDirectoryIfMissing False partitionDir + h <- IO.openFile (partitionDir ++ "/data.avro") IO.WriteMode + pushAvroHeader compression syncMarker encSchema h + let newSink = Sink{buffer,position,handle=h} + let sinks' = Map.insert k newSink sinks + pure (newSink, sinks') + Just s -> pure (s,sinks) + +computeBucket :: + Int64 + -> Int64 -- seconds since epoch + -> Int64 +computeBucket !sz !seconds = case sz of + 0 -> 0 + _ -> div (sz * div seconds sz) 300 + +handleUntilEof :: + Int64 -- time bucket + -> Compression -- compression codec + -> Chunks -- encoded schema + -> Word128 -- sync marker + -> IpEncoding + -> IO (Map Int64 Sink) +handleUntilEof !bucketSize !compression !encSchema !syncMarker !ipEncoding = do + let go :: Map Int64 Sink -> Int -> IO (Map Int64 Sink) + go !sinks !fileIx = catchJust (bool Nothing (Just ()) . isEOFError) (Just <$> ByteString.getLine) (\() -> pure Nothing) >>= \case + Nothing -> pure sinks + Just b0 -> do + b1 <- b2b b0 + case decode (Bytes.fromByteArray b1) of + Right (LogTraffic tr) -> do + let secondsSinceEpoch = div (Chronos.getTime (Chronos.datetimeToTime (Traffic.timeGenerated tr))) 1_000_000_000 + let bucket = computeBucket bucketSize secondsSinceEpoch + (Sink{buffer=dst,position,handle=h},sinks') <- initializeSinkIfNotExists compression syncMarker encSchema sinks bucket + !dstIx0 <- readPrimVar position + !dstIx <- if dstIx0 == batchSize + then do + dst' <- PM.freezeSmallArray dst 0 batchSize + Chunks.hPut h (encodeTrafficLogBatch compression ipEncoding syncMarker dst') + pure 0 + else pure dstIx0 + case ipEncoding of + IpEncodingU128 + | isKnownIpProtocol (Traffic.ipProtocol tr) -> do + PM.writeSmallArray dst dstIx tr + writePrimVar position (dstIx + 1) + go sinks (fileIx + 1) + | otherwise -> go sinks (fileIx + 1) + IpEncodingS64 + | IP.isIPv4 (Traffic.sourceAddress tr) + , IP.isIPv4 (Traffic.destinationAddress tr) + , isKnownIpProtocol (Traffic.ipProtocol tr) -> do + PM.writeSmallArray dst dstIx tr + writePrimVar position (dstIx + 1) + go sinks' (fileIx + 1) + | otherwise -> go sinks' (fileIx + 1) + Right _ -> go sinks (fileIx + 1) + Left err -> fail ("On line " ++ show fileIx ++ ", " ++ show err) + go Map.empty (0 :: Int) + +batchSize :: Int +batchSize = 8192 + +isKnownIpProtocol :: Bytes -> Bool +isKnownIpProtocol !proto = proto == Ascii.fromString "tcp" || proto == Ascii.fromString "udp" + +data IpEncoding = IpEncodingU128 | IpEncodingS64 + +syncMarkerToBytes :: Word128 -> Bytes +syncMarkerToBytes !w = Bytes.fromByteArray $ runByteArrayST $ do + dst <- PM.newByteArray 16 + BigEndian.writeByteArray dst 0 w + PM.unsafeFreezeByteArray dst + +encodeTrafficLogBatch :: Compression -> IpEncoding -> Word128 -> SmallArray Traffic -> Chunks +encodeTrafficLogBatch !compression !ipEnc !syncMarker !xs = + -- Note: we have to subtract the length of the sync marker from + -- the payload length. + let payload = Builder.run 512 (foldMap (encodeTrafficLog ipEnc) xs) + in case compression of + CompressionNone -> + Builder.runOnto 128 + (Avro.int (PM.sizeofSmallArray xs) <> Avro.int (Chunks.length payload)) + (payload <> ChunksCons (syncMarkerToBytes syncMarker) ChunksNil) + CompressionDeflate -> + let compressedPayload = Deflate.compress (LBS.fromStrict (Chunks.concatByteString payload)) + in Builder.runOnto 128 + (Avro.int (PM.sizeofSmallArray xs) <> Avro.int64 (LBS.length compressedPayload)) + (ChunksCons (Bytes.fromLazyByteString compressedPayload) (ChunksCons (syncMarkerToBytes syncMarker) ChunksNil)) + +encodeIp :: IpEncoding -> IP -> Builder +encodeIp !enc !ip = case enc of + IpEncodingS64 -> Avro.int64 (extractIPv4 ip) + IpEncodingU128 -> Avro.word128 (ipToW128 ip) + +encodeTrafficLog :: IpEncoding -> Traffic -> Builder +encodeTrafficLog ipEnc t = + let millisecondsSinceEpoch = div (Chronos.getTime (Chronos.datetimeToTime (Traffic.timeGenerated t))) 1_000_000 in + Avro.int64 millisecondsSinceEpoch + <> + Avro.int64 (div millisecondsSinceEpoch 300_000) + <> + encodeNullableString Traffic.application t + <> + Avro.int32 (if Traffic.ipProtocol t == Ascii.fromString "tcp" then 6 else 17) + <> + encodeIp ipEnc (Traffic.sourceAddress t) + <> + Avro.word16 (Traffic.sourcePort t) + <> + Avro.int64 (fromIntegral (Traffic.packetsReceived t)) + <> + Avro.int64 (fromIntegral (Traffic.bytesReceived t)) + <> + encodeNullableString Traffic.sourceUser t + <> + encodeIp ipEnc (Traffic.destinationAddress t) + <> + Avro.word16 (Traffic.destinationPort t) + <> + Avro.int64 (fromIntegral (Traffic.packetsSent t)) + <> + Avro.int64 (fromIntegral (Traffic.bytesSent t)) + <> + encodeNullableString Traffic.destinationUser t + +extractIPv4 :: IP -> Int64 +extractIPv4 (IP (IPv6 (Word128 _ b))) = fromIntegral (b .&. 0x0000_0000_FFFF_FFFF) + +encodeNullableString :: (Traffic -> Bytes) -> Traffic -> Builder +encodeNullableString project t = + let x = project t + in case Bytes.null x of + True -> Builder.word8 0x00 + False -> Builder.word8 0x02 <> Avro.bytes x + +ipToW128 :: IP -> Word128 +ipToW128 (IP (IPv6 w)) = w + +errorThunk :: Traffic +{-# noinline errorThunk #-} +errorThunk = errorWithoutStackTrace "pan-os-syslog-to-avro: implementation mistake" + +b2b :: ByteString -> IO ByteArray +b2b !b = ByteString.useAsCStringLen b $ \(ptr,len) -> do + arr <- PM.newByteArray len + PM.copyPtrToMutablePrimArray (castArray arr) 0 ptr len + PM.unsafeFreezeByteArray arr + +castArray :: PM.MutableByteArray s -> PM.MutablePrimArray s CChar +castArray (PM.MutableByteArray x) = PM.MutablePrimArray x diff --git a/common/Sample.hs b/common/Sample.hs --- a/common/Sample.hs +++ b/common/Sample.hs @@ -1,8 +1,11 @@ {-# language TypeApplications #-} module Sample - ( traffic_8_1_A + ( correlation_A + , traffic_8_1_A , traffic_8_1_B + , traffic_9_0_A + , traffic_prisma_A , threat_8_1_A , threat_8_1_B , threat_8_1_C @@ -12,7 +15,14 @@ , threat_8_1_G , threat_8_1_H , threat_8_1_I + , threat_9_0_A + , threat_9_1_A + , threat_9_1_B + , threat_prisma_A + , threat_prisma_B + , threat_prisma_C , system_8_1_A + , user_A ) where import Data.Bytes (Bytes) @@ -63,6 +73,35 @@ , "0,0,0,0" ] +traffic_9_0_A :: Bytes +traffic_9_0_A = pack $ concat + [ "<14> Mar 9 14:13:44 NY-PAN-FW-5.example.com 1,2020/03/09 14:13:44," + , "019624168632,TRAFFIC,end,2304,2020/03/09 14:13:44,192.0.2.73," + , "192.0.2.117,0.0.0.0,0.0.0.0,NEAR to FAR,,,insufficient-data," + , "vsys1,NEAR-ZONE,FAR-ZONE,tunnel.163,ethernet1/4,Forward-The-Logs," + , "2020/03/09 14:13:44,385404,1,56451,8475,0,0,0x401c,tcp,allow,1760," + , "1051,709,19,2020/03/09 14:13:24,16,any,0,6412095715,0x0,United States," + , "10.0.0.0-10.255.255.255,0,12,7,tcp-fin,11,0,0,0,,NY-PAN-FW-5," + , "my-policy,,,0,,0,,N/A,0,0,0,0,321ef4bf-801e-1c89-b341-efb2898ba2be,0" + ] + +-- This is PAN-OS 10.x. +traffic_prisma_A :: Bytes +traffic_prisma_A = pack $ concat + [ "<14>1 2021-10-27T19:21:00.034Z stream-logfwd20-548106107-12876850-z5tm-harness-44k2 " + , "logforwarder - panwlogs - 2021-10-27T19:20:59.000000Z,no-serial,TRAFFIC," + , "end,10.0,2021-10-27T19:20:40.000000Z,192.0.2.12,192.0.2.18," + , "192.0.2.112,192.0.2.118,INSIDE-TO-OUTSIDE,,," + , "web-browsing,vsys1,Zone-One,Zone-Two,tunnel.101," + , "ethernet1/1,Send-The-Logs,170172,1,42493,80,46254,80,tcp," + , "allow,892,818,74,12,2021-10-27T19:19:08.000000Z,1,low-risk," + , "9091497,10.0.0.0-10.255.255.255,JP,11,1,threat,28,31,0,0,," + , "The-Device-Name,from-policy,,,0,,0," + , "1970-01-01T00:00:00.000000Z,N/A,0,0,0,0," + , "89092ab0-606f-4c5a-b44c-c271420f3972,0,0,,,,,,,,,,,,,,,,,,," + , ",,,,,,,,,,,,,,,,2021-10-27T19:20:41.652000Z,," + ] + -- Threat log for web browsing threat_8_1_A :: Bytes threat_8_1_A = pack $ concat @@ -215,6 +254,80 @@ , "AppThreat-3218-3729,0x0,0,4294967295," ] +-- Web browsing threat log from PAN-OS 9.0 +threat_9_0_A :: Bytes +threat_9_0_A = pack $ concat + [ "Mar 9 14:47:44 firewall1.example.com 1,2020/03/09 14:47:44,001701012545," + , "THREAT,url,2304,2020/03/09 14:47:44,192.0.2.101,192.0.2.102," + , "192.0.2.103,192.0.2.104,FOO to BAR,example\\jdoe,,ssl,vsys1,FOO,BAR," + , "ethernet1/6,ethernet1/7,My-Log-Forwarding,2020/03/09 14:47:44,455102," + , "1,62475,443,31963,443,0x40f000,tcp,alert,\"dt.adsafeprotected.com/\"," + , "(9999),web-advertisements,informational,client-to-server,2314781488," + , "0x2000000000000000,United States,United States,0,,0,,,0,,,,,,,,0,11," + , "0,0,0,,firewall1,,,,,0,,0,,N/A,unknown,AppThreat-0-0,0x0,0,4294967295," + , ",\"web-advertisements,low-risk\",edd29e10-d927-1753-867f-0108b01b80de,0" + ] + +threat_prisma_A :: Bytes +threat_prisma_A = pack $ concat + [ "<14>1 2021-10-28T02:58:53.586Z stream-foobar-baz logforwarder - " + , "panwlogs - 2021-10-28T02:58:52.000000Z,no-serial,THREAT,url," + , "10.0,2021-10-28T02:58:50.000000Z,192.0.2.99,190.0.2.100,192.0.2.101," + , "192.0.2.102,IN-TO-OUT,,,google-play,vsys1,trust,untrust,tunnel.105," + , "ethernet1/1,The-Forward,28182,1,59580,443,3502,443,tcp,block-url," + , "play.google.com/,shopping,Informational,client to server,2719193," + , "10.0.0.0-10.255.255.255,US,,0,0,,,,28,31,0,0,,big-ol-device,,," + , "unknown,0,,0,1970-01-01T00:00:00.000000Z,N/A,unknown,0,0,," + , "\"shopping,low-risk\",89ea2a40-656e-4c5a-b44c-c27b52de3982,0," + , ",,,,,,,,,,,,,,,,,,,,,,,,,,,2021-10-28T02:58:50.719000Z," + ] + +threat_prisma_B :: Bytes +threat_prisma_B = pack $ concat + [ "<14>1 2021-11-01T15:14:08.069Z stream-logfwd-xyz logforwarder - panwlogs " + , "- 2021-11-01T15:14:06.000000Z,no-serial,THREAT,spyware,10.0," + , "2021-11-01T15:13:56.000000Z,192.0.2.20,8.8.8.8,192.0.2.111,8.8.8.8," + , "in-to-out-dns,domain\\exampleuser,,dns,vsys1,ZoneA,ZoneB,tunnel.101," + , "ethernet1/1,My-Forwarding,1355037,1,55517,53,45783,53,udp,drop," + , "example.com,Phishing:example.com(109010001),Low,client-to-server," + , "12088427,10.0.0.0-10.255.255.255,US,1152921504606899096,,,0,,,,,0," + , "28,31,0,0,,Big-Firewall,,,0,,0,1970-01-01T00:00:00.000000Z,N/A," + , "dns-phishing,0,0x0,ceb0229f-7a4a-5c62-8960-f66b27e2e01e,0," + , ",,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2021-11-01T15:13:57.399000Z," + ] + +threat_prisma_C :: Bytes +threat_prisma_C = pack $ concat + [ "<14>1 2021-11-02T00:53:49.421Z stream-logfwd-xyz logforwarder - panwlogs " + , "- 2021-11-02T00:53:48.000000Z,no-serial,THREAT,wildfire,10.0," + , "2021-11-02T00:53:37.000000Z,192.0.2.20,192.0.2.21,192.0.2.22,192.0.2.23," + , "in-to-out,foo\\bar,,office365-enterprise-access,vsys1,trust," + , "untrust,tunnel.101,ethernet1/1,My-Forwarding,1294093,1," + , "64810,443,32485,443,tcp,allow,Exchange.asmx," + , "Adobe Portable Document Format (PDF)(52021),Informational," + , "server to client,13548583,10.0.0.0-10.255.255.255,US,0," + , "85dde742f22216725f0ff6b172ca68f4f30f02029f08b3079cec34da804db793," + , "wildfire.paloaltonetworks.com,69,pdf,,,,47136308381,28,31,0,0,," + , "Big-Firewall,,,0,,0,1970-01-01T00:00:00.000000Z," + , "N/A,unknown,0,0x0,f0ebabfb-ef45-451a-89a5-30f87238d62b," + , "0,,,,,,,,,,,,,,,,,,,,,,,,,,,,,0,2021-11-02T00:53:37.797000Z," + ] + + +-- This is a log from a firewall whose license had expired. +threat_9_1_A :: Bytes +threat_9_1_A = pack $ concat + [ "<14>Mar 2 11:30:47 MY-PA-5250-3 1,2021/03/02 11:30:47,012911492893," + , "THREAT,url,2305,2021/03/02 11:30:47,192.0.2.22,192.0.2.17,0.0.0.0," + , "0.0.0.0,rule99,foo\\_bar,,ssl,vsys1,MY-trust,MY-dmz,ethernet1/1," + , "ethernet1/2,My-Log-Forwarding,2021/03/02 11:30:47,38578210,1,63421," + , "5986,0,0,0x10b000,tcp,allow,\"192.0.2.17_solarwinds_zero_configuration:5986/\"," + , "(9999),license-expired,informational,client-to-server,4901795984744840635," + , "0xa000000000000000,10.0.0.0-10.255.255.255,10.0.0.0-10.255.255.255,0,,0,,,0," + , ",,,,,,,0,56,0,0,0,,MY-PA-5250-1,,,,,0,,0,,N/A,unknown,AppThreat-0-0,0x0,0," + , "4196987195,,\"license-expired\",9871ab80-6061-4f07-a06e-0036b0206f73,0," + ] + -- System log (IKE delete) system_8_1_A :: Bytes system_8_1_A = pack $ concat @@ -223,4 +336,43 @@ , "To-FOO-BAR-NET,0,0,general,informational,\"IKE protocol IPSec SA " , "delete message sent to peer. SPI:0xA1CD910F.\",18249042," , "0x8000000000000000,0,0,0,0,,NY-DC-FW-2" + ] + +-- User ID log (login) +user_A :: Bytes +user_A = pack $ concat + [ "1,2021/09/19 11:55:38,013201027000,USERID,login,2305,2021/09/19 11:55:38," + , "vsys1,192.0.2.112,BIGDAWG10$@EXAMPLE.COM,server.example.com,0,1,2700,0,0," + , "active-directory,,6991364276409747475,0x0,92,0,0,0,,MY-PA5220-A,1,," + , "2021/09/19 11:55:25,1,0x0,BIGDAWG10$@EXAMPLE.COM" + ] + +-- Correlation log +correlation_A :: Bytes +correlation_A = pack $ concat + [ "<14>Mar 7 14:43:42 panorama.example.com 1,2022/03/07 14:43:17,013201019829," + , "CORRELATION,,,2022/03/07 14:43:17,192.0.2.75,,,compromised-host,medium," + , "146,0,0,0,,panorama,1869504768,Beacon Detection,6005,Host visited known " + , "malware URL (11 times)." + ] + +-- Threat log +threat_9_1_B :: Bytes +threat_9_1_B = pack $ concat + [ "<14>Jul 25 09:39:39 Panorama 1,2022/07/25 09:39:39,012801204178," + , "THREAT,url,2560,2022/07/25 09:40:03,192.0.2.254,192.0.2.201," + , "192.0.2.13,192.0.2.12,trust-to-untrust-allow-any,,,ssl,vsys1," + , "ABC-trust,ABC-untrust,ethernet1/2.14,ethernet1/3," + , "Forward_To_Panorama,2022/07/25 09:40:03,31669,1,53686,443,15492," + , "443,0x403400,tcp,block-url,\"t.co/\",(9999),social-networking," + , "informational,client-to-server,30720475,0xa000000000000000," + , "10.0.0.0-10.255.255.255,United States,,,0,,,0,,,,,,,,0,15,0,0,0," + , ",NYC-ABC-PA-220,,,,,0,,0,,N/A,unknown,AppThreat-0-0,0x0,0," + , "4294967295,,\"social-networking,low-risk\"," + , "a40f8ec5-109d-4dc5-b09d-33b848f5bd09,0,,0.0.0.0," + , ",,,,,,,,,,,,,,,,,,,,,,,,,,0,2022-07-25T09:40:03.909-04:00," + , ",,,encrypted-tunnel,networking,browser-based," + , "4,\"used-by-malware,able-to-transfer-file," + , "has-known-vulnerability,tunnel-other-application,pervasive-use\"," + , ",ssl,no,no" ] diff --git a/pan-os-syslog.cabal b/pan-os-syslog.cabal --- a/pan-os-syslog.cabal +++ b/pan-os-syslog.cabal @@ -1,6 +1,6 @@ cabal-version: 2.2 name: pan-os-syslog -version: 0.1.0.0 +version: 0.2.0.0 synopsis: Parse syslog traffic from PAN-OS description: Parse syslog traffic from PAN-OS. The data types in this library are @@ -44,20 +44,61 @@ library exposed-modules: Panos.Syslog + Panos.Syslog.Correlation Panos.Syslog.System - Panos.Syslog.Traffic Panos.Syslog.Threat + Panos.Syslog.Traffic Panos.Syslog.Unsafe + Panos.Syslog.User + other-modules: + -- These were split into separate modules to improve compile times. + -- GHC was using 8GB of memory to compile the megamodule that used + -- to contain them. + Panos.Syslog.Internal.Common + Panos.Syslog.Internal.Correlation + Panos.Syslog.Internal.System + Panos.Syslog.Internal.Threat + Panos.Syslog.Internal.Traffic + Panos.Syslog.Internal.User build-depends: , base >=4.12.0.0 && <5 , byteslice >=0.1.3 && <0.3 , bytesmith >=0.3.1 && <0.4 - , chronos >=1.0.6 && <1.1 + , chronos >=1.1.3 && <1.2 , ip >=1.6 && <1.8 - , primitive >=0.7 && <0.8 + , primitive >=0.7 && <0.10 , primitive-addr >=0.1.0.2 && <2 , run-st >=0.1 && <0.2 + , uuid-bytes >=0.1.1 && <0.2 + , wide-word >=0.1.0.9 && <0.2 hs-source-dirs: src + ghc-options: -Wall -O2 + default-language: Haskell2010 + +executable pan-os-syslog-to-avro + hs-source-dirs: app + main-is: Main.hs + build-depends: + , base + , pan-os-syslog + , primitive >=0.9 + , byteslice + , optparse-generic >=1.5.2 + , bytebuild >=0.3.14 + , json-syntax >=0.2.7 + , bytestring >=0.11.5.3 + , text-short >=0.1.5 + , text >=2.0 + , contiguous >=0.6.4 + , wide-word >=0.1.5 + , transformers >=0.6.1 + , ip >= 1.7.6 + , chronos >=1.1.5 + , containers >=0.6 + , zlib >=0.6.3 + , byte-order >=0.1.3 + , run-st >=0.1.3.2 + , directory >=1.3.8.1 ghc-options: -Wall -O2 default-language: Haskell2010 diff --git a/src/Panos/Syslog.hs b/src/Panos/Syslog.hs --- a/src/Panos/Syslog.hs +++ b/src/Panos/Syslog.hs @@ -4,6 +4,8 @@ , U.Traffic , U.Threat , U.System + , U.User + , U.Correlation , U.Field -- * Decoding , U.decode diff --git a/src/Panos/Syslog/Correlation.hs b/src/Panos/Syslog/Correlation.hs new file mode 100644 --- /dev/null +++ b/src/Panos/Syslog/Correlation.hs @@ -0,0 +1,104 @@ +{-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language MagicHash #-} +{-# language NamedFieldPuns #-} +{-# language NumericUnderscores #-} +{-# language OverloadedRecordDot #-} + +-- | Fields for correlation logs. +module Panos.Syslog.Correlation + ( category + , deviceGroupHierarchyLevel1 + , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 + , deviceGroupHierarchyLevel4 + , deviceName + , evidence + , objectId + , objectName + , serialNumber + , severity + , sourceAddress + , sourceUser + , syslogHost + , timeGenerated + ) where + +import Chronos (Datetime) +import Data.Bytes.Types (Bytes(..)) +import Data.Word (Word64) +import Net.Types (IP) +import Panos.Syslog.Unsafe (Correlation(Correlation),Bounds(Bounds)) + +import qualified Panos.Syslog.Unsafe as U + +-- | The hostname of the firewall on which the session was logged. +deviceName :: Correlation -> Bytes +deviceName (Correlation{deviceName=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +objectName :: Correlation -> Bytes +objectName (Correlation{objectName=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +sourceUser :: Correlation -> Bytes +sourceUser (Correlation{sourceUser=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | Time the log was generated on the dataplane. +timeGenerated :: Correlation -> Datetime +{-# inline timeGenerated #-} +timeGenerated u = u.timeGenerated + +-- | Serial number of the firewall that generated the log. These +-- occassionally contain non-numeric characters, so do not attempt +-- to parse this as a decimal number. +serialNumber :: Correlation -> Bytes +serialNumber (Correlation{serialNumber=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | The hostname from the syslog header appended to the PAN-OS log. +-- This field is not documented by Palo Alto Network and technically +-- is not part of the log, but in practice, it is always present. +-- This is similar to @deviceName@. +syslogHost :: Correlation -> Bytes +syslogHost (Correlation{syslogHost=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +severity :: Correlation -> Bytes +severity (Correlation{severity=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +category :: Correlation -> Bytes +category (Correlation{category=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +evidence :: Correlation -> Bytes +evidence u = u.evidence + +objectId :: Correlation -> Word64 +objectId u = u.objectId + +deviceGroupHierarchyLevel1 :: Correlation -> Word64 +{-# inline deviceGroupHierarchyLevel1 #-} +deviceGroupHierarchyLevel1 u = u.deviceGroupHierarchyLevel1 + +deviceGroupHierarchyLevel2 :: Correlation -> Word64 +{-# inline deviceGroupHierarchyLevel2 #-} +deviceGroupHierarchyLevel2 u = u.deviceGroupHierarchyLevel2 + +deviceGroupHierarchyLevel3 :: Correlation -> Word64 +{-# inline deviceGroupHierarchyLevel3 #-} +deviceGroupHierarchyLevel3 u = u.deviceGroupHierarchyLevel3 + +deviceGroupHierarchyLevel4 :: Correlation -> Word64 +{-# inline deviceGroupHierarchyLevel4 #-} +deviceGroupHierarchyLevel4 u = u.deviceGroupHierarchyLevel4 + +-- | Original session source IP address. +sourceAddress :: Correlation -> IP +sourceAddress u = u.sourceAddress + + diff --git a/src/Panos/Syslog/Internal/Common.hs b/src/Panos/Syslog/Internal/Common.hs new file mode 100644 --- /dev/null +++ b/src/Panos/Syslog/Internal/Common.hs @@ -0,0 +1,857 @@ +{-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language LambdaCase #-} +{-# language MagicHash #-} +{-# language NamedFieldPuns #-} +{-# language NumericUnderscores #-} +{-# language ScopedTypeVariables #-} +module Panos.Syslog.Internal.Common + ( -- * Parsers + untilComma + , skipDigitsThroughComma + , skipThroughComma + , w64Comma + , w16Comma + , parserDatetime + , parserOptionallyQuoted + , parserOptionallyQuoted_ + , finalOptionallyQuoted + -- * Types + , Field(..) + , Bounds(..) + -- * Fields + , actionField + , actionFlagsField + , actionSourceField + , applicationField + , bytesField + , bytesReceivedField + , bytesSentField + , categoryField + , cloudField + , contentTypeField + , contentVersionField + , dataSourceField + , dataSourceNameField + , dataSourceTypeField + , descriptionField + , destinationAddressField + , destinationCountryField + , destinationPortField + , destinationUserField + , destinationVmUuidField + , destinationZoneField + , deviceGroupHierarchyLevel1Field + , deviceGroupHierarchyLevel2Field + , deviceGroupHierarchyLevel3Field + , deviceGroupHierarchyLevel4Field + , deviceNameField + , directionField + , elapsedTimeField + , eventIdField + , evidenceField + , fileDigestField + , fileTypeField + , flagsField + , forwardedForField + , futureUseAField + , futureUseBField + , futureUseCField + , futureUseDField + , futureUseEField + , futureUseFField + , futureUseGField + , http2ConnectionField + , httpHeadersField + , httpMethodField + , inboundInterfaceField + , ipField + , ipProtocolField + , leftoversField + , linkChangeCountField + , logActionField + , miscellaneousField + , moduleField + , monitorTagField + , natDestinationIpField + , natDestinationPortField + , natSourceIpField + , natSourcePortField + , objectField + , objectIdField + , objectNameField + , outboundInterfaceField + , packetsField + , packetsReceivedField + , packetsSentField + , parentSessionIdField + , parentStartTimeField + , payloadProtocolField + , pcapIdField + , prismaDataField + , receiveTimeDateField + , receiveTimeTimeField + , recipientField + , refererField + , repeatCountField + , reportIdField + , ruleNameField + , ruleUuidField + , sctpAssociationIdField + , sctpChunksField + , sctpChunksReceivedField + , sctpChunksSentField + , senderField + , sequenceNumberField + , serialNumberField + , sessionEndReasonField + , sessionIdField + , severityField + , sourceAddressField + , sourceCountryField + , sourceIpField + , sourcePortField + , sourceUserField + , sourceVmUuidField + , sourceZoneField + , startTimeDateField + , startTimeTimeField + , subjectField + , subtypeField + , syslogDatetimeField + , syslogHostField + , syslogPriorityField + , threatCategoryField + , threatIdField + , timeGeneratedDateField + , timeGeneratedTimeField + , timeoutField + , tunnelIdField + , tunnelTypeField + , typeField + , urlCategoryListField + , urlIndexField + , userAgentField + , userField + , virtualSystemField + , virtualSystemIdField + , virtualSystemNameField + ) where + +import Chronos (DayOfMonth(..),Date(..),Offset(..),OffsetDatetime(..)) +import Chronos (Year(..),Month(..),Datetime(..),TimeOfDay(..)) +import Control.Exception (Exception) +import Control.Monad.ST.Run (runByteArrayST) +import Data.Bytes.Parser (Parser) +import Data.Bytes.Types (Bytes(..),UnmanagedBytes(UnmanagedBytes)) +import Data.Char (ord) +import Data.Primitive (ByteArray) +import Data.Primitive.Addr (Addr(Addr)) +import Data.Word (Word64,Word16,Word8) +import GHC.Exts (Ptr(Ptr),Int(I#),Int#,Addr#) + +import qualified Chronos +import qualified Control.Exception +import qualified Data.Primitive as PM +import qualified Data.Primitive.Ptr as PM +import qualified Data.Bytes.Parser as P +import qualified Data.Bytes.Parser.Unsafe as Unsafe +import qualified Data.Bytes.Parser.Latin as Latin +import qualified GHC.Pack + +-- In PAN-OS 10, datetimes started being encoded with ISO-8601 +-- rather than with the YYYY/MM/DD HH:mm:ss scheme. So, we +-- allow either. +parserDatetime :: e -> e -> Parser e s Datetime +{-# noinline parserDatetime #-} +parserDatetime edate etime = do + _ <- P.take edate 4 + Latin.any edate >>= \case + '-' -> do + Unsafe.unconsume 5 + OffsetDatetime t (Offset f) <- P.orElse Chronos.parserUtf8BytesIso8601 (P.fail edate) + Latin.char etime ',' + case f of + 0 -> pure t + _ -> P.fail edate + '/' -> do + Unsafe.unconsume 5 + year <- Latin.decWord edate + Latin.char edate '/' + monthPlusOne <- Latin.decWord edate + let month = monthPlusOne - 1 + if month > 11 + then P.fail edate + else pure () + Latin.char edate '/' + day <- Latin.decWord edate + Latin.char etime ' ' + hour <- Latin.decWord etime + Latin.char etime ':' + minute <- Latin.decWord etime + Latin.char etime ':' + second <- Latin.decWord etime + Latin.char etime ',' + pure $ Datetime + (Date + (Year (fromIntegral year)) + (Month (fromIntegral month)) + (DayOfMonth (fromIntegral day)) + ) + (TimeOfDay + (fromIntegral hour) + (fromIntegral minute) + (1_000_000_000 * fromIntegral second) + ) + _ -> P.fail edate + +-- This does not require that any digits are +-- actually present. +skipDigitsThroughComma :: e -> Parser e s () +{-# inline skipDigitsThroughComma #-} +skipDigitsThroughComma e = + Latin.skipDigits *> Latin.char e ',' + +skipThroughComma :: e -> Parser e s () +{-# inline skipThroughComma #-} +skipThroughComma e = Latin.skipTrailedBy e ',' + +w64Comma :: e -> Parser e s Word64 +{-# inline w64Comma #-} +w64Comma e = do + w <- Latin.decWord64 e + Latin.char e ',' + pure w + +w16Comma :: e -> Parser e s Word16 +{-# inline w16Comma #-} +w16Comma e = Latin.decWord16 e <* Latin.char e ',' + + +untilComma :: e -> Parser e s Bounds +{-# inline untilComma #-} +untilComma e = do + start <- Unsafe.cursor + Latin.skipTrailedBy e ',' + endSucc <- Unsafe.cursor + let end = endSucc - 1 + pure (Bounds start (end - start)) + +data Bounds = Bounds + {-# UNPACK #-} !Int -- offset + {-# UNPACK #-} !Int -- length + + +-- | The field that was being parsed when a parse failure occurred. +-- This is typically for useful for libary developers, but to present +-- it to the end user, call @show@ or @throwIO@. +newtype Field = Field UnmanagedBytes + +instance Show Field where + showsPrec _ (Field (UnmanagedBytes (Addr addr) _)) s = + '"' : GHC.Pack.unpackAppendCString# addr ('"' : s) + +instance Exception Field where + displayException (Field (UnmanagedBytes (Addr addr) _)) = + GHC.Pack.unpackCString# addr + +syslogPriorityField :: Field +syslogPriorityField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "syslogPriority"# + +syslogHostField :: Field +syslogHostField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "syslogHost"# + +syslogDatetimeField :: Field +syslogDatetimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "syslogDatetime"# + +prismaDataField :: Field +prismaDataField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "prismaData"# + +receiveTimeDateField :: Field +receiveTimeDateField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "receiveTime:date"# + +receiveTimeTimeField :: Field +receiveTimeTimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "receiveTime:time"# + +serialNumberField :: Field +serialNumberField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "serialNumber"# + +typeField :: Field +typeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "type"# + +subtypeField :: Field +subtypeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "subtype"# + +timeGeneratedDateField :: Field +timeGeneratedDateField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "timeGenerated:date"# + +timeGeneratedTimeField :: Field +timeGeneratedTimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "timeGenerated:time"# + +sourceAddressField :: Field +sourceAddressField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "sourceAddress"# + +destinationAddressField :: Field +destinationAddressField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "destinationAddress"# + +natSourceIpField :: Field +natSourceIpField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "natSourceIp"# + +natDestinationIpField :: Field +natDestinationIpField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "natDestinationIp"# + +ruleNameField :: Field +ruleNameField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "ruleName"# + +sourceUserField :: Field +sourceUserField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "sourceUser"# + +destinationUserField :: Field +destinationUserField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "destinationUser"# + +applicationField :: Field +applicationField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "application"# + +virtualSystemField :: Field +virtualSystemField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "virtualSystem"# + +sourceZoneField :: Field +sourceZoneField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "sourceZone"# + +destinationZoneField :: Field +destinationZoneField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "destinationZone"# + +inboundInterfaceField :: Field +inboundInterfaceField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "inboundInterface"# + +outboundInterfaceField :: Field +outboundInterfaceField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "outboundInterface"# + +logActionField :: Field +logActionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "logAction"# + +sessionIdField :: Field +sessionIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "sessionId"# + +repeatCountField :: Field +repeatCountField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "repeatCount"# + +sourcePortField :: Field +sourcePortField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "sourcePort"# + +destinationPortField :: Field +destinationPortField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "destinationPort"# + +natSourcePortField :: Field +natSourcePortField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "natSourcePort"# + +natDestinationPortField :: Field +natDestinationPortField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "natDestinationPort"# + +flagsField :: Field +flagsField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "flags"# + +ipProtocolField :: Field +ipProtocolField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "ipProtocol"# + +urlCategoryListField :: Field +urlCategoryListField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "urlCategoryList"# + +actionField :: Field +actionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "action"# + +bytesField :: Field +bytesField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "bytes"# + +bytesSentField :: Field +bytesSentField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "bytesSent"# + +bytesReceivedField :: Field +bytesReceivedField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "bytesReceived"# + +packetsField :: Field +packetsField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "packets"# + +startTimeDateField :: Field +startTimeDateField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "startTime:date"# + +startTimeTimeField :: Field +startTimeTimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "startTime:time"# + +elapsedTimeField :: Field +elapsedTimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "elapsedTime"# + +categoryField :: Field +categoryField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "category"# + +sequenceNumberField :: Field +sequenceNumberField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "sequenceNumber"# + +actionFlagsField :: Field +actionFlagsField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "actionFlags"# + +sourceCountryField :: Field +sourceCountryField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "sourceCountry"# + +destinationCountryField :: Field +destinationCountryField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "destinationCountry"# + +packetsSentField :: Field +packetsSentField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "packetsSent"# + +packetsReceivedField :: Field +packetsReceivedField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "packetsReceived"# + +sessionEndReasonField :: Field +sessionEndReasonField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "sessionEndReason"# + +deviceGroupHierarchyLevel1Field :: Field +deviceGroupHierarchyLevel1Field = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "deviceGroupHierarchyLevel1"# + +deviceGroupHierarchyLevel2Field :: Field +deviceGroupHierarchyLevel2Field = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "deviceGroupHierarchyLevel2"# + +deviceGroupHierarchyLevel3Field :: Field +deviceGroupHierarchyLevel3Field = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "deviceGroupHierarchyLevel3"# + +deviceGroupHierarchyLevel4Field :: Field +deviceGroupHierarchyLevel4Field = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "deviceGroupHierarchyLevel4"# + +virtualSystemNameField :: Field +virtualSystemNameField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "virtualSystemName"# + +payloadProtocolField :: Field +payloadProtocolField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:payloadProtocol"# + +senderField :: Field +senderField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:sender"# + +recipientField :: Field +recipientField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:recipient"# + +refererField :: Field +refererField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:referer"# + +pcapIdField :: Field +pcapIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:pcapId"# + +directionField :: Field +directionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:direction"# + +contentTypeField :: Field +contentTypeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:contentType"# + +severityField :: Field +severityField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:severity"# + +cloudField :: Field +cloudField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:cloud"# + +threatCategoryField :: Field +threatCategoryField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:threatCategory"# + +urlIndexField :: Field +urlIndexField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:urlIndex"# + +fileDigestField :: Field +fileDigestField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:fileDigest"# + +fileTypeField :: Field +fileTypeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:fileType"# + +forwardedForField :: Field +forwardedForField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:forwardedFor"# + +userAgentField :: Field +userAgentField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:userAgent"# + +subjectField :: Field +subjectField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:subject"# + +contentVersionField :: Field +contentVersionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:contentVersion"# + +httpMethodField :: Field +httpMethodField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:httpMethod"# + +httpHeadersField :: Field +httpHeadersField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:httpHeaders"# + +reportIdField :: Field +reportIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:reportId"# + +miscellaneousField :: Field +miscellaneousField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:miscellaneous"# + +threatIdField :: Field +threatIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:threatId"# + +deviceNameField :: Field +deviceNameField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "deviceName"# + +actionSourceField :: Field +actionSourceField = Field (UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "actionSource"# + +futureUseAField :: Field +futureUseAField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "futureUse:A"# + +futureUseBField :: Field +futureUseBField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "futureUse:B"# + +futureUseCField :: Field +futureUseCField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "futureUse:C"# + +futureUseDField :: Field +futureUseDField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "futureUse:D"# + +futureUseEField :: Field +futureUseEField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "futureUse:E"# + +futureUseFField :: Field +futureUseFField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "futureUse:F"# + +futureUseGField :: Field +futureUseGField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "futureUse:G"# + +leftoversField :: Field +leftoversField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "framing:leftovers"# + +sourceVmUuidField :: Field +sourceVmUuidField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:source_uuid"# + +destinationVmUuidField :: Field +destinationVmUuidField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:dst_uuid"# + +tunnelIdField :: Field +tunnelIdField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:tunnelid"# + +monitorTagField :: Field +monitorTagField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:monitortag"# + +parentSessionIdField :: Field +parentSessionIdField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:parent_session_id"# + +parentStartTimeField :: Field +parentStartTimeField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:parent_start_time"# + +tunnelTypeField :: Field +tunnelTypeField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:tunnel"# + +sctpAssociationIdField :: Field +sctpAssociationIdField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:assoc_id"# + +sctpChunksField :: Field +sctpChunksField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:chunks"# + +sctpChunksSentField :: Field +sctpChunksSentField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:chunks_sent"# + +sctpChunksReceivedField :: Field +sctpChunksReceivedField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:chunks_received"# + +ruleUuidField :: Field +ruleUuidField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:rule_uuid"# + +http2ConnectionField :: Field +http2ConnectionField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:http_2_connection"# + +linkChangeCountField :: Field +linkChangeCountField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) + where !x# = "field:link_change_count_field"# + +moduleField :: Field +moduleField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:module"# + +descriptionField :: Field +descriptionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:description"# + +eventIdField :: Field +eventIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:eventId"# + +objectField :: Field +objectField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:object"# + +userField :: Field +userField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:user"# + +dataSourceNameField :: Field +dataSourceNameField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:datasourcename"# + +timeoutField :: Field +timeoutField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:timeout"# + +ipField :: Field +ipField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:ip"# + +dataSourceField :: Field +dataSourceField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:datasource"# + +dataSourceTypeField :: Field +dataSourceTypeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:datasourcetype"# + +sourceIpField :: Field +sourceIpField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:source_ip"# + +evidenceField :: Field +evidenceField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:evidence"# + +objectIdField :: Field +objectIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:objectid"# + +objectNameField :: Field +objectNameField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:objectname"# + +virtualSystemIdField :: Field +virtualSystemIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) + where !x# = "field:virtualsystemid"# + +-- TODO: switch to the known-key cstrlen that comes with GHC +cstringLen# :: Addr# -> Int# +{-# noinline cstringLen# #-} +cstringLen# ptr = go 0 where + go !ix@(I# ix#) = if PM.indexOffPtr (Ptr ptr) ix == (0 :: Word8) + then ix# + else go (ix + 1) + +parserOptionallyQuoted_ :: e -> Parser e s () +parserOptionallyQuoted_ e = Latin.any e >>= \case + '"' -> do + _ <- consumeQuoted e 0 + pure () + ',' -> pure () + _ -> Latin.skipTrailedBy e ',' + +-- Precondition: the cursor is placed at the beginning of the +-- possibly-quoted content. That is, the comma preceeding has +-- already been consumed. This is very similar to parserOptionallyQuoted, +-- but it differs slightly because might not be a trailing comma. If +-- there is, it gets left alone. +finalOptionallyQuoted :: e -> Parser e s Bytes +finalOptionallyQuoted e = Latin.opt >>= \case + Nothing -> do + !array <- Unsafe.expose + pure $! Bytes{array,offset=0,length=0} + Just c -> case c of + '"' -> do + -- First, we do a run through just to see if anything + -- actually needs to be escaped. + start <- Unsafe.cursor + !n <- consumeFinalQuoted e 0 + !array <- Unsafe.expose + !endSucc <- Unsafe.cursor + let end = endSucc - 1 + if n == 0 + then pure Bytes{array,offset=start,length=(end - start)} + else do + let !r = escapeQuotes Bytes{array,offset=start,length=(end - start)} + pure $! Bytes{array=r,offset=0,length=PM.sizeofByteArray r} + ',' -> do + Unsafe.unconsume 1 + !arr <- Unsafe.expose + pure $! Bytes arr 0 0 + _ -> do + !startSucc <- Unsafe.cursor + Latin.skipUntil ',' + !end <- Unsafe.cursor + !arr <- Unsafe.expose + let start = startSucc - 1 + pure $! Bytes arr start (end - start) + +-- Precondition: the cursor is placed at the beginning of the +-- possibly-quoted content. That is, the comma preceeding has +-- already been consumed. +parserOptionallyQuoted :: e -> Parser e s Bytes +parserOptionallyQuoted e = Latin.any e >>= \case + '"' -> do + -- First, we do a run through just to see if anything + -- actually needs to be escaped. + start <- Unsafe.cursor + !n <- consumeQuoted e 0 + !array <- Unsafe.expose + !endSuccSucc <- Unsafe.cursor + let end = endSuccSucc - 2 + if n == 0 + then pure Bytes{array,offset=start,length=(end - start)} + else do + let !r = escapeQuotes Bytes{array,offset=start,length=(end - start)} + pure $! Bytes{array=r,offset=0,length=PM.sizeofByteArray r} + ',' -> do + !array <- Unsafe.expose + pure $! Bytes{array,offset=0,length=0} + _ -> do + !startSucc <- Unsafe.cursor + Latin.skipTrailedBy e ',' + !endSucc <- Unsafe.cursor + !arr <- Unsafe.expose + let start = startSucc - 1 + let end = endSucc - 1 + pure $! (Bytes arr start (end - start)) + +-- Precondition: the input is a valid CSV-style quoted-escaped +-- string. That is, any double quote character is guaranteed to +-- be followed by another one. +escapeQuotes :: Bytes -> ByteArray +escapeQuotes (Bytes arr off0 len0) = runByteArrayST $ do + marr <- PM.newByteArray len0 + let go !soff !doff !len = if len > 0 + then do + let w :: Word8 = PM.indexByteArray arr soff + PM.writeByteArray marr doff w + if w /= c2w '"' + then go (soff + 1) (doff + 1) (len - 1) + else go (soff + 2) (doff + 1) (len - 2) + else pure doff + finalSz <- go off0 0 len0 + marr' <- PM.resizeMutableByteArray marr finalSz + PM.unsafeFreezeByteArray marr' + +-- When this parser completed, the position in the input will be +-- just after the comma that followed the quoted field. +-- This is defined recursively. +consumeQuoted :: + e + -> Int -- the number of escaped quotes we have encountered + -> Parser e s Int +consumeQuoted e !n = do + Latin.skipTrailedBy e '"' + Latin.any e >>= \case + ',' -> pure n + '"' -> consumeQuoted e (n + 1) + _ -> P.fail e + +-- Like consumeQuoted except that we are expected end-of-input +-- instead of a comma at the end. +consumeFinalQuoted :: + e + -> Int -- the number of escaped quotes we have encountered + -> Parser e s Int +consumeFinalQuoted e !n = do + Latin.skipTrailedBy e '"' + Latin.opt >>= \case + Nothing -> pure n + Just c -> case c of + '"' -> consumeFinalQuoted e (n + 1) + ',' -> do + Unsafe.unconsume 1 + pure n + _ -> P.fail e + +c2w :: Char -> Word8 +{-# inline c2w #-} +c2w = fromIntegral . ord diff --git a/src/Panos/Syslog/Internal/Correlation.hs b/src/Panos/Syslog/Internal/Correlation.hs new file mode 100644 --- /dev/null +++ b/src/Panos/Syslog/Internal/Correlation.hs @@ -0,0 +1,104 @@ +{-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language LambdaCase #-} +{-# language MagicHash #-} +{-# language MultiWayIf #-} +{-# language NamedFieldPuns #-} +{-# language NumericUnderscores #-} +{-# language ScopedTypeVariables #-} + + +module Panos.Syslog.Internal.Correlation + ( Correlation(..) + , parserCorrelation + ) where + +import Chronos (Datetime) +import Data.Bytes.Parser (Parser) +import Data.Bytes.Types (Bytes(..)) +import Data.Primitive (ByteArray) +import Data.Word (Word64) +import Panos.Syslog.Internal.Common +import Net.Types (IP) + +import qualified Data.Bytes.Parser.Latin as Latin +import qualified Data.Bytes.Parser.Unsafe as Unsafe +import qualified Net.IP as IP + +-- | A PAN-OS correlation log. Read-only accessors are found in +-- @Panos.Syslog.Correlation@. +data Correlation = Correlation + { message :: {-# UNPACK #-} !ByteArray + -- The original log + , syslogHost :: {-# UNPACK #-} !Bounds + -- The host as presented in the syslog preamble that + -- prefixes the message. + , receiveTime :: {-# UNPACK #-} !Datetime + -- In log, presented as: 2019/06/18 15:10:20 + , serialNumber :: {-# UNPACK #-} !Bounds + -- In log, presented as: 002610378847 + , subtype :: {-# UNPACK #-} !Bounds + -- Presented as: dhcp, dnsproxy, dos, general, etc. + , timeGenerated :: {-# UNPACK #-} !Datetime + -- Presented as: 2019/11/04 08:39:05 + , sourceAddress :: {-# UNPACK #-} !IP + , sourceUser :: {-# UNPACK #-} !Bounds + , virtualSystem :: {-# UNPACK #-} !Bounds + , category :: {-# UNPACK #-} !Bounds + , severity :: {-# UNPACK #-} !Bounds + , deviceGroupHierarchyLevel1 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel2 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel3 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel4 :: {-# UNPACK #-} !Word64 + , deviceName :: {-# UNPACK #-} !Bounds + , objectName :: {-# UNPACK #-} !Bounds + , objectId :: {-# UNPACK #-} !Word64 + , evidence :: {-# UNPACK #-} !Bytes + } + +parserCorrelation :: Bounds -> Datetime -> Bounds -> Parser Field s Correlation +parserCorrelation !syslogHost receiveTime !serialNumber = do + !message <- Unsafe.expose + subtype <- untilComma subtypeField -- usually empty for correlations + skipThroughComma futureUseAField + -- The datetime parser consumes the trailing comma + timeGenerated <- parserDatetime timeGeneratedDateField timeGeneratedTimeField + sourceAddress <- IP.parserUtf8Bytes sourceIpField + Latin.char sourceAddressField ',' + sourceUser <- untilComma sourceUserField + virtualSystem <- untilComma virtualSystemField + category <- untilComma categoryField + severity <- untilComma severityField + deviceGroupHierarchyLevel1 <- w64Comma deviceGroupHierarchyLevel1Field + deviceGroupHierarchyLevel2 <- w64Comma deviceGroupHierarchyLevel2Field + deviceGroupHierarchyLevel3 <- w64Comma deviceGroupHierarchyLevel3Field + deviceGroupHierarchyLevel4 <- w64Comma deviceGroupHierarchyLevel4Field + skipThroughComma virtualSystemNameField + deviceName <- untilComma deviceNameField + skipThroughComma virtualSystemIdField + objectName <- untilComma objectNameField + objectId <- w64Comma objectIdField + evidence <- finalOptionallyQuoted evidenceField + pure Correlation + { message + , syslogHost + , receiveTime + , serialNumber + , subtype + , timeGenerated + , virtualSystem + , category + , severity + , deviceGroupHierarchyLevel1 + , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 + , deviceGroupHierarchyLevel4 + , deviceName + , objectName + , objectId + , evidence + , sourceAddress + , sourceUser + } diff --git a/src/Panos/Syslog/Internal/System.hs b/src/Panos/Syslog/Internal/System.hs new file mode 100644 --- /dev/null +++ b/src/Panos/Syslog/Internal/System.hs @@ -0,0 +1,108 @@ +{-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language LambdaCase #-} +{-# language MagicHash #-} +{-# language NamedFieldPuns #-} +{-# language NumericUnderscores #-} +{-# language ScopedTypeVariables #-} +module Panos.Syslog.Internal.System + ( System(..) + , parserSystem + ) where + +import Panos.Syslog.Internal.Common + +import Chronos (Datetime) +import Data.Bytes.Parser (Parser) +import Data.Bytes.Types (Bytes(..)) +import Data.Primitive (ByteArray) +import Data.Word (Word64) + +import qualified Data.Bytes.Parser.Latin as Latin +import qualified Data.Bytes.Parser.Unsafe as Unsafe + +-- | A PAN-OS system log. Read-only accessors are found in +-- @Panos.Syslog.System@. +data System = System + { message :: {-# UNPACK #-} !ByteArray + -- The original log + , syslogHost :: {-# UNPACK #-} !Bounds + -- The host as presented in the syslog preamble that + -- prefixes the message. + , receiveTime :: {-# UNPACK #-} !Datetime + -- In log, presented as: 2019/06/18 15:10:20 + , serialNumber :: {-# UNPACK #-} !Bounds + -- In log, presented as: 002610378847 + , subtype :: {-# UNPACK #-} !Bounds + -- Presented as: dhcp, dnsproxy, dos, general, etc. + , timeGenerated :: {-# UNPACK #-} !Datetime + -- Presented as: 2019/11/04 08:39:05 + , virtualSystem :: {-# UNPACK #-} !Bounds + , eventId :: {-# UNPACK #-} !Bounds + , object :: {-# UNPACK #-} !Bounds + , module_ :: {-# UNPACK #-} !Bounds + , severity :: {-# UNPACK #-} !Bounds + , descriptionBounds :: {-# UNPACK #-} !Bounds + , descriptionByteArray :: {-# UNPACK #-} !ByteArray + , sequenceNumber :: {-# UNPACK #-} !Word64 + , actionFlags :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel1 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel2 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel3 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel4 :: {-# UNPACK #-} !Word64 + , virtualSystemName :: {-# UNPACK #-} !Bounds + , deviceName :: {-# UNPACK #-} !Bounds + } + +parserSystem :: Bounds -> Datetime -> Bounds -> Parser Field s System +parserSystem syslogHost receiveTime serialNumber = do + subtype <- untilComma subtypeField + skipThroughComma futureUseAField + -- The datetime parser consumes the trailing comma + timeGenerated <- parserDatetime timeGeneratedDateField timeGeneratedTimeField + virtualSystem <- untilComma virtualSystemField + eventId <- untilComma eventIdField + object <- untilComma objectField + skipThroughComma futureUseBField + skipThroughComma futureUseCField + module_ <- untilComma moduleField + severity <- untilComma severityField + Bytes{array=descriptionByteArray,offset=descrOff,length=descrLen} <- + parserOptionallyQuoted descriptionField + let descriptionBounds = Bounds descrOff descrLen + sequenceNumber <- w64Comma sequenceNumberField + -- TODO: handle action flags + Latin.char actionFlagsField '0' + Latin.char actionFlagsField 'x' + _ <- untilComma actionFlagsField + let actionFlags = 0 + deviceGroupHierarchyLevel1 <- w64Comma deviceGroupHierarchyLevel1Field + deviceGroupHierarchyLevel2 <- w64Comma deviceGroupHierarchyLevel2Field + deviceGroupHierarchyLevel3 <- w64Comma deviceGroupHierarchyLevel3Field + deviceGroupHierarchyLevel4 <- w64Comma deviceGroupHierarchyLevel4Field + virtualSystemName <- untilComma virtualSystemNameField + deviceName <- finalField + message <- Unsafe.expose + pure System + { subtype , timeGenerated + , sequenceNumber + , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 + , virtualSystemName , deviceName , receiveTime + , serialNumber, actionFlags, message + , syslogHost, virtualSystem, eventId, object, module_ + , severity, descriptionBounds, descriptionByteArray + } + +-- There should not be any more commas left in the input. +-- This takes until it finds a comma or until end of input +-- is reached. +finalField :: Parser e s Bounds +{-# inline finalField #-} +finalField = do + start <- Unsafe.cursor + Latin.skipUntil ',' + end <- Unsafe.cursor + pure (Bounds start (end - start)) diff --git a/src/Panos/Syslog/Internal/Threat.hs b/src/Panos/Syslog/Internal/Threat.hs new file mode 100644 --- /dev/null +++ b/src/Panos/Syslog/Internal/Threat.hs @@ -0,0 +1,486 @@ +{-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language LambdaCase #-} +{-# language MagicHash #-} +{-# language MultiWayIf #-} +{-# language NamedFieldPuns #-} +{-# language NumericUnderscores #-} +{-# language ScopedTypeVariables #-} +module Panos.Syslog.Internal.Threat + ( Threat(..) + , parserThreat + ) where + +import Panos.Syslog.Internal.Common + +import Chronos (Datetime) +import Control.Monad (when) +import Data.Bytes.Parser (Parser) +import Data.Bytes.Types (Bytes(..)) +import Data.Char (isAsciiUpper,isAsciiLower) +import Data.Primitive (ByteArray) +import Data.Word (Word64,Word32,Word16) +import GHC.Exts (Int(I#),Ptr(Ptr)) +import Net.Types (IP(IP),IPv6(IPv6)) +import Data.WideWord (Word128) + +import qualified Data.Bytes as Bytes +import qualified Data.Bytes.Parser as P +import qualified Data.Bytes.Parser.Latin as Latin +import qualified Data.Bytes.Parser.Unsafe as Unsafe +import qualified Data.Primitive as PM +import qualified GHC.Exts as Exts +import qualified Net.IP as IP +import qualified UUID + +-- | A PAN-OS threat log. Read-only accessors are found in +-- @Panos.Syslog.Threat@. +data Threat = Threat + { message :: {-# UNPACK #-} !ByteArray + -- The original log + , syslogHost :: {-# UNPACK #-} !Bounds + -- The host as presented in the syslog preamble that + -- prefixes the message. + , receiveTime :: {-# UNPACK #-} !Datetime + -- In log, presented as: 2019/06/18 15:10:20 + , serialNumber :: {-# UNPACK #-} !Bounds + -- In log, presented as: 002610378847 + , subtype :: {-# UNPACK #-} !Bounds + -- Presented as: data, file, flood, packet, scan, spyware, url, + -- virus, vulnerability, wildfire, or wildfire-virus. + , timeGenerated :: {-# UNPACK #-} !Datetime + -- Presented as: 2018/04/11 23:19:22 + , sourceAddress :: {-# UNPACK #-} !IP + , destinationAddress :: {-# UNPACK #-} !IP + , natSourceIp :: {-# UNPACK #-} !IP + , natDestinationIp :: {-# UNPACK #-} !IP + , ruleName :: {-# UNPACK #-} !Bounds + , sourceUser :: {-# UNPACK #-} !Bounds + , destinationUser :: {-# UNPACK #-} !Bounds + , application :: {-# UNPACK #-} !Bounds + , virtualSystem :: {-# UNPACK #-} !Bounds + , sourceZone :: {-# UNPACK #-} !Bounds + , destinationZone :: {-# UNPACK #-} !Bounds + , inboundInterface :: {-# UNPACK #-} !Bounds + , outboundInterface :: {-# UNPACK #-} !Bounds + , logAction :: {-# UNPACK #-} !Bounds + , sessionId :: {-# UNPACK #-} !Word64 + , repeatCount :: {-# UNPACK #-} !Word64 + , sourcePort :: {-# UNPACK #-} !Word16 + , destinationPort :: {-# UNPACK #-} !Word16 + , natSourcePort :: {-# UNPACK #-} !Word16 + , natDestinationPort :: {-# UNPACK #-} !Word16 + , action :: {-# UNPACK #-} !Bounds + , ipProtocol :: {-# UNPACK #-} !Bounds + , flags :: {-# UNPACK #-} !Word32 + , miscellaneousBounds :: {-# UNPACK #-} !Bounds + , miscellaneousByteArray :: {-# UNPACK #-} !ByteArray + , threatName :: {-# UNPACK #-} !Bounds + , threatId :: {-# UNPACK #-} !Word64 + , category :: {-# UNPACK #-} !Bounds + , severity :: {-# UNPACK #-} !Bounds + , direction :: {-# UNPACK #-} !Bounds + , sequenceNumber :: {-# UNPACK #-} !Word64 + , actionFlags :: {-# UNPACK #-} !Word64 + -- Presented as: 0x8000000000000000 + , sourceCountry :: {-# UNPACK #-} !Bounds + , destinationCountry :: {-# UNPACK #-} !Bounds + , contentType :: {-# UNPACK #-} !Bounds + , pcapId :: {-# UNPACK #-} !Word64 + , fileDigest :: {-# UNPACK #-} !Bounds + -- Only used by wildfire subtype + -- TODO: make the file digest a 128-bit or 256-bit word + , cloud :: {-# UNPACK #-} !Bounds + -- Only used by wildfire subtype + , urlIndex :: {-# UNPACK #-} !Word64 + -- Only used by wildfire subtype + , userAgentBounds :: {-# UNPACK #-} !Bounds + , userAgentByteArray :: {-# UNPACK #-} !ByteArray + -- Only used by url filtering subtype. This field may have + -- escaped characters, so we include the possibility of + -- using a byte array distinct from the original log. + , fileType :: {-# UNPACK #-} !Bounds + -- Only used by wildfire subtype + , forwardedFor :: {-# UNPACK #-} !Bounds + -- Only used by url filtering subtype + , referer :: {-# UNPACK #-} !Bytes + -- Only used by url filtering subtype + , sender :: {-# UNPACK #-} !Bytes + -- Only used by wildfire subtype + , subject :: {-# UNPACK #-} !Bytes + -- Only used by wildfire subtype + , recipient :: {-# UNPACK #-} !Bytes + -- Only used by wildfire subtype + , reportId :: {-# UNPACK #-} !Bounds + -- Only used by wildfire subtype + , deviceGroupHierarchyLevel1 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel2 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel3 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel4 :: {-# UNPACK #-} !Word64 + , virtualSystemName :: {-# UNPACK #-} !Bounds + , deviceName :: {-# UNPACK #-} !Bounds + -- TODO: skipping over uuid fields for now + , httpMethod :: {-# UNPACK #-} !Bounds + , tunnelId :: {-# UNPACK #-} !Word64 + , parentSessionId :: {-# UNPACK #-} !Word64 + -- Only used by url subtype + , threatCategory :: {-# UNPACK #-} !Bounds + , contentVersion :: {-# UNPACK #-} !Bounds + -- TODO: skipping some fields here + , sctpAssociationId :: {-# UNPACK #-} !Word64 + , payloadProtocolId :: {-# UNPACK #-} !Word64 + -- TODO: skipping over other fields here + , httpHeaders :: {-# UNPACK #-} !Bytes + , urlCategoryList :: {-# UNPACK #-} !Bytes + , ruleUuid :: {-# UNPACK #-} !Word128 + } + + +parserThreat :: Bounds -> Datetime -> Bounds -> Parser Field s Threat +parserThreat !syslogHost receiveTime !serialNumber = do + !message <- Unsafe.expose + subtype@(Bounds subtypeOff subtypeLen) <- untilComma subtypeField + let !subtypeB = Bytes message subtypeOff subtypeLen + let !isWildfire = + if | Bytes.equalsCString (Ptr "wildfire"# ) subtypeB -> 1 :: Int + | otherwise -> 0 :: Int + let !isUrl = + if | Bytes.equalsCString (Ptr "url"# ) subtypeB -> 1 :: Int + | otherwise -> 0 :: Int + let !isSpyware = + if | Bytes.equalsCString (Ptr "spyware"# ) subtypeB -> 1 :: Int + | otherwise -> 0 :: Int + Bounds futureUseAOff futureUseALen <- untilComma futureUseAField + let !futureUseA = Bytes message futureUseAOff futureUseALen + let !version = + if | Bytes.equalsCString (Ptr "10.0"# ) futureUseA -> 100 :: Int + | Bytes.equalsCString (Ptr "10.1"# ) futureUseA -> 101 :: Int + | otherwise -> 0 :: Int + -- the datetime parser also grabs the trailing comma + timeGenerated <- parserDatetime timeGeneratedDateField timeGeneratedTimeField + sourceAddress <- IP.parserUtf8Bytes sourceAddressField + Latin.char sourceAddressField ',' + destinationAddress <- IP.parserUtf8Bytes destinationAddressField + Latin.char destinationAddressField ',' + -- Use the ip address zero when no NAT address is present. + natSourceIp <- Latin.trySatisfy (==',') >>= \case + True -> pure (IP (IPv6 0)) + False -> do + natSourceIp <- IP.parserUtf8Bytes natSourceIpField + Latin.char natSourceIpField ',' + pure natSourceIp + natDestinationIp <- Latin.trySatisfy (==',') >>= \case + True -> pure (IP (IPv6 0)) + False -> do + natDestinationIp <- IP.parserUtf8Bytes natDestinationIpField + Latin.char natDestinationIpField ',' + pure natDestinationIp + ruleName <- untilComma ruleNameField + sourceUser <- untilComma sourceUserField + destinationUser <- untilComma destinationUserField + application <- untilComma applicationField + virtualSystem <- untilComma virtualSystemField + sourceZone <- untilComma sourceZoneField + destinationZone <- untilComma destinationZoneField + inboundInterface <- untilComma inboundInterfaceField + outboundInterface <- untilComma outboundInterfaceField + logAction <- untilComma logActionField + -- According to Palo Alto's documentation, the field after log_action + -- is a future use field. However, in some (possibly all) PAN-OS 10 + -- logs, this field is missing. In all other logs, it is a + -- YYYY/MM/DD HH:mm:ss timestamp. If the field is exactly 19 bytes long, + -- we assume that it is the unused field. Otherwise, we assume that the + -- unused field is missing, and we try to interpret these bytes as the + -- session_id. + futureCursorB <- Unsafe.cursor + Bounds _ futureLenB <- untilComma futureUseBField + sessionId <- case futureLenB of + 19 -> w64Comma sessionIdField + _ -> do + Unsafe.jump futureCursorB + w64Comma sessionIdField + repeatCount <- w64Comma repeatCountField + sourcePort <- w16Comma sourcePortField + destinationPort <- w16Comma destinationPortField + natSourcePort <- w16Comma natSourcePortField + natDestinationPort <- w16Comma natDestinationPortField + -- Note: Flags are ignored. Also, in either PAN-OS 10 or Prisma + -- (not sure which one causes this), flags are missing. + Latin.trySatisfy (=='0') >>= \case + True -> do + Latin.char flagsField 'x' + _ <- untilComma flagsField + pure () + False -> pure () + let flags = 0 + ipProtocol <- untilComma ipProtocolField + action <- untilComma actionField + Bytes{array=miscellaneousByteArray,offset=miscOff,length=miscLen} <- + parserOptionallyQuoted miscellaneousField + let miscellaneousBounds = Bounds miscOff miscLen + -- Detect presence of threat name and id by looking for trailing + -- close paren on field. + threatIdCursor <- Unsafe.cursor + Bounds threatIdOff threatIdLen <- untilComma threatIdField + Unsafe.jump threatIdCursor + (threatName,threatId) <- case Bytes.isByteSuffixOf 0x29 (Bytes message threatIdOff threatIdLen) of + True -> parserThreatId + False -> pure (Bounds threatIdOff 0, 0) + category <- + if | version < 100 || isUrl == 1 -> untilComma categoryField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + severity <- untilComma severityField + direction <- untilComma directionField + sequenceNumber <- w64Comma sequenceNumberField + -- Note: Action flags are ignored. See note on Flags as well. + Latin.trySatisfy (=='0') >>= \case + True -> do + Latin.char actionFlagsField 'x' + _ <- untilComma actionFlagsField + pure () + False -> pure () + let actionFlags = 0 + sourceCountry <- untilComma sourceCountryField + destinationCountry <- untilComma destinationCountryField + -- This future use is always either zero or empty before PAN-OS 10.x, + -- and in newer versions it is suppressed entirely. However, the + -- following field is content_type, which never starts with zero. + if | version >= 100, isUrl == 1 -> pure () + | version >= 100, isSpyware == 1 -> do + -- I cannot figure out why, in PAN-OS 10, there is an extra field + -- in spyware logs. + Latin.skipDigits1 futureUseEField + Latin.char2 futureUseEField ',' ',' + | otherwise -> do + Latin.skipDigits + Latin.char futureUseEField ',' + -- In PAN-OS 10.x, content_type only shows up in URL logs. + -- I've added spyware here as well, but I'm not sure if this + -- is correct. Content type definitely does not show up in + -- wildfire logs. + contentType <- + if | version < 100 || isUrl == 1 || isSpyware == 1 -> untilComma contentTypeField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + -- Wildfire logs in PAN-OS 10.x do not have a pcap_id field. + pcapId <- + if | version < 100 || isUrl == 1 || isSpyware == 1 -> w64Comma pcapIdField + | otherwise -> pure 0 + -- In PAN-OS 10.x, file_digest and cloud only show up in wildfire logs. + fileDigest <- + if | version < 100 || isWildfire == 1 || isSpyware == 1 -> untilComma fileDigestField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + cloud <- + if | version < 100 || isWildfire == 1 || isSpyware == 1 -> untilComma cloudField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + urlIndex <- + if | version < 100 || isUrl == 1 || isWildfire == 1 -> w64Comma urlIndexField + | otherwise -> pure 0 + Bytes{array=userAgentByteArray,offset=uaOff,length=uaLen} <- + if | version < 100 || isUrl == 1 -> parserOptionallyQuoted userAgentField + | otherwise -> do + off <- Unsafe.cursor + pure (Bytes message off 0) + let userAgentBounds = Bounds uaOff uaLen + fileType <- + if | version < 100 || isWildfire == 1 || isSpyware == 1 -> untilComma fileTypeField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + forwardedFor <- + if | version < 100 || isUrl == 1 -> untilComma forwardedForField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + referer <- + if | version < 100 || isUrl == 1 -> parserOptionallyQuoted refererField + | otherwise -> do + off <- Unsafe.cursor + pure (Bytes message off 0) + sender <- + if | version < 100 || isWildfire == 1 -> parserOptionallyQuoted senderField + | otherwise -> do + off <- Unsafe.cursor + pure (Bytes message off 0) + subject <- + if | version < 100 || isWildfire == 1 -> parserOptionallyQuoted subjectField + | otherwise -> do + off <- Unsafe.cursor + pure (Bytes message off 0) + recipient <- + if | version < 100 || isWildfire == 1 -> parserOptionallyQuoted recipientField + | otherwise -> do + off <- Unsafe.cursor + pure (Bytes message off 0) + reportId <- + if | version < 100 || isWildfire == 1 || isSpyware == 1 -> untilComma reportIdField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + deviceGroupHierarchyLevel1 <- w64Comma deviceGroupHierarchyLevel1Field + deviceGroupHierarchyLevel2 <- w64Comma deviceGroupHierarchyLevel2Field + deviceGroupHierarchyLevel3 <- w64Comma deviceGroupHierarchyLevel3Field + deviceGroupHierarchyLevel4 <- w64Comma deviceGroupHierarchyLevel4Field + virtualSystemName <- untilComma virtualSystemNameField + deviceName <- untilComma deviceNameField + -- On PAN-OS 10+, this future use field is omitted. + when (version < 100) (parserOptionallyQuoted_ futureUseFField) + skipThroughComma sourceVmUuidField + skipThroughComma destinationVmUuidField + httpMethod <- + if | version < 100 || isUrl == 1 || isSpyware == 1 -> untilComma httpMethodField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + tunnelId <- w64Comma tunnelIdField + skipThroughComma monitorTagField + parentSessionId <- w64Comma parentSessionIdField + skipThroughComma parentStartTimeField + skipThroughComma tunnelTypeField + threatCategory <- untilComma threatCategoryField + contentVersion <- + if | version < 100 -> untilComma contentVersionField + | otherwise -> do + off <- Unsafe.cursor + pure (Bounds off 0) + when (version < 100) (skipThroughComma futureUseGField) + sctpAssociationId <- w64Comma sctpAssociationIdField + payloadProtocolId <- + if | version < 100 -> do + w64Comma payloadProtocolField + | isUrl == 1 -> do + w64Comma payloadProtocolField + | isSpyware == 1 || isWildfire == 1 -> do + -- In PAN-OS 10.x, this field looks like it is always zero, + -- represented in hexadecimal, in spyware logs. + Latin.char4 payloadProtocolField '0' 'x' '0' ',' + pure 0 + | otherwise -> pure 0 + -- TODO: Escape or parse HTTP Headers correctly + httpHeaders <- + if | version < 100 || isUrl == 1 -> do + finalOptionallyQuoted httpHeadersField + | otherwise -> do + off <- Unsafe.cursor + pure (Bytes message off 0) + -- In PAN-OS 8.1, threat logs end after http headers. + -- PAN-OS 9.0 adds three more fields. + P.isEndOfInput >>= \case + False -> do + !urlCategoryList <- + if | version < 100 || isUrl == 1 -> do + Latin.char urlCategoryListField ',' + parserOptionallyQuoted urlCategoryListField + | otherwise -> do + off <- Unsafe.cursor + pure (Bytes message off 0) + ruleUuid <- UUID.parserHyphenated ruleUuidField + Latin.char http2ConnectionField ',' + Latin.skipDigits1 http2ConnectionField + pure Threat + { subtype , timeGenerated , sourceAddress , destinationAddress + , natSourceIp , natDestinationIp , ruleName , sourceUser + , destinationUser , application , virtualSystem , sourceZone + , destinationZone , inboundInterface , outboundInterface , logAction + , sessionId , repeatCount , sourcePort , destinationPort + , natSourcePort , natDestinationPort , ipProtocol + , action , category + , sequenceNumber , sourceCountry , destinationCountry + , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 + , virtualSystemName , deviceName , receiveTime + , serialNumber, actionFlags, flags, message + , syslogHost, threatId, severity, direction, threatName + , contentType, pcapId + , fileDigest, cloud, urlIndex + , userAgentBounds, sctpAssociationId + , userAgentByteArray, fileType + , forwardedFor, referer + , sender, subject, recipient + , reportId, httpMethod, contentVersion + , threatCategory, miscellaneousBounds, miscellaneousByteArray + , payloadProtocolId, parentSessionId, tunnelId + , httpHeaders, ruleUuid, urlCategoryList + } + True -> pure Threat + { subtype , timeGenerated , sourceAddress , destinationAddress + , natSourceIp , natDestinationIp , ruleName , sourceUser + , destinationUser , application , virtualSystem , sourceZone + , destinationZone , inboundInterface , outboundInterface , logAction + , sessionId , repeatCount , sourcePort , destinationPort + , natSourcePort , natDestinationPort , ipProtocol + , action , category + , sequenceNumber , sourceCountry , destinationCountry + , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 + , virtualSystemName , deviceName , receiveTime + , serialNumber, actionFlags, flags, message + , syslogHost, threatId, severity, direction, threatName + , contentType, pcapId + , fileDigest, cloud, urlIndex + , userAgentBounds, sctpAssociationId + , userAgentByteArray, fileType + , forwardedFor, referer + , sender, subject, recipient + , reportId, httpMethod, contentVersion + , threatCategory, miscellaneousBounds, miscellaneousByteArray + , payloadProtocolId, parentSessionId, tunnelId + , httpHeaders + , ruleUuid = 0 + , urlCategoryList = Bytes message 0 0 + } + +-- Threat IDs are weird. There are three different kinds of +-- strings that can show up here: +-- +-- * (9999) +-- * Microsoft RPC Endpoint Mapper Detection(30845) +-- * Windows Executable (EXE)(52020) +-- +-- URL logs have a threat id of 9999, and there is no description. +-- Everything else has a human-readable description. Sometimes, +-- this description is suffixed by a space and a parenthesized +-- acronym (EXE, DLL, etc.). +parserThreatId :: Parser Field s (Bounds,Word64) +parserThreatId = Latin.any threatIdField >>= \case + '(' -> do + theId <- Latin.decWord64 threatIdField + Latin.char threatIdField ')' + Latin.char threatIdField ',' + pure (Bounds 0 0, theId) + _ -> do + startSucc <- Unsafe.cursor + Latin.skipTrailedBy threatIdField '(' + end <- Latin.trySatisfy (\c -> isAsciiUpper c || isAsciiLower c) >>= \case + True -> do + endSuccSucc <- Unsafe.cursor + Latin.skipTrailedBy threatIdField '(' + arr <- Unsafe.expose + -- We go back an extra character to remove the trailing + -- space. I do not believe this can lead to negative-length + -- slices, but the line of reasoning is muddy. + case indexCharArray arr (endSuccSucc - 3) of + ' ' -> pure (endSuccSucc - 3) + _ -> P.fail threatIdField + False -> do + endSucc <- Unsafe.cursor + pure (endSucc - 1) + theId <- Latin.decWord64 threatIdField + Latin.char threatIdField ')' + Latin.char threatIdField ',' + let start = startSucc - 1 + pure (Bounds start (end - start), theId) + +indexCharArray :: ByteArray -> Int -> Char +indexCharArray (PM.ByteArray x) (I# i) = + Exts.C# (Exts.indexCharArray# x i) + diff --git a/src/Panos/Syslog/Internal/Traffic.hs b/src/Panos/Syslog/Internal/Traffic.hs new file mode 100644 --- /dev/null +++ b/src/Panos/Syslog/Internal/Traffic.hs @@ -0,0 +1,265 @@ +{-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language LambdaCase #-} +{-# language MagicHash #-} +{-# language NamedFieldPuns #-} +{-# language NumericUnderscores #-} +{-# language ScopedTypeVariables #-} +module Panos.Syslog.Internal.Traffic + ( Traffic(..) + , parserTraffic + ) where + +import Panos.Syslog.Internal.Common + +import Chronos (Datetime) +import Data.Bytes.Parser (Parser) +import Data.Primitive (ByteArray) +import Data.Word (Word64,Word32,Word16) +import GHC.Exts (Ptr(Ptr)) +import Net.Types (IP(IP),IPv6(IPv6)) +import Data.WideWord (Word128) + +import qualified Data.Bytes.Parser as P +import qualified Data.Bytes.Parser.Latin as Latin +import qualified Data.Bytes.Parser.Unsafe as Unsafe +import qualified Net.IP as IP +import qualified UUID + +-- | A PAN-OS traffic log. Read-only accessors are found in +-- @Panos.Syslog.Traffic@. +data Traffic = Traffic + { message :: {-# UNPACK #-} !ByteArray + -- The original log + , syslogHost :: {-# UNPACK #-} !Bounds + -- The host as presented in the syslog preamble that + -- prefixes the message. + , receiveTime :: {-# UNPACK #-} !Datetime + -- In log, presented as: 2019/06/18 15:10:20 + , serialNumber :: {-# UNPACK #-} !Bounds + -- In log, presented as: 002610378847 + , subtype :: {-# UNPACK #-} !Bounds + -- Presented as: start, end, drop, and deny + , timeGenerated :: {-# UNPACK #-} !Datetime + -- Presented as: 2018/04/11 23:19:22 + , sourceAddress :: {-# UNPACK #-} !IP + , destinationAddress :: {-# UNPACK #-} !IP + , natSourceIp :: {-# UNPACK #-} !IP + , natDestinationIp :: {-# UNPACK #-} !IP + , ruleName :: {-# UNPACK #-} !Bounds + , sourceUser :: {-# UNPACK #-} !Bounds + , destinationUser :: {-# UNPACK #-} !Bounds + , application :: {-# UNPACK #-} !Bounds + , virtualSystem :: {-# UNPACK #-} !Bounds + , sourceZone :: {-# UNPACK #-} !Bounds + , destinationZone :: {-# UNPACK #-} !Bounds + , inboundInterface :: {-# UNPACK #-} !Bounds + , outboundInterface :: {-# UNPACK #-} !Bounds + , logAction :: {-# UNPACK #-} !Bounds + , sessionId :: {-# UNPACK #-} !Word64 + , repeatCount :: {-# UNPACK #-} !Word64 + , sourcePort :: {-# UNPACK #-} !Word16 + , destinationPort :: {-# UNPACK #-} !Word16 + , natSourcePort :: {-# UNPACK #-} !Word16 + , natDestinationPort :: {-# UNPACK #-} !Word16 + , flags :: {-# UNPACK #-} !Word32 + -- Presented as: 0x400053 + , ipProtocol :: {-# UNPACK #-} !Bounds + -- Presented as: tcp, udp, etc. + , action :: {-# UNPACK #-} !Bounds + , bytes :: {-# UNPACK #-} !Word64 + , bytesSent :: {-# UNPACK #-} !Word64 + , bytesReceived :: {-# UNPACK #-} !Word64 + , packets :: {-# UNPACK #-} !Word64 + , startTime :: {-# UNPACK #-} !Datetime + , elapsedTime :: {-# UNPACK #-} !Word64 + , category :: {-# UNPACK #-} !Bounds + , sequenceNumber :: {-# UNPACK #-} !Word64 + , actionFlags :: {-# UNPACK #-} !Word64 + -- Presented as: 0x8000000000000000 + , sourceCountry :: {-# UNPACK #-} !Bounds + , destinationCountry :: {-# UNPACK #-} !Bounds + , packetsSent :: {-# UNPACK #-} !Word64 + , packetsReceived :: {-# UNPACK #-} !Word64 + , sessionEndReason :: {-# UNPACK #-} !Bounds + , deviceGroupHierarchyLevel1 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel2 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel3 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel4 :: {-# UNPACK #-} !Word64 + , virtualSystemName :: {-# UNPACK #-} !Bounds + , deviceName :: {-# UNPACK #-} !Bounds + , actionSource :: {-# UNPACK #-} !Bounds + , ruleUuid :: {-# UNPACK #-} !Word128 + } + + +parserTraffic :: Bounds -> Datetime -> Bounds -> Parser Field s Traffic +parserTraffic !syslogHost receiveTime !serialNumber = do + subtype <- untilComma subtypeField + skipThroughComma futureUseAField + -- The datetime parser consumes the trailing comma + timeGenerated <- parserDatetime timeGeneratedDateField timeGeneratedTimeField + sourceAddress <- IP.parserUtf8Bytes sourceAddressField + Latin.char sourceAddressField ',' + destinationAddress <- IP.parserUtf8Bytes destinationAddressField + Latin.char destinationAddressField ',' + -- Use the ip address zero when no NAT address is present. + natSourceIp <- Latin.trySatisfy (==',') >>= \case + True -> pure (IP (IPv6 0)) + False -> do + natSourceIp <- IP.parserUtf8Bytes natSourceIpField + Latin.char natSourceIpField ',' + pure natSourceIp + natDestinationIp <- Latin.trySatisfy (==',') >>= \case + True -> pure (IP (IPv6 0)) + False -> do + natDestinationIp <- IP.parserUtf8Bytes natDestinationIpField + Latin.char natDestinationIpField ',' + pure natDestinationIp + ruleName <- untilComma ruleNameField + sourceUser <- untilComma sourceUserField + destinationUser <- untilComma destinationUserField + application <- untilComma applicationField + virtualSystem <- untilComma virtualSystemField + sourceZone <- untilComma sourceZoneField + destinationZone <- untilComma destinationZoneField + inboundInterface <- untilComma inboundInterfaceField + outboundInterface <- untilComma outboundInterfaceField + logAction <- untilComma logActionField + -- According to Palo Alto's documentation, the field after log_action + -- is a future use field. However, in some (possibly all) PAN-OS 10 + -- logs, this field is missing. In all other logs, it is a + -- YYYY/MM/DD HH:mm:ss timestamp. If the field is exactly 19 bytes long, + -- we assume that it is the unused field. Otherwise, we assume that the + -- unused field is missing, and we try to interpret these bytes as the + -- session_id. + futureCursorB <- Unsafe.cursor + Bounds _ futureLenB <- untilComma futureUseBField + sessionId <- case futureLenB of + 19 -> w64Comma sessionIdField + _ -> do + Unsafe.jump futureCursorB + w64Comma sessionIdField + repeatCount <- w64Comma repeatCountField + sourcePort <- w16Comma sourcePortField + destinationPort <- w16Comma destinationPortField + natSourcePort <- w16Comma natSourcePortField + natDestinationPort <- w16Comma natDestinationPortField + -- Note: Flags are ignored. Also, in either PAN-OS 10 or Prisma + -- (not sure which one causes this), flags are missing. + Latin.trySatisfy (=='0') >>= \case + True -> do + Latin.char flagsField 'x' + _ <- untilComma flagsField + pure () + False -> pure () + let flags = 0 + ipProtocol <- untilComma ipProtocolField + action <- untilComma actionField + bytes <- w64Comma bytesField + bytesSent <- w64Comma bytesSentField + bytesReceived <- w64Comma bytesReceivedField + packets <- w64Comma packetsField + startTime <- parserDatetime startTimeDateField startTimeTimeField + elapsedTime <- w64Comma elapsedTimeField + category <- untilComma categoryField + futureCursorC <- Unsafe.cursor + Bounds _ futureLenC <- untilComma futureUseCField + -- Here, we find another future use fields that is missing in PAN-OS 10.x. + -- In older versions, this was always the single digit 0. So, we treat + -- length-1 fields as future use. + sequenceNumber <- case futureLenC of + 0 -> w64Comma sequenceNumberField + 1 -> w64Comma sequenceNumberField + _ -> do + Unsafe.jump futureCursorC + w64Comma sequenceNumberField + -- Note: Action flags are ignored. See note on Flags as well. + Latin.trySatisfy (=='0') >>= \case + True -> do + Latin.char actionFlagsField 'x' + _ <- untilComma actionFlagsField + pure () + False -> pure () + let actionFlags = 0 + sourceCountry <- untilComma sourceCountryField + destinationCountry <- untilComma destinationCountryField + -- Future use field is optional. Here, we hop forward to what is either + -- the session_end_reason or the packets_received. We use the first byte + -- of this to figure out if the future use field is missing. + futureCursorE <- Unsafe.cursor + skipThroughComma futureUseEField + skipThroughComma futureUseEField + Latin.any futureUseEField >>= \case + c | c >= '0', c <= '9' -> do + -- Future use field was present. Skip it. + Unsafe.jump futureCursorE + skipThroughComma futureUseEField + _ -> Unsafe.jump futureCursorE + packetsSent <- w64Comma packetsSentField + packetsReceived <- w64Comma packetsReceivedField + sessionEndReason <- untilComma sessionEndReasonField + deviceGroupHierarchyLevel1 <- w64Comma deviceGroupHierarchyLevel1Field + deviceGroupHierarchyLevel2 <- w64Comma deviceGroupHierarchyLevel2Field + deviceGroupHierarchyLevel3 <- w64Comma deviceGroupHierarchyLevel3Field + deviceGroupHierarchyLevel4 <- w64Comma deviceGroupHierarchyLevel4Field + virtualSystemName <- untilComma virtualSystemNameField + deviceName <- untilComma deviceNameField + actionSource <- untilComma actionSourceField + skipThroughComma sourceVmUuidField + skipThroughComma destinationVmUuidField + skipThroughComma tunnelIdField + skipThroughComma monitorTagField + skipThroughComma parentSessionIdField + skipThroughComma parentStartTimeField + skipThroughComma tunnelTypeField + skipThroughComma sctpAssociationIdField + skipDigitsThroughComma sctpChunksField + skipDigitsThroughComma sctpChunksSentField + Latin.skipDigits1 sctpChunksReceivedField + message <- Unsafe.expose + -- In PAN-OS 8.1, traffic logs end after SCTP chunks received. + -- PAN-OS 9.0 adds two additional fields: rule uuid and a number + -- that has something to do with HTTP/2. + P.isEndOfInput >>= \case + False -> do + Latin.char ruleUuidField ',' + ruleUuid <- UUID.parserHyphenated ruleUuidField + Latin.char http2ConnectionField ',' + Latin.skipDigits1 http2ConnectionField + pure Traffic + { subtype , timeGenerated , sourceAddress , destinationAddress + , natSourceIp , natDestinationIp , ruleName , sourceUser + , destinationUser , application , virtualSystem , sourceZone + , destinationZone , inboundInterface , outboundInterface , logAction + , sessionId , repeatCount , sourcePort , destinationPort + , natSourcePort , natDestinationPort , ipProtocol + , action , bytes , bytesSent , bytesReceived + , packets , startTime , elapsedTime , category + , sequenceNumber , sourceCountry , destinationCountry + , packetsSent , sessionEndReason + , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 + , virtualSystemName , deviceName , actionSource , receiveTime + , serialNumber, packetsReceived, actionFlags, flags, message + , syslogHost, ruleUuid + } + True -> pure Traffic + { subtype , timeGenerated , sourceAddress , destinationAddress + , natSourceIp , natDestinationIp , ruleName , sourceUser + , destinationUser , application , virtualSystem , sourceZone + , destinationZone , inboundInterface , outboundInterface , logAction + , sessionId , repeatCount , sourcePort , destinationPort + , natSourcePort , natDestinationPort , ipProtocol + , action , bytes , bytesSent , bytesReceived + , packets , startTime , elapsedTime , category + , sequenceNumber , sourceCountry , destinationCountry + , packetsSent , sessionEndReason + , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 + , virtualSystemName , deviceName , actionSource , receiveTime + , serialNumber, packetsReceived, actionFlags, flags, message + , syslogHost, ruleUuid = 0 + } diff --git a/src/Panos/Syslog/Internal/User.hs b/src/Panos/Syslog/Internal/User.hs new file mode 100644 --- /dev/null +++ b/src/Panos/Syslog/Internal/User.hs @@ -0,0 +1,104 @@ +{-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language LambdaCase #-} +{-# language MagicHash #-} +{-# language NamedFieldPuns #-} +{-# language NumericUnderscores #-} +{-# language ScopedTypeVariables #-} +module Panos.Syslog.Internal.User + ( User(..) + , parserUser + ) where + +import Panos.Syslog.Internal.Common + +import Chronos (Datetime) +import Data.Bytes.Parser (Parser) +import Data.Primitive (ByteArray) +import Data.Word (Word64) +import Net.Types (IP) + +import qualified Data.Bytes.Parser as P +import qualified Data.Bytes.Parser.Latin as Latin +import qualified Data.Bytes.Parser.Unsafe as Unsafe +import qualified Net.IP as IP + +-- | A PAN-OS user id log. Read-only accessors are found in +-- @Panos.Syslog.User@. +data User = User + { message :: {-# UNPACK #-} !ByteArray + -- The original log + , syslogHost :: {-# UNPACK #-} !Bounds + -- The host as presented in the syslog preamble that + -- prefixes the message. + , receiveTime :: {-# UNPACK #-} !Datetime + -- In log, presented as: 2019/06/18 15:10:20 + , serialNumber :: {-# UNPACK #-} !Bounds + -- In log, presented as: 002610378847 + , subtype :: {-# UNPACK #-} !Bounds + -- Presented as: dhcp, dnsproxy, dos, general, etc. + , timeGenerated :: {-# UNPACK #-} !Datetime + -- Presented as: 2019/11/04 08:39:05 + , virtualSystem :: {-# UNPACK #-} !Bounds + , sourceIp :: {-# UNPACK #-} !IP + , user :: {-# UNPACK #-} !Bounds + , dataSourceName :: {-# UNPACK #-} !Bounds + , repeatCount :: {-# UNPACK #-} !Word64 + , dataSource :: {-# UNPACK #-} !Bounds + , sequenceNumber :: {-# UNPACK #-} !Word64 + , actionFlags :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel1 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel2 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel3 :: {-# UNPACK #-} !Word64 + , deviceGroupHierarchyLevel4 :: {-# UNPACK #-} !Word64 + , virtualSystemName :: {-# UNPACK #-} !Bounds + , deviceName :: {-# UNPACK #-} !Bounds + } + +parserUser :: Bounds -> Datetime -> Bounds -> Parser Field s User +parserUser !syslogHost receiveTime !serialNumber = do + subtype <- untilComma subtypeField -- login or logout + skipThroughComma futureUseAField + -- The datetime parser consumes the trailing comma + timeGenerated <- parserDatetime timeGeneratedDateField timeGeneratedTimeField + virtualSystem <- untilComma virtualSystemField + sourceIp <- IP.parserUtf8Bytes sourceIpField + Latin.char ipField ',' + user <- untilComma userField + dataSourceName <- untilComma dataSourceNameField + skipThroughComma eventIdField + repeatCount <- w64Comma repeatCountField + skipThroughComma timeoutField + skipThroughComma sourcePortField + skipThroughComma destinationPortField + dataSource <- untilComma dataSourceField + skipThroughComma dataSourceTypeField + sequenceNumber <- w64Comma sequenceNumberField + -- TODO: handle action flags + Latin.char actionFlagsField '0' + Latin.char actionFlagsField 'x' + _ <- untilComma actionFlagsField + let actionFlags = 0 + deviceGroupHierarchyLevel1 <- w64Comma deviceGroupHierarchyLevel1Field + deviceGroupHierarchyLevel2 <- w64Comma deviceGroupHierarchyLevel2Field + deviceGroupHierarchyLevel3 <- w64Comma deviceGroupHierarchyLevel3Field + deviceGroupHierarchyLevel4 <- w64Comma deviceGroupHierarchyLevel4Field + virtualSystemName <- untilComma virtualSystemNameField + deviceName <- untilComma deviceNameField + -- Ignore all fields after device name since they are low-value. + _ <- P.remaining + message <- Unsafe.expose + pure User + { subtype + , timeGenerated + , sourceIp + , sequenceNumber + , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 + , virtualSystemName , deviceName , receiveTime + , serialNumber, actionFlags, message + , syslogHost, virtualSystem + , dataSourceName, dataSource, user, repeatCount + } diff --git a/src/Panos/Syslog/System.hs b/src/Panos/Syslog/System.hs --- a/src/Panos/Syslog/System.hs +++ b/src/Panos/Syslog/System.hs @@ -1,10 +1,11 @@ {-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} {-# language MagicHash #-} {-# language NamedFieldPuns #-} -{-# language DuplicateRecordFields #-} {-# language NumericUnderscores #-} -{-# language DerivingStrategies #-} -{-# language GeneralizedNewtypeDeriving #-} +{-# language OverloadedRecordDot #-} -- | Fields for system logs. module Panos.Syslog.System @@ -71,12 +72,14 @@ -- | Time the log was generated on the dataplane. timeGenerated :: System -> Datetime -timeGenerated = U.timeGenerated +{-# inline timeGenerated #-} +timeGenerated u = u.timeGenerated -- | A 64-bit log entry identifier incremented sequentially; -- each log type has a unique number space. sequenceNumber :: System -> Word64 -sequenceNumber = U.sequenceNumber +{-# inline sequenceNumber #-} +sequenceNumber u = u.sequenceNumber -- | Serial number of the firewall that generated the log. These -- occassionally contain non-numeric characters, so do not attempt @@ -84,4 +87,3 @@ serialNumber :: System -> Bytes serialNumber (System{serialNumber=Bounds off len,message=msg}) = Bytes{offset=off,length=len,array=msg} - diff --git a/src/Panos/Syslog/Threat.hs b/src/Panos/Syslog/Threat.hs --- a/src/Panos/Syslog/Threat.hs +++ b/src/Panos/Syslog/Threat.hs @@ -1,10 +1,11 @@ {-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} {-# language MagicHash #-} {-# language NamedFieldPuns #-} -{-# language DuplicateRecordFields #-} {-# language NumericUnderscores #-} -{-# language DerivingStrategies #-} -{-# language GeneralizedNewtypeDeriving #-} +{-# language OverloadedRecordDot #-} -- | Fields for threat logs. module Panos.Syslog.Threat @@ -18,9 +19,11 @@ , destinationUser , destinationZone , deviceName + , fileDigest , httpHeaders , httpMethod , inboundInterface + , ipProtocol , miscellaneous , natDestinationIp , natDestinationPort @@ -29,10 +32,12 @@ , outboundInterface , recipient , referer + , repeatCount , ruleName , sender , sequenceNumber , serialNumber + , sessionId , severity , sourceAddress , sourceCountry @@ -45,6 +50,7 @@ , threatId , threatName , timeGenerated + , urlCategoryList , virtualSystemName ) where @@ -53,6 +59,7 @@ import Data.Word (Word64,Word16) import Chronos (Datetime) import Net.Types (IP) +import qualified Data.Bytes as Bytes import qualified Panos.Syslog.Unsafe as U -- | Subtype of threat log. Values include: @data@, @file@, @flood@, @@ -97,6 +104,15 @@ deviceName (Threat{deviceName=Bounds off len,message=msg}) = Bytes{offset=fromIntegral off,length=fromIntegral len,array=msg} +-- | The file digest string shows the binary hash of the file sent +-- to be analyzed by the WildFire service. +-- +-- This is sometimes MD5 and sometimes SHA256. Rather than attempting +-- to decode it, it is just left as bytes. +fileDigest :: Threat -> Bytes +fileDigest (Threat{fileDigest=Bounds off len,message=msg}) = + Bytes{offset=fromIntegral off,length=fromIntegral len,array=msg} + -- | Palo Alto Networks identifier for the threat. It is a description -- string followed by a 64-bit numerical identifier in parentheses for -- some subtypes. @@ -113,7 +129,8 @@ -- | Time the log was generated on the dataplane. timeGenerated :: Threat -> Datetime -timeGenerated = U.timeGenerated +{-# inline timeGenerated #-} +timeGenerated u = u.timeGenerated -- | For URL Subtype, it is the URL Category; For WildFire subtype, -- it is the verdict on the file and is either @malicious@, @grayware@, @@ -147,7 +164,8 @@ -- type has a unique number space. This field is not supported on -- PA-7000 Series firewalls. sequenceNumber :: Threat -> Word64 -sequenceNumber = U.sequenceNumber +{-# inline sequenceNumber #-} +sequenceNumber u = u.sequenceNumber -- | Serial number of the firewall that generated the log. These -- occassionally contain non-numeric characters, so do not attempt @@ -188,19 +206,23 @@ -- | Original session destination IP address. destinationAddress :: Threat -> IP -destinationAddress = U.destinationAddress +{-# inline destinationAddress #-} +destinationAddress u = u.destinationAddress -- | Original session source IP address. sourceAddress :: Threat -> IP -sourceAddress = U.sourceAddress +{-# inline sourceAddress #-} +sourceAddress u = u.sourceAddress -- | Source port utilized by the session. sourcePort :: Threat -> Word16 -sourcePort = U.sourcePort +{-# inline sourcePort #-} +sourcePort u = u.sourcePort -- | Destination port utilized by the session. destinationPort :: Threat -> Word16 -destinationPort = U.destinationPort +{-# inline destinationPort #-} +destinationPort u = u.destinationPort sender :: Threat -> Bytes sender = U.sender @@ -213,19 +235,23 @@ -- | Post-NAT destination port. natDestinationPort :: Threat -> Word16 -natDestinationPort = U.natDestinationPort +{-# inline natDestinationPort #-} +natDestinationPort u = u.natDestinationPort -- | If Source NAT performed, the post-NAT Source IP address. natSourceIp :: Threat -> IP -natSourceIp = U.natSourceIp +{-# inline natSourceIp #-} +natSourceIp u = u.natSourceIp -- | If Destination NAT performed, the post-NAT Destination IP address. natDestinationIp :: Threat -> IP -natDestinationIp = U.natDestinationIp +{-# inline natDestinationIp #-} +natDestinationIp u = u.natDestinationIp -- | Post-NAT source port. natSourcePort :: Threat -> Word16 -natSourcePort = U.natSourcePort +{-# inline natSourcePort #-} +natSourcePort u = u.natSourcePort -- | Source country or Internal region for private addresses; -- maximum length is 32 bytes. @@ -238,3 +264,28 @@ destinationCountry :: Threat -> Bytes destinationCountry (Threat{destinationCountry=Bounds off len,message=msg}) = Bytes{offset=off,length=len,array=msg} + +-- | Lists the URL Filtering categories that the firewall used to enforce policy. +-- +-- This field only exists in PAN-OS 9.0 and up. +urlCategoryList :: Threat -> Maybe Bytes +{-# inline urlCategoryList #-} +urlCategoryList (Threat{urlCategoryList=x}) = case Bytes.length x of + 0 -> Nothing + _ -> Just x + +-- | IP protocol associated with the session. +ipProtocol :: Threat -> Bytes +ipProtocol (Threat{ipProtocol=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + + +-- | Number of total bytes (transmit and receive) for the session. +repeatCount :: Threat -> Word64 +{-# inline repeatCount #-} +repeatCount u = u.repeatCount + +-- | An internal numerical identifier applied to each session. +sessionId :: Threat -> Word64 +{-# inline sessionId #-} +sessionId u = u.sessionId diff --git a/src/Panos/Syslog/Traffic.hs b/src/Panos/Syslog/Traffic.hs --- a/src/Panos/Syslog/Traffic.hs +++ b/src/Panos/Syslog/Traffic.hs @@ -1,15 +1,17 @@ {-# language BangPatterns #-} +{-# language DerivingStrategies #-} +{-# language DuplicateRecordFields #-} +{-# language GeneralizedNewtypeDeriving #-} {-# language MagicHash #-} {-# language NamedFieldPuns #-} -{-# language DuplicateRecordFields #-} {-# language NumericUnderscores #-} -{-# language DerivingStrategies #-} -{-# language GeneralizedNewtypeDeriving #-} +{-# language OverloadedRecordDot #-} -- | Fields for traffic logs. module Panos.Syslog.Traffic ( -- * Fields action + , actionSource , application , bytes , bytesReceived @@ -32,10 +34,13 @@ , packets , packetsReceived , packetsSent + , repeatCount , ruleName + , ruleUuid , sequenceNumber , serialNumber , sessionEndReason + , sessionId , sourceAddress , sourceCountry , sourcePort @@ -58,6 +63,7 @@ import Panos.Syslog.Unsafe (Traffic(Traffic),Bounds(Bounds)) import Data.Word (Word64,Word16) import Net.Types (IP) +import Data.WideWord (Word128) import qualified Panos.Syslog.Unsafe as U -- | IP protocol associated with the session. @@ -90,6 +96,14 @@ ruleName (Traffic{ruleName=Bounds off len,message=msg}) = Bytes{offset=off,length=len,array=msg} +-- | Specifies whether the action taken to allow or block an +-- application was defined in the application or in policy. The +-- actions can be @allow@, @deny@, @drop@, @reset-server@, @reset-client@ +-- or @reset-both@ for the session. +actionSource :: Traffic -> Bytes +actionSource (Traffic{actionSource=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + -- | Interface that the session was sourced from. inboundInterface :: Traffic -> Bytes inboundInterface (Traffic{inboundInterface=Bounds off len,message=msg}) = @@ -154,44 +168,59 @@ -- | Source port utilized by the session. sourcePort :: Traffic -> Word16 -sourcePort = U.sourcePort +{-# inline sourcePort #-} +sourcePort u = u.sourcePort -- | Time the log was generated on the dataplane. timeGenerated :: Traffic -> Datetime -timeGenerated = U.timeGenerated +{-# inline timeGenerated #-} +timeGenerated u = u.timeGenerated -- | A 64-bit log entry identifier incremented sequentially; -- each log type has a unique number space. sequenceNumber :: Traffic -> Word64 -sequenceNumber = U.sequenceNumber +{-# inline sequenceNumber #-} +sequenceNumber u = u.sequenceNumber -- | Post-NAT source port. natSourcePort :: Traffic -> Word16 -natSourcePort = U.natSourcePort +{-# inline natSourcePort #-} +natSourcePort u = u.natSourcePort +-- | The UUID that permanently identifies the rule. +ruleUuid :: Traffic -> Word128 +{-# inline ruleUuid #-} +ruleUuid u = u.ruleUuid + -- | Destination port utilized by the session. destinationPort :: Traffic -> Word16 -destinationPort = U.destinationPort +{-# inline destinationPort #-} +destinationPort u = u.destinationPort -- | Post-NAT destination port. natDestinationPort :: Traffic -> Word16 -natDestinationPort = U.natDestinationPort +{-# inline natDestinationPort #-} +natDestinationPort u = u.natDestinationPort -- | If Source NAT performed, the post-NAT Source IP address. natSourceIp :: Traffic -> IP -natSourceIp = U.natSourceIp +{-# inline natSourceIp #-} +natSourceIp u = u.natSourceIp -- | If Destination NAT performed, the post-NAT Destination IP address. natDestinationIp :: Traffic -> IP -natDestinationIp = U.natDestinationIp +{-# inline natDestinationIp #-} +natDestinationIp u = u.natDestinationIp -- | Original session source IP address. sourceAddress :: Traffic -> IP -sourceAddress = U.sourceAddress +{-# inline sourceAddress #-} +sourceAddress u = u.sourceAddress -- | Original session destination IP address. destinationAddress :: Traffic -> IP -destinationAddress = U.destinationAddress +{-# inline destinationAddress #-} +destinationAddress u = u.destinationAddress -- | Number of total packets (transmit and receive) for the session. packets :: Traffic -> Word64 @@ -209,6 +238,11 @@ bytes :: Traffic -> Word64 bytes = U.bytes +-- | Number of total bytes (transmit and receive) for the session. +repeatCount :: Traffic -> Word64 +{-# inline repeatCount #-} +repeatCount u = u.repeatCount + -- | Serial number of the firewall that generated the log. These -- occassionally contain non-numeric characters, so do not attempt -- to parse this as a decimal number. @@ -217,16 +251,20 @@ Bytes{offset=off,length=len,array=msg} deviceGroupHierarchyLevel1 :: Traffic -> Word64 -deviceGroupHierarchyLevel1 = U.deviceGroupHierarchyLevel1 +{-# inline deviceGroupHierarchyLevel1 #-} +deviceGroupHierarchyLevel1 u = u.deviceGroupHierarchyLevel1 deviceGroupHierarchyLevel2 :: Traffic -> Word64 -deviceGroupHierarchyLevel2 = U.deviceGroupHierarchyLevel2 +{-# inline deviceGroupHierarchyLevel2 #-} +deviceGroupHierarchyLevel2 u = u.deviceGroupHierarchyLevel2 deviceGroupHierarchyLevel3 :: Traffic -> Word64 -deviceGroupHierarchyLevel3 = U.deviceGroupHierarchyLevel3 +{-# inline deviceGroupHierarchyLevel3 #-} +deviceGroupHierarchyLevel3 u = u.deviceGroupHierarchyLevel3 deviceGroupHierarchyLevel4 :: Traffic -> Word64 -deviceGroupHierarchyLevel4 = U.deviceGroupHierarchyLevel4 +{-# inline deviceGroupHierarchyLevel4 #-} +deviceGroupHierarchyLevel4 u = u.deviceGroupHierarchyLevel4 -- | The reason a session terminated. sessionEndReason :: Traffic -> Bytes @@ -265,3 +303,8 @@ destinationCountry :: Traffic -> Bytes destinationCountry (Traffic{destinationCountry=Bounds off len,message=msg}) = Bytes{offset=off,length=len,array=msg} + +-- | An internal numerical identifier applied to each session. +sessionId :: Traffic -> Word64 +{-# inline sessionId #-} +sessionId u = u.sessionId diff --git a/src/Panos/Syslog/Unsafe.hs b/src/Panos/Syslog/Unsafe.hs --- a/src/Panos/Syslog/Unsafe.hs +++ b/src/Panos/Syslog/Unsafe.hs @@ -14,1232 +14,157 @@ , Traffic(..) , Threat(..) , System(..) - , Field(..) - , Bounds(..) - -- * Decoding - , decode - ) where - -import Chronos (DayOfMonth(..),Date(..)) -import Chronos (Year(..),Month(..),Datetime(..),TimeOfDay(..)) -import Control.Exception (Exception) -import Control.Monad.ST.Run (runByteArrayST) -import Data.Bytes.Parser (Parser) -import Data.Bytes.Types (Bytes(..),UnmanagedBytes(UnmanagedBytes)) -import Data.Char (ord,isAsciiUpper,isAsciiLower) -import Data.Primitive (ByteArray) -import Data.Primitive.Addr (Addr(Addr)) -import Data.Word (Word64,Word32,Word16,Word8) -import GHC.Exts (Ptr(Ptr),Int(I#),Int#,Addr#) -import Net.Types (IP) - -import qualified Control.Exception -import qualified Data.Bytes.Parser as P -import qualified Data.Bytes.Parser.Ascii as Ascii -import qualified Data.Bytes.Parser.Latin as Latin -import qualified Data.Bytes.Parser.Unsafe as Unsafe -import qualified Data.Primitive as PM -import qualified Data.Primitive.Ptr as PM -import qualified GHC.Exts as Exts -import qualified GHC.Pack -import qualified Net.IP as IP -import qualified Net.IPv4 as IPv4 - --- | Sum that represents all known PAN-OS syslog types. Use 'decode' --- to parse a byte sequence into a structured log. -data Log - = LogTraffic !Traffic - | LogThreat !Threat - | LogSystem !System - | LogOther - -data Bounds = Bounds - {-# UNPACK #-} !Int -- offset - {-# UNPACK #-} !Int -- length - --- | A PAN-OS system log. Read-only accessors are found in --- @Panos.Syslog.System@. -data System = System - { message :: {-# UNPACK #-} !ByteArray - -- The original log - , syslogHost :: {-# UNPACK #-} !Bounds - -- The host as presented in the syslog preamble that - -- prefixes the message. - , receiveTime :: {-# UNPACK #-} !Datetime - -- In log, presented as: 2019/06/18 15:10:20 - , serialNumber :: {-# UNPACK #-} !Bounds - -- In log, presented as: 002610378847 - , subtype :: {-# UNPACK #-} !Bounds - -- Presented as: dhcp, dnsproxy, dos, general, etc. - , timeGenerated :: {-# UNPACK #-} !Datetime - -- Presented as: 2019/11/04 08:39:05 - , virtualSystem :: {-# UNPACK #-} !Bounds - , eventId :: {-# UNPACK #-} !Bounds - , object :: {-# UNPACK #-} !Bounds - , module_ :: {-# UNPACK #-} !Bounds - , severity :: {-# UNPACK #-} !Bounds - , descriptionBounds :: {-# UNPACK #-} !Bounds - , descriptionByteArray :: {-# UNPACK #-} !ByteArray - , sequenceNumber :: {-# UNPACK #-} !Word64 - , actionFlags :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel1 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel2 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel3 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel4 :: {-# UNPACK #-} !Word64 - , virtualSystemName :: {-# UNPACK #-} !Bounds - , deviceName :: {-# UNPACK #-} !Bounds - } - --- | A PAN-OS traffic log. Read-only accessors are found in --- @Panos.Syslog.Traffic@. -data Traffic = Traffic - { message :: {-# UNPACK #-} !ByteArray - -- The original log - , syslogHost :: {-# UNPACK #-} !Bounds - -- The host as presented in the syslog preamble that - -- prefixes the message. - , receiveTime :: {-# UNPACK #-} !Datetime - -- In log, presented as: 2019/06/18 15:10:20 - , serialNumber :: {-# UNPACK #-} !Bounds - -- In log, presented as: 002610378847 - , subtype :: {-# UNPACK #-} !Bounds - -- Presented as: start, end, drop, and deny - , timeGenerated :: {-# UNPACK #-} !Datetime - -- Presented as: 2018/04/11 23:19:22 - , sourceAddress :: {-# UNPACK #-} !IP - , destinationAddress :: {-# UNPACK #-} !IP - , natSourceIp :: {-# UNPACK #-} !IP - , natDestinationIp :: {-# UNPACK #-} !IP - , ruleName :: {-# UNPACK #-} !Bounds - , sourceUser :: {-# UNPACK #-} !Bounds - , destinationUser :: {-# UNPACK #-} !Bounds - , application :: {-# UNPACK #-} !Bounds - , virtualSystem :: {-# UNPACK #-} !Bounds - , sourceZone :: {-# UNPACK #-} !Bounds - , destinationZone :: {-# UNPACK #-} !Bounds - , inboundInterface :: {-# UNPACK #-} !Bounds - , outboundInterface :: {-# UNPACK #-} !Bounds - , logAction :: {-# UNPACK #-} !Bounds - , sessionId :: {-# UNPACK #-} !Word64 - , repeatCount :: {-# UNPACK #-} !Word64 - , sourcePort :: {-# UNPACK #-} !Word16 - , destinationPort :: {-# UNPACK #-} !Word16 - , natSourcePort :: {-# UNPACK #-} !Word16 - , natDestinationPort :: {-# UNPACK #-} !Word16 - , flags :: {-# UNPACK #-} !Word32 - -- Presented as: 0x400053 - , ipProtocol :: {-# UNPACK #-} !Bounds - -- Presented as: tcp, udp, etc. - , action :: {-# UNPACK #-} !Bounds - , bytes :: {-# UNPACK #-} !Word64 - , bytesSent :: {-# UNPACK #-} !Word64 - , bytesReceived :: {-# UNPACK #-} !Word64 - , packets :: {-# UNPACK #-} !Word64 - , startTime :: {-# UNPACK #-} !Datetime - , elapsedTime :: {-# UNPACK #-} !Word64 - , category :: {-# UNPACK #-} !Bounds - , sequenceNumber :: {-# UNPACK #-} !Word64 - , actionFlags :: {-# UNPACK #-} !Word64 - -- Presented as: 0x8000000000000000 - , sourceCountry :: {-# UNPACK #-} !Bounds - , destinationCountry :: {-# UNPACK #-} !Bounds - , packetsSent :: {-# UNPACK #-} !Word64 - , packetsReceived :: {-# UNPACK #-} !Word64 - , sessionEndReason :: {-# UNPACK #-} !Bounds - , deviceGroupHierarchyLevel1 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel2 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel3 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel4 :: {-# UNPACK #-} !Word64 - , virtualSystemName :: {-# UNPACK #-} !Bounds - , deviceName :: {-# UNPACK #-} !Bounds - , actionSource :: {-# UNPACK #-} !Bounds - } - --- | A PAN-OS threat log. Read-only accessors are found in --- @Panos.Syslog.Threat@. -data Threat = Threat - { message :: {-# UNPACK #-} !ByteArray - -- The original log - , syslogHost :: {-# UNPACK #-} !Bounds - -- The host as presented in the syslog preamble that - -- prefixes the message. - , receiveTime :: {-# UNPACK #-} !Datetime - -- In log, presented as: 2019/06/18 15:10:20 - , serialNumber :: {-# UNPACK #-} !Bounds - -- In log, presented as: 002610378847 - , subtype :: {-# UNPACK #-} !Bounds - -- Presented as: data, file, flood, packet, scan, spyware, url, - -- virus, vulnerability, wildfire, or wildfire-virus. - , timeGenerated :: {-# UNPACK #-} !Datetime - -- Presented as: 2018/04/11 23:19:22 - , sourceAddress :: {-# UNPACK #-} !IP - , destinationAddress :: {-# UNPACK #-} !IP - , natSourceIp :: {-# UNPACK #-} !IP - , natDestinationIp :: {-# UNPACK #-} !IP - , ruleName :: {-# UNPACK #-} !Bounds - , sourceUser :: {-# UNPACK #-} !Bounds - , destinationUser :: {-# UNPACK #-} !Bounds - , application :: {-# UNPACK #-} !Bounds - , virtualSystem :: {-# UNPACK #-} !Bounds - , sourceZone :: {-# UNPACK #-} !Bounds - , destinationZone :: {-# UNPACK #-} !Bounds - , inboundInterface :: {-# UNPACK #-} !Bounds - , outboundInterface :: {-# UNPACK #-} !Bounds - , logAction :: {-# UNPACK #-} !Bounds - , sessionId :: {-# UNPACK #-} !Word64 - , repeatCount :: {-# UNPACK #-} !Word64 - , sourcePort :: {-# UNPACK #-} !Word16 - , destinationPort :: {-# UNPACK #-} !Word16 - , natSourcePort :: {-# UNPACK #-} !Word16 - , natDestinationPort :: {-# UNPACK #-} !Word16 - , action :: {-# UNPACK #-} !Bounds - , ipProtocol :: {-# UNPACK #-} !Bounds - , flags :: {-# UNPACK #-} !Word32 - , miscellaneousBounds :: {-# UNPACK #-} !Bounds - , miscellaneousByteArray :: {-# UNPACK #-} !ByteArray - , threatName :: {-# UNPACK #-} !Bounds - , threatId :: {-# UNPACK #-} !Word64 - , category :: {-# UNPACK #-} !Bounds - , severity :: {-# UNPACK #-} !Bounds - , direction :: {-# UNPACK #-} !Bounds - , sequenceNumber :: {-# UNPACK #-} !Word64 - , actionFlags :: {-# UNPACK #-} !Word64 - -- Presented as: 0x8000000000000000 - , sourceCountry :: {-# UNPACK #-} !Bounds - , destinationCountry :: {-# UNPACK #-} !Bounds - , contentType :: {-# UNPACK #-} !Bounds - , pcapId :: {-# UNPACK #-} !Word64 - , fileDigest :: {-# UNPACK #-} !Bounds - -- Only used by wildfire subtype - -- TODO: make the file digest a 128-bit or 256-bit word - , cloud :: {-# UNPACK #-} !Bounds - -- Only used by wildfire subtype - , urlIndex :: {-# UNPACK #-} !Word64 - -- Only used by wildfire subtype - , userAgentBounds :: {-# UNPACK #-} !Bounds - , userAgentByteArray :: {-# UNPACK #-} !ByteArray - -- Only used by url filtering subtype. This field may have - -- escaped characters, so we include the possibility of - -- using a byte array distinct from the original log. - , fileType :: {-# UNPACK #-} !Bounds - -- Only used by wildfire subtype - , forwardedFor :: {-# UNPACK #-} !Bounds - -- Only used by url filtering subtype - , referer :: {-# UNPACK #-} !Bytes - -- Only used by url filtering subtype - , sender :: {-# UNPACK #-} !Bytes - -- Only used by wildfire subtype - , subject :: {-# UNPACK #-} !Bytes - -- Only used by wildfire subtype - , recipient :: {-# UNPACK #-} !Bytes - -- Only used by wildfire subtype - , reportId :: {-# UNPACK #-} !Bounds - -- Only used by wildfire subtype - , deviceGroupHierarchyLevel1 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel2 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel3 :: {-# UNPACK #-} !Word64 - , deviceGroupHierarchyLevel4 :: {-# UNPACK #-} !Word64 - , virtualSystemName :: {-# UNPACK #-} !Bounds - , deviceName :: {-# UNPACK #-} !Bounds - -- TODO: skipping over uuid fields for now - , httpMethod :: {-# UNPACK #-} !Bounds - , tunnelId :: {-# UNPACK #-} !Word64 - , parentSessionId :: {-# UNPACK #-} !Word64 - -- Only used by url subtype - , threatCategory :: {-# UNPACK #-} !Bounds - , contentVersion :: {-# UNPACK #-} !Bounds - -- TODO: skipping some fields here - , sctpAssociationId :: {-# UNPACK #-} !Word64 - , payloadProtocolId :: {-# UNPACK #-} !Word64 - -- TODO: skipping over other fields here - , httpHeaders :: {-# UNPACK #-} !Bytes - } - --- | The field that was being parsed when a parse failure occurred. --- This is typically for useful for libary developers, but to present --- it to the end user, call @show@ or @throwIO@. -newtype Field = Field UnmanagedBytes - -instance Show Field where - showsPrec _ (Field (UnmanagedBytes (Addr addr) _)) s = - '"' : GHC.Pack.unpackAppendCString# addr ('"' : s) - -instance Exception Field where - displayException (Field (UnmanagedBytes (Addr addr) _)) = - GHC.Pack.unpackCString# addr - -syslogPriorityField :: Field -syslogPriorityField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "syslogPriority"# - -syslogHostField :: Field -syslogHostField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "syslogHost"# - -syslogDatetimeField :: Field -syslogDatetimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "syslogDatetime"# - -receiveTimeDateField :: Field -receiveTimeDateField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "receiveTime:date"# - -receiveTimeTimeField :: Field -receiveTimeTimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "receiveTime:time"# - -serialNumberField :: Field -serialNumberField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "serialNumber"# - -typeField :: Field -typeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "type"# - -subtypeField :: Field -subtypeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "subtype"# - -timeGeneratedDateField :: Field -timeGeneratedDateField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "timeGenerated:date"# - -timeGeneratedTimeField :: Field -timeGeneratedTimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "timeGenerated:time"# - -sourceAddressField :: Field -sourceAddressField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "sourceAddress"# - -destinationAddressField :: Field -destinationAddressField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "destinationAddress"# - -natSourceIpField :: Field -natSourceIpField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "natSourceIp"# - -natDestinationIpField :: Field -natDestinationIpField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "natDestinationIp"# - -ruleNameField :: Field -ruleNameField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "ruleName"# - -sourceUserField :: Field -sourceUserField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "sourceUser"# - -destinationUserField :: Field -destinationUserField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "destinationUser"# - -applicationField :: Field -applicationField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "application"# - -virtualSystemField :: Field -virtualSystemField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "virtualSystem"# - -sourceZoneField :: Field -sourceZoneField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "sourceZone"# - -destinationZoneField :: Field -destinationZoneField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "destinationZone"# - -inboundInterfaceField :: Field -inboundInterfaceField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "inboundInterface"# - -outboundInterfaceField :: Field -outboundInterfaceField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "outboundInterface"# - -logActionField :: Field -logActionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "logAction"# - -sessionIdField :: Field -sessionIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "sessionId"# - -repeatCountField :: Field -repeatCountField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "repeatCount"# - -sourcePortField :: Field -sourcePortField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "sourcePort"# - -destinationPortField :: Field -destinationPortField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "destinationPort"# - -natSourcePortField :: Field -natSourcePortField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "natSourcePort"# - -natDestinationPortField :: Field -natDestinationPortField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "natDestinationPort"# - -flagsField :: Field -flagsField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "flags"# - -ipProtocolField :: Field -ipProtocolField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "ipProtocol"# - -actionField :: Field -actionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "action"# - -bytesField :: Field -bytesField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "bytes"# - -bytesSentField :: Field -bytesSentField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "bytesSent"# - -bytesReceivedField :: Field -bytesReceivedField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "bytesReceived"# - -packetsField :: Field -packetsField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "packets"# - -startTimeDateField :: Field -startTimeDateField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "startTime:date"# - -startTimeTimeField :: Field -startTimeTimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "startTime:time"# - -elapsedTimeField :: Field -elapsedTimeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "elapsedTime"# - -categoryField :: Field -categoryField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "category"# - -sequenceNumberField :: Field -sequenceNumberField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "sequenceNumber"# - -actionFlagsField :: Field -actionFlagsField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "actionFlags"# - -sourceCountryField :: Field -sourceCountryField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "sourceCountry"# - -destinationCountryField :: Field -destinationCountryField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "destinationCountry"# - -packetsSentField :: Field -packetsSentField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "packetsSent"# - -packetsReceivedField :: Field -packetsReceivedField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "packetsReceived"# - -sessionEndReasonField :: Field -sessionEndReasonField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "sessionEndReason"# - -deviceGroupHierarchyLevel1Field :: Field -deviceGroupHierarchyLevel1Field = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "deviceGroupHierarchyLevel1"# - -deviceGroupHierarchyLevel2Field :: Field -deviceGroupHierarchyLevel2Field = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "deviceGroupHierarchyLevel2"# - -deviceGroupHierarchyLevel3Field :: Field -deviceGroupHierarchyLevel3Field = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "deviceGroupHierarchyLevel3"# - -deviceGroupHierarchyLevel4Field :: Field -deviceGroupHierarchyLevel4Field = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "deviceGroupHierarchyLevel4"# - -virtualSystemNameField :: Field -virtualSystemNameField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "virtualSystemName"# - -payloadProtocolField :: Field -payloadProtocolField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:payloadProtocol"# - -senderField :: Field -senderField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:sender"# - -recipientField :: Field -recipientField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:recipient"# - -refererField :: Field -refererField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:referer"# - -pcapIdField :: Field -pcapIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:pcapId"# - -directionField :: Field -directionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:direction"# - -contentTypeField :: Field -contentTypeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:contentType"# - -severityField :: Field -severityField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:severity"# - -cloudField :: Field -cloudField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:cloud"# - -threatCategoryField :: Field -threatCategoryField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:threatCategory"# - -urlIndexField :: Field -urlIndexField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:urlIndex"# - -fileDigestField :: Field -fileDigestField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:fileDigest"# - -fileTypeField :: Field -fileTypeField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:fileType"# - -forwardedForField :: Field -forwardedForField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:forwardedFor"# - -userAgentField :: Field -userAgentField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:userAgent"# - -subjectField :: Field -subjectField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:subject"# - -contentVersionField :: Field -contentVersionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:contentVersion"# - -httpMethodField :: Field -httpMethodField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:httpMethod"# - -httpHeadersField :: Field -httpHeadersField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:httpHeaders"# - -reportIdField :: Field -reportIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:reportId"# - -miscellaneousField :: Field -miscellaneousField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:miscellaneous"# - -threatIdField :: Field -threatIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:threatId"# - -deviceNameField :: Field -deviceNameField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "deviceName"# - -actionSourceField :: Field -actionSourceField = Field (UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "actionSource"# - -futureUseAField :: Field -futureUseAField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "futureUse:A"# - -futureUseBField :: Field -futureUseBField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "futureUse:B"# - -futureUseCField :: Field -futureUseCField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "futureUse:C"# - -futureUseDField :: Field -futureUseDField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "futureUse:D"# - -futureUseEField :: Field -futureUseEField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "futureUse:E"# - -futureUseFField :: Field -futureUseFField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "futureUse:F"# - -futureUseGField :: Field -futureUseGField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "futureUse:G"# - -leftoversField :: Field -leftoversField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "framing:leftovers"# - -sourceVmUuidField :: Field -sourceVmUuidField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:source_uuid"# - -destinationVmUuidField :: Field -destinationVmUuidField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:dst_uuid"# - -tunnelIdField :: Field -tunnelIdField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:tunnelid"# - -monitorTagField :: Field -monitorTagField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:monitortag"# - -parentSessionIdField :: Field -parentSessionIdField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:parent_session_id"# - -parentStartTimeField :: Field -parentStartTimeField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:parent_start_time"# - -tunnelTypeField :: Field -tunnelTypeField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:tunnel"# - -sctpAssociationIdField :: Field -sctpAssociationIdField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:assoc_id"# - -sctpChunksField :: Field -sctpChunksField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:chunks"# - -sctpChunksSentField :: Field -sctpChunksSentField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:chunks_sent"# - -sctpChunksReceivedField :: Field -sctpChunksReceivedField = Field (UnmanagedBytes (Addr x#) (I# (cstringLen# x#))) - where !x# = "field:chunks_received"# - -moduleField :: Field -moduleField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:module"# - -descriptionField :: Field -descriptionField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:description"# - -eventIdField :: Field -eventIdField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:eventId"# - -objectField :: Field -objectField = Field ( UnmanagedBytes (Addr x#) (I# ( cstringLen# x#))) - where !x# = "field:object"# - - -untilSpace :: e -> Parser e s Bounds -{-# inline untilSpace #-} -untilSpace e = do - start <- Unsafe.cursor - Latin.skipTrailedBy e ' ' - endSucc <- Unsafe.cursor - let end = endSucc - 1 - pure (Bounds start (end - start)) - -untilComma :: e -> Parser e s Bounds -{-# inline untilComma #-} -untilComma e = do - start <- Unsafe.cursor - Latin.skipTrailedBy e ',' - endSucc <- Unsafe.cursor - let end = endSucc - 1 - pure (Bounds start (end - start)) - -skipThroughComma :: e -> Parser e s () -{-# inline skipThroughComma #-} -skipThroughComma e = Latin.skipTrailedBy e ',' - --- There should not be any more commas left in the input. --- This takes until it finds a comma or until end of input --- is reached. -finalField :: Parser e s Bounds -{-# inline finalField #-} -finalField = do - start <- Unsafe.cursor - Latin.skipUntil ',' - end <- Unsafe.cursor - pure (Bounds start (end - start)) - --- This does not require that any digits are --- actually present. -skipDigitsThroughComma :: e -> Parser e s () -{-# inline skipDigitsThroughComma #-} -skipDigitsThroughComma e = - Latin.skipDigits *> Latin.char e ',' - -w64Comma :: e -> Parser e s Word64 -{-# inline w64Comma #-} -w64Comma e = do - w <- Latin.decWord64 e - Latin.char e ',' - pure w - -w16Comma :: e -> Parser e s Word16 -{-# inline w16Comma #-} -w16Comma e = Latin.decWord16 e <* Latin.char e ',' - --- Returns the receive time and the serial number. There is a --- little subtlety here. The PANOS guide says that logs should --- start with something like: --- 1,2019/07/14 10:26:22,005923187997 --- The leading field is reserved for future use. However, there --- is typically an additional prefix consisting of a syslog priority, --- another datetime (in a different format), and a hostname: --- <14> Jul 14 11:26:23 MY-HOST.example.com 1,... --- The datetime is within typically within a second of the other one. --- Additionally, it's missing the year. So, we discard it. The --- syslog priority is worthless, so we throw it out as well. The --- host name, however, does provide useful information that does --- not exist elsewhere in the log. We should be as flexible --- as possible with this somewhat fragile part of the log. -parserPrefix :: Parser Field s (Bounds,Datetime,Bounds) -{-# inline parserPrefix #-} -parserPrefix = do - Latin.skipChar ' ' - -- We allow the syslog priority (the number in angle brackets) - -- to be absent. - Latin.trySatisfy (== '<') >>= \case - True -> do - Latin.skipTrailedBy syslogPriorityField '>' - Latin.skipChar ' ' - False -> pure () - Ascii.skipAlpha1 syslogDatetimeField -- Month - Latin.skipChar1 syslogDatetimeField ' ' - Latin.skipDigits1 syslogDatetimeField -- Day - Latin.skipChar1 syslogDatetimeField ' ' - Latin.skipDigits1 syslogDatetimeField -- Hour - Latin.char syslogDatetimeField ':' - Latin.skipDigits1 syslogDatetimeField -- Minute - Latin.char syslogDatetimeField ':' - Latin.skipDigits1 syslogDatetimeField -- Second - Latin.skipChar1 syslogDatetimeField ' ' - hostBounds <- untilSpace syslogHostField - Latin.skipChar ' ' - skipThroughComma futureUseDField - !recv <- parserDatetime receiveTimeDateField receiveTimeTimeField - !ser <- untilComma serialNumberField - pure (hostBounds,recv,ser) - --- | Decode a PAN-OS syslog message of an unknown type. -decode :: Bytes -> Either Field Log -decode b = case P.parseBytes parserLog b of - P.Failure e -> Left e - P.Success (P.Slice _ len r) -> case len of - 0 -> Right r - _ -> Left leftoversField - -parserLog :: Parser Field s Log -parserLog = do - (!hostBounds,!receiveTime,!serialNumber) <- parserPrefix - Latin.any typeField >>= \case - 'S' -> do - Latin.char6 typeField 'Y' 'S' 'T' 'E' 'M' ',' - !x <- parserSystem hostBounds receiveTime serialNumber - pure (LogSystem x) - 'T' -> Latin.any typeField >>= \case - 'R' -> do - Latin.char6 typeField 'A' 'F' 'F' 'I' 'C' ',' - !x <- parserTraffic hostBounds receiveTime serialNumber - pure (LogTraffic x) - 'H' -> do - Latin.char5 typeField 'R' 'E' 'A' 'T' ',' - !x <- parserThreat hostBounds receiveTime serialNumber - pure (LogThreat x) - _ -> P.fail typeField - _ -> P.fail typeField - -parserTraffic :: Bounds -> Datetime -> Bounds -> Parser Field s Traffic -parserTraffic !syslogHost receiveTime !serialNumber = do - subtype <- untilComma subtypeField - skipThroughComma futureUseAField - -- The datetime parser consumes the trailing comma - timeGenerated <- parserDatetime timeGeneratedDateField timeGeneratedTimeField - sourceAddress <- IP.fromIPv4 <$> IPv4.parserUtf8Bytes sourceAddressField - Latin.char sourceAddressField ',' - destinationAddress <- IP.fromIPv4 <$> IPv4.parserUtf8Bytes destinationAddressField - Latin.char destinationAddressField ',' - natSourceIp <- IP.fromIPv4 <$> IPv4.parserUtf8Bytes natSourceIpField - Latin.char natSourceIpField ',' - natDestinationIp <- IP.fromIPv4 <$> IPv4.parserUtf8Bytes natDestinationIpField - Latin.char natDestinationIpField ',' - ruleName <- untilComma ruleNameField - sourceUser <- untilComma sourceUserField - destinationUser <- untilComma destinationUserField - application <- untilComma applicationField - virtualSystem <- untilComma virtualSystemField - sourceZone <- untilComma sourceZoneField - destinationZone <- untilComma destinationZoneField - inboundInterface <- untilComma inboundInterfaceField - outboundInterface <- untilComma outboundInterfaceField - logAction <- untilComma logActionField - skipThroughComma futureUseBField - sessionId <- w64Comma sessionIdField - repeatCount <- w64Comma repeatCountField - sourcePort <- w16Comma sourcePortField - destinationPort <- w16Comma destinationPortField - natSourcePort <- w16Comma natSourcePortField - natDestinationPort <- w16Comma natDestinationPortField - -- TODO: handle the flags - Latin.char actionFlagsField '0' - Latin.char actionFlagsField 'x' - _ <- untilComma flagsField - let flags = 0 - ipProtocol <- untilComma ipProtocolField - action <- untilComma actionField - bytes <- w64Comma bytesField - bytesSent <- w64Comma bytesSentField - bytesReceived <- w64Comma bytesReceivedField - packets <- w64Comma packetsField - startTime <- parserDatetime startTimeDateField startTimeTimeField - elapsedTime <- w64Comma elapsedTimeField - category <- untilComma categoryField - skipThroughComma futureUseCField - sequenceNumber <- w64Comma sequenceNumberField - -- TODO: handle action flags - Latin.char actionFlagsField '0' - Latin.char actionFlagsField 'x' - _ <- untilComma actionFlagsField - let actionFlags = 0 - sourceCountry <- untilComma sourceCountryField - destinationCountry <- untilComma destinationCountryField - skipThroughComma futureUseEField - packetsSent <- w64Comma packetsSentField - packetsReceived <- w64Comma packetsReceivedField - sessionEndReason <- untilComma sessionEndReasonField - deviceGroupHierarchyLevel1 <- w64Comma deviceGroupHierarchyLevel1Field - deviceGroupHierarchyLevel2 <- w64Comma deviceGroupHierarchyLevel2Field - deviceGroupHierarchyLevel3 <- w64Comma deviceGroupHierarchyLevel3Field - deviceGroupHierarchyLevel4 <- w64Comma deviceGroupHierarchyLevel4Field - virtualSystemName <- untilComma virtualSystemNameField - deviceName <- untilComma deviceNameField - actionSource <- untilComma actionSourceField - skipThroughComma sourceVmUuidField - skipThroughComma destinationVmUuidField - skipThroughComma tunnelIdField - skipThroughComma monitorTagField - skipThroughComma parentSessionIdField - skipThroughComma parentStartTimeField - skipThroughComma tunnelTypeField - skipThroughComma sctpAssociationIdField - skipDigitsThroughComma sctpChunksField - skipDigitsThroughComma sctpChunksSentField - Latin.skipDigits1 sctpChunksReceivedField - message <- Unsafe.expose - pure Traffic - { subtype , timeGenerated , sourceAddress , destinationAddress - , natSourceIp , natDestinationIp , ruleName , sourceUser - , destinationUser , application , virtualSystem , sourceZone - , destinationZone , inboundInterface , outboundInterface , logAction - , sessionId , repeatCount , sourcePort , destinationPort - , natSourcePort , natDestinationPort , ipProtocol - , action , bytes , bytesSent , bytesReceived - , packets , startTime , elapsedTime , category - , sequenceNumber , sourceCountry , destinationCountry - , packetsSent , sessionEndReason - , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 - , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 - , virtualSystemName , deviceName , actionSource , receiveTime - , serialNumber, packetsReceived, actionFlags, flags, message - , syslogHost - } - -parserSystem :: Bounds -> Datetime -> Bounds -> Parser Field s System -parserSystem syslogHost receiveTime serialNumber = do - subtype <- untilComma subtypeField - skipThroughComma futureUseAField - -- The datetime parser consumes the trailing comma - timeGenerated <- parserDatetime timeGeneratedDateField timeGeneratedTimeField - virtualSystem <- untilComma virtualSystemField - eventId <- untilComma eventIdField - object <- untilComma objectField - skipThroughComma futureUseBField - skipThroughComma futureUseCField - module_ <- untilComma moduleField - severity <- untilComma severityField - Bytes{array=descriptionByteArray,offset=descrOff,length=descrLen} <- - parserOptionallyQuoted descriptionField - let descriptionBounds = Bounds descrOff descrLen - sequenceNumber <- w64Comma sequenceNumberField - -- TODO: handle action flags - Latin.char actionFlagsField '0' - Latin.char actionFlagsField 'x' - _ <- untilComma actionFlagsField - let actionFlags = 0 - deviceGroupHierarchyLevel1 <- w64Comma deviceGroupHierarchyLevel1Field - deviceGroupHierarchyLevel2 <- w64Comma deviceGroupHierarchyLevel2Field - deviceGroupHierarchyLevel3 <- w64Comma deviceGroupHierarchyLevel3Field - deviceGroupHierarchyLevel4 <- w64Comma deviceGroupHierarchyLevel4Field - virtualSystemName <- untilComma virtualSystemNameField - deviceName <- finalField - message <- Unsafe.expose - pure System - { subtype , timeGenerated - , sequenceNumber - , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 - , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 - , virtualSystemName , deviceName , receiveTime - , serialNumber, actionFlags, message - , syslogHost, virtualSystem, eventId, object, module_ - , severity, descriptionBounds, descriptionByteArray - } - -parserThreat :: Bounds -> Datetime -> Bounds -> Parser Field s Threat -parserThreat !syslogHost receiveTime !serialNumber = do - subtype <- untilComma subtypeField - skipThroughComma futureUseAField - -- the datetime parser also grabs the trailing comma - timeGenerated <- parserDatetime timeGeneratedDateField timeGeneratedTimeField - sourceAddress <- IP.fromIPv4 <$> IPv4.parserUtf8Bytes sourceAddressField - Latin.char sourceAddressField ',' - destinationAddress <- IP.fromIPv4 <$> IPv4.parserUtf8Bytes destinationAddressField - Latin.char destinationAddressField ',' - natSourceIp <- IP.fromIPv4 <$> IPv4.parserUtf8Bytes natSourceIpField - Latin.char natSourceIpField ',' - natDestinationIp <- IP.fromIPv4 <$> IPv4.parserUtf8Bytes natDestinationIpField - Latin.char natDestinationIpField ',' - ruleName <- untilComma ruleNameField - sourceUser <- untilComma sourceUserField - destinationUser <- untilComma destinationUserField - application <- untilComma applicationField - virtualSystem <- untilComma virtualSystemField - sourceZone <- untilComma sourceZoneField - destinationZone <- untilComma destinationZoneField - inboundInterface <- untilComma inboundInterfaceField - outboundInterface <- untilComma outboundInterfaceField - logAction <- untilComma logActionField - skipThroughComma futureUseBField - sessionId <- w64Comma sessionIdField - repeatCount <- w64Comma repeatCountField - sourcePort <- w16Comma sourcePortField - destinationPort <- w16Comma destinationPortField - natSourcePort <- w16Comma natSourcePortField - natDestinationPort <- w16Comma natDestinationPortField - -- TODO: handle the flags - Latin.char actionFlagsField '0' - Latin.char actionFlagsField 'x' - _ <- untilComma flagsField - let flags = 0 - ipProtocol <- untilComma ipProtocolField - action <- untilComma actionField - Bytes{array=miscellaneousByteArray,offset=miscOff,length=miscLen} <- - parserOptionallyQuoted miscellaneousField - let miscellaneousBounds = Bounds miscOff miscLen - (threatName,threatId) <- parserThreatId - category <- untilComma categoryField - severity <- untilComma severityField - direction <- untilComma directionField - sequenceNumber <- w64Comma sequenceNumberField - -- TODO: handle action flags - Latin.char actionFlagsField '0' - Latin.char actionFlagsField 'x' - _ <- untilComma actionFlagsField - let actionFlags = 0 - sourceCountry <- untilComma sourceCountryField - destinationCountry <- untilComma destinationCountryField - skipThroughComma futureUseEField - contentType <- untilComma contentTypeField - pcapId <- w64Comma pcapIdField - fileDigest <- untilComma fileDigestField - cloud <- untilComma cloudField - urlIndex <- w64Comma urlIndexField - Bytes{array=userAgentByteArray,offset=uaOff,length=uaLen} <- - parserOptionallyQuoted userAgentField - let userAgentBounds = Bounds uaOff uaLen - fileType <- untilComma fileTypeField - forwardedFor <- untilComma forwardedForField - referer <- parserOptionallyQuoted refererField - sender <- parserOptionallyQuoted senderField - subject <- parserOptionallyQuoted subjectField - recipient <- parserOptionallyQuoted recipientField - reportId <- untilComma reportIdField - deviceGroupHierarchyLevel1 <- w64Comma deviceGroupHierarchyLevel1Field - deviceGroupHierarchyLevel2 <- w64Comma deviceGroupHierarchyLevel2Field - deviceGroupHierarchyLevel3 <- w64Comma deviceGroupHierarchyLevel3Field - deviceGroupHierarchyLevel4 <- w64Comma deviceGroupHierarchyLevel4Field - virtualSystemName <- untilComma virtualSystemNameField - deviceName <- untilComma deviceNameField - parserOptionallyQuoted_ futureUseFField - skipThroughComma sourceVmUuidField - skipThroughComma destinationVmUuidField - httpMethod <- untilComma httpMethodField - tunnelId <- w64Comma tunnelIdField - skipThroughComma monitorTagField - parentSessionId <- w64Comma parentSessionIdField - skipThroughComma parentStartTimeField - skipThroughComma tunnelTypeField - threatCategory <- untilComma threatCategoryField - contentVersion <- untilComma contentVersionField - skipThroughComma futureUseGField - sctpAssociationId <- w64Comma sctpAssociationIdField - payloadProtocolId <- w64Comma payloadProtocolField - -- TODO: Handle HTTP Headers correctly - httpHeaders <- finalOptionallyQuoted httpHeadersField - message <- Unsafe.expose - pure Threat - { subtype , timeGenerated , sourceAddress , destinationAddress - , natSourceIp , natDestinationIp , ruleName , sourceUser - , destinationUser , application , virtualSystem , sourceZone - , destinationZone , inboundInterface , outboundInterface , logAction - , sessionId , repeatCount , sourcePort , destinationPort - , natSourcePort , natDestinationPort , ipProtocol - , action , category - , sequenceNumber , sourceCountry , destinationCountry - , deviceGroupHierarchyLevel1 , deviceGroupHierarchyLevel2 - , deviceGroupHierarchyLevel3 , deviceGroupHierarchyLevel4 - , virtualSystemName , deviceName , receiveTime - , serialNumber, actionFlags, flags, message - , syslogHost, threatId, severity, direction, threatName - , contentType, pcapId - , fileDigest, cloud, urlIndex - , userAgentBounds, sctpAssociationId - , userAgentByteArray, fileType - , forwardedFor, referer - , sender, subject, recipient - , reportId, httpMethod, contentVersion - , threatCategory, miscellaneousBounds, miscellaneousByteArray - , payloadProtocolId, parentSessionId, tunnelId - , httpHeaders - } - --- Threat IDs are weird. There are three different kinds of --- strings that can show up here: --- --- * (9999) --- * Microsoft RPC Endpoint Mapper Detection(30845) --- * Windows Executable (EXE)(52020) --- --- URL logs have a threat id of 9999, and there is no description. --- Everything else has a human-readable description. Sometimes, --- this description is suffixed by a space and a parenthesized --- acronym (EXE, DLL, etc.). -parserThreatId :: Parser Field s (Bounds,Word64) -parserThreatId = Latin.any threatIdField >>= \case - '(' -> do - theId <- Latin.decWord64 threatIdField - Latin.char threatIdField ')' - Latin.char threatIdField ',' - pure (Bounds 0 0, theId) - _ -> do - startSucc <- Unsafe.cursor - Latin.skipTrailedBy threatIdField '(' - end <- Latin.trySatisfy (\c -> isAsciiUpper c || isAsciiLower c) >>= \case - True -> do - endSuccSucc <- Unsafe.cursor - Latin.skipTrailedBy threatIdField '(' - arr <- Unsafe.expose - -- We go back an extra character to remove the trailing - -- space. I do not believe this can lead to negative-length - -- slices, but the line of reasoning is muddy. - case indexCharArray arr (endSuccSucc - 3) of - ' ' -> pure (endSuccSucc - 3) - _ -> P.fail threatIdField - False -> do - endSucc <- Unsafe.cursor - pure (endSucc - 1) - theId <- Latin.decWord64 threatIdField - Latin.char threatIdField ')' - Latin.char threatIdField ',' - let start = startSucc - 1 - pure (Bounds start (end - start), theId) - -parserOptionallyQuoted_ :: e -> Parser e s () -parserOptionallyQuoted_ e = Latin.any e >>= \case - '"' -> do - _ <- consumeQuoted e 0 - pure () - ',' -> pure () - _ -> Latin.skipTrailedBy e ',' - --- Precondition: the cursor is placed at the beginning of the --- possibly-quoted content. That is, the comma preceeding has --- already been consumed. This is very similar to parserOptionallyQuoted, --- but it differs slightly because there is no trailing comma. -finalOptionallyQuoted :: e -> Parser e s Bytes -finalOptionallyQuoted e = Latin.opt >>= \case - Nothing -> do - !array <- Unsafe.expose - pure $! Bytes{array,offset=0,length=0} - Just c -> case c of - '"' -> do - -- First, we do a run through just to see if anything - -- actually needs to be escaped. - start <- Unsafe.cursor - !n <- consumeFinalQuoted e 0 - !array <- Unsafe.expose - !endSucc <- Unsafe.cursor - let end = endSucc - 1 - if n == 0 - then pure Bytes{array,offset=start,length=(end - start)} - else do - let !r = escapeQuotes Bytes{array,offset=start,length=(end - start)} - pure $! Bytes{array=r,offset=0,length=PM.sizeofByteArray r} - _ -> do - !startSucc <- Unsafe.cursor - Latin.skipUntil ',' - !end <- Unsafe.cursor - !arr <- Unsafe.expose - let start = startSucc - 1 - pure $! Bytes arr start (end - start) - --- Precondition: the cursor is placed at the beginning of the --- possibly-quoted content. That is, the comma preceeding has --- already been consumed. -parserOptionallyQuoted :: e -> Parser e s Bytes -parserOptionallyQuoted e = Latin.any e >>= \case - '"' -> do - -- First, we do a run through just to see if anything - -- actually needs to be escaped. - start <- Unsafe.cursor - !n <- consumeQuoted e 0 - !array <- Unsafe.expose - !endSuccSucc <- Unsafe.cursor - let end = endSuccSucc - 2 - if n == 0 - then pure Bytes{array,offset=start,length=(end - start)} - else do - let !r = escapeQuotes Bytes{array,offset=start,length=(end - start)} - pure $! Bytes{array=r,offset=0,length=PM.sizeofByteArray r} - ',' -> do - !array <- Unsafe.expose - pure $! Bytes{array,offset=0,length=0} - _ -> do - !startSucc <- Unsafe.cursor - Latin.skipTrailedBy e ',' - !endSucc <- Unsafe.cursor - !arr <- Unsafe.expose - let start = startSucc - 1 - let end = endSucc - 1 - pure $! (Bytes arr start (end - start)) - --- Precondition: the input is a valid CSV-style quoted-escaped --- string. That is, any double quote character is guaranteed to --- be followed by another one. -escapeQuotes :: Bytes -> ByteArray -escapeQuotes (Bytes arr off0 len0) = runByteArrayST $ do - marr <- PM.newByteArray len0 - let go !soff !doff !len = if len > 0 - then do - let w :: Word8 = PM.indexByteArray arr soff - PM.writeByteArray marr doff w - if w /= c2w '"' - then go (soff + 1) (doff + 1) (len - 1) - else go (soff + 2) (doff + 1) (len - 2) - else pure doff - finalSz <- go off0 0 len0 - marr' <- PM.resizeMutableByteArray marr finalSz - PM.unsafeFreezeByteArray marr' - --- When this parser completed, the position in the input will be --- just after the comma that followed the quoted field. --- This is defined recursively. -consumeQuoted :: - e - -> Int -- the number of escaped quotes we have encountered - -> Parser e s Int -consumeQuoted e !n = do - Latin.skipTrailedBy e '"' - Latin.any e >>= \case - ',' -> pure n - '"' -> consumeQuoted e (n + 1) - _ -> P.fail e - --- Like consumeQuoted except that we are expected end-of-input --- instead of a comma at the end. -consumeFinalQuoted :: - e - -> Int -- the number of escaped quotes we have encountered - -> Parser e s Int -consumeFinalQuoted e !n = do - Latin.skipTrailedBy e '"' - Latin.opt >>= \case - Nothing -> pure n - Just c -> case c of - '"' -> consumeFinalQuoted e (n + 1) - _ -> P.fail e - -parserDatetime :: e -> e -> Parser e s Datetime -{-# noinline parserDatetime #-} -parserDatetime edate etime = do - year <- Latin.decWord edate - Latin.char edate '/' - monthPlusOne <- Latin.decWord edate - let month = monthPlusOne - 1 - if month > 11 - then P.fail edate - else pure () - Latin.char edate '/' - day <- Latin.decWord edate - Latin.char etime ' ' - hour <- Latin.decWord etime - Latin.char etime ':' - minute <- Latin.decWord etime - Latin.char etime ':' - second <- Latin.decWord etime - Latin.char etime ',' - pure $ Datetime - (Date - (Year (fromIntegral year)) - (Month (fromIntegral month)) - (DayOfMonth (fromIntegral day)) - ) - (TimeOfDay - (fromIntegral hour) - (fromIntegral minute) - (1_000_000_000 * fromIntegral second) - ) - -cstringLen# :: Addr# -> Int# -{-# noinline cstringLen# #-} -cstringLen# ptr = go 0 where - go !ix@(I# ix#) = if PM.indexOffPtr (Ptr ptr) ix == (0 :: Word8) - then ix# - else go (ix + 1) - -c2w :: Char -> Word8 -c2w = fromIntegral . ord - -indexCharArray :: ByteArray -> Int -> Char -indexCharArray (PM.ByteArray x) (I# i) = - Exts.C# (Exts.indexCharArray# x i) + , User(..) + , Correlation(..) + , Field(..) + , Bounds(..) + -- * Decoding + , decode + ) where + +import Panos.Syslog.Internal.Common + +import Chronos (Datetime,Offset(Offset),OffsetDatetime(OffsetDatetime)) +import Data.Bytes.Parser (Parser) +import Data.Bytes.Types (Bytes(..)) +import GHC.Exts (Ptr(Ptr)) +import Panos.Syslog.Internal.Traffic (Traffic(..),parserTraffic) +import Panos.Syslog.Internal.Threat (Threat(..),parserThreat) +import Panos.Syslog.Internal.System (System(..),parserSystem) +import Panos.Syslog.Internal.User (User(..),parserUser) +import Panos.Syslog.Internal.Correlation (Correlation(..),parserCorrelation) + +import qualified Chronos +import qualified Data.Bytes.Parser as P +import qualified Data.Bytes.Parser.Ascii as Ascii +import qualified Data.Bytes.Parser.Latin as Latin +import qualified Data.Bytes.Parser.Unsafe as Unsafe + +-- | Sum that represents all known PAN-OS syslog types. Use 'decode' +-- to parse a byte sequence into a structured log. +data Log + = LogTraffic !Traffic + | LogThreat !Threat + | LogSystem !System + | LogUser !User + | LogCorrelation !Correlation + | LogOther + +untilSpace :: e -> Parser e s Bounds +{-# inline untilSpace #-} +untilSpace e = do + start <- Unsafe.cursor + Latin.skipTrailedBy e ' ' + endSucc <- Unsafe.cursor + let end = endSucc - 1 + pure (Bounds start (end - start)) + +-- Returns the receive time and the serial number. There is a +-- little subtlety here. The PANOS guide says that logs should +-- start with something like: +-- 1,2019/07/14 10:26:22,005923187997 +-- The leading field is reserved for future use. However, there +-- is typically an additional prefix consisting of a syslog priority, +-- another datetime (in a different format), and a hostname: +-- <14> Jul 14 11:26:23 MY-HOST.example.com 1,... +-- The datetime is within typically within a second of the other one. +-- Additionally, it's missing the year. So, we discard it. The +-- syslog priority is worthless, so we throw it out as well. The +-- host name, however, does provide useful information that does +-- not exist elsewhere in the log. We should be as flexible +-- as possible with this somewhat fragile part of the log. +-- +-- Prisma logs add another wrinkle. These begin with: +-- <14>1 2021-10-27T19:21:00.034Z stream-logfwd20-example logforwarder - panwlogs - 2021-10-27T19:20:59.000000Z,no-serial,... +-- Notice that the timestamps are now ISO-8601 encoded. The hostname is +-- in Prisma logs is worthless, but the device_name from the PAN log +-- inside gives the end user the information they will want. +parserPrefix :: Parser Field s (Bounds,Datetime,Bounds) +{-# inline parserPrefix #-} +parserPrefix = do + Latin.skipChar ' ' + -- We allow the syslog priority (the number in angle brackets) + -- to be absent. + Latin.trySatisfy (== '<') >>= \case + True -> do + Latin.skipTrailedBy syslogPriorityField '>' + Latin.skipChar ' ' + False -> pure () + Latin.trySatisfy (== '1') >>= \case + True -> do + Latin.any futureUseDField >>= \case + ',' -> do + !recv <- parserDatetime receiveTimeDateField receiveTimeTimeField + !ser <- untilComma serialNumberField + pure (Bounds 0 0,recv,ser) + ' ' -> do -- Prisma logs + -- TODO: In chronos, add a datetime parser that discards the + -- datetime instead of constructing it. + _ <- P.orElse Chronos.parserUtf8BytesIso8601 (P.fail syslogDatetimeField) + Latin.char syslogDatetimeField ' ' + hostBounds <- untilSpace syslogHostField + P.cstring prismaDataField (Ptr "logforwarder - panwlogs - "# ) + OffsetDatetime recv (Offset off) <- P.orElse Chronos.parserUtf8BytesIso8601 (P.fail receiveTimeDateField) + Latin.char syslogDatetimeField ',' + case off of + 0 -> pure () + _ -> P.fail syslogDatetimeField + !ser <- untilComma serialNumberField + pure (hostBounds,recv,ser) + _ -> P.fail futureUseDField + False -> do + Ascii.skipAlpha1 syslogDatetimeField -- Month + Latin.skipChar1 syslogDatetimeField ' ' + Latin.skipDigits1 syslogDatetimeField -- Day + Latin.skipChar1 syslogDatetimeField ' ' + Latin.skipDigits1 syslogDatetimeField -- Hour + Latin.char syslogDatetimeField ':' + Latin.skipDigits1 syslogDatetimeField -- Minute + Latin.char syslogDatetimeField ':' + Latin.skipDigits1 syslogDatetimeField -- Second + Latin.skipChar1 syslogDatetimeField ' ' + hostBounds <- untilSpace syslogHostField + Latin.skipChar ' ' + skipThroughComma futureUseDField + !recv <- parserDatetime receiveTimeDateField receiveTimeTimeField + !ser <- untilComma serialNumberField + pure (hostBounds,recv,ser) + +-- | Decode a PAN-OS syslog message of an unknown type. If there are +-- leftovers, we still succeed. We do this because every release of PAN-OS +-- adds a few more fields to the end, and it\'s good to have this library +-- be able to parse these logs even if it means ignoring the new fields. +decode :: Bytes -> Either Field Log +decode b = case P.parseBytes parserLog b of + P.Failure e -> Left e + P.Success (P.Slice _ _ r) -> Right r + +parserLog :: Parser Field s Log +parserLog = do + (!hostBounds,!receiveTime,!serialNumber) <- parserPrefix + Latin.any typeField >>= \case + 'C' -> do + Latin.char11 typeField 'O' 'R' 'R' 'E' 'L' 'A' 'T' 'I' 'O' 'N' ',' + !x <- parserCorrelation hostBounds receiveTime serialNumber + pure (LogCorrelation x) + 'U' -> do + Latin.char6 typeField 'S' 'E' 'R' 'I' 'D' ',' + !x <- parserUser hostBounds receiveTime serialNumber + pure (LogUser x) + 'S' -> do + Latin.char6 typeField 'Y' 'S' 'T' 'E' 'M' ',' + !x <- parserSystem hostBounds receiveTime serialNumber + pure (LogSystem x) + 'T' -> Latin.any typeField >>= \case + 'R' -> do + Latin.char6 typeField 'A' 'F' 'F' 'I' 'C' ',' + !x <- parserTraffic hostBounds receiveTime serialNumber + pure (LogTraffic x) + 'H' -> do + Latin.char5 typeField 'R' 'E' 'A' 'T' ',' + !x <- parserThreat hostBounds receiveTime serialNumber + pure (LogThreat x) + _ -> P.fail typeField + _ -> P.fail typeField + + diff --git a/src/Panos/Syslog/User.hs b/src/Panos/Syslog/User.hs new file mode 100644 --- /dev/null +++ b/src/Panos/Syslog/User.hs @@ -0,0 +1,129 @@ +{-# language BangPatterns #-} +{-# language MagicHash #-} +{-# language NamedFieldPuns #-} +{-# language DuplicateRecordFields #-} +{-# language NumericUnderscores #-} +{-# language DerivingStrategies #-} +{-# language GeneralizedNewtypeDeriving #-} +{-# language OverloadedRecordDot #-} + +-- | Fields for user logs. +module Panos.Syslog.User + ( dataSource + , dataSourceName + , deviceGroupHierarchyLevel1 + , deviceGroupHierarchyLevel2 + , deviceGroupHierarchyLevel3 + , deviceGroupHierarchyLevel4 + , deviceName + , sourceIp + , repeatCount + , sequenceNumber + , serialNumber + , subtype + , syslogHost + , timeGenerated + , user + , virtualSystem + , virtualSystemName + ) where + +import Chronos (Datetime) +import Data.Bytes.Types (Bytes(..)) +import Data.Word (Word64) +import Net.Types (IP) +import Panos.Syslog.Unsafe (User(User),Bounds(Bounds)) + +import qualified Panos.Syslog.Unsafe as U + +-- | Subtype of the system log; refers to the system daemon +-- generating the log; values are @crypto@, @dhcp@, @dnsproxy@, +-- @dos@, @general@, @global-protect@, @ha@, @hw@, @nat@, @ntpd@, +-- @pbf@, @port@, @pppoe@, @ras@, @routing@, @satd@, @sslmgr@, +-- @sslvpn@, @userid@, @url-filtering@, @vpn@. +subtype :: User -> Bytes +subtype (User{subtype=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | The hostname of the firewall on which the session was logged. +deviceName :: User -> Bytes +deviceName (User{deviceName=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | Identifies the end user. +user :: User -> Bytes +user (User{user=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | Source from which mapping information is collected. +dataSource :: User -> Bytes +dataSource (User{dataSource=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | User-ID source that sends the IP (Port)-User Mapping. +dataSourceName :: User -> Bytes +dataSourceName (User{dataSourceName=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | Time the log was generated on the dataplane. +timeGenerated :: User -> Datetime +{-# inline timeGenerated #-} +timeGenerated u = u.timeGenerated + +-- | A 64-bit log entry identifier incremented sequentially; +-- each log type has a unique number space. +sequenceNumber :: User -> Word64 +{-# inline sequenceNumber #-} +sequenceNumber u = u.sequenceNumber + +-- | Serial number of the firewall that generated the log. These +-- occassionally contain non-numeric characters, so do not attempt +-- to parse this as a decimal number. +serialNumber :: User -> Bytes +serialNumber (User{serialNumber=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | Virtual System associated with the session. +virtualSystem :: User -> Bytes +virtualSystem (User{virtualSystem=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | The name of the virtual system associated with the session; only valid +-- on firewalls enabled for multiple virtual systems. +virtualSystemName :: User -> Bytes +virtualSystemName (User{virtualSystemName=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | The hostname from the syslog header appended to the PAN-OS log. +-- This field is not documented by Palo Alto Network and technically +-- is not part of the log, but in practice, it is always present. +-- This is similar to @deviceName@. +syslogHost :: User -> Bytes +syslogHost (User{syslogHost=Bounds off len,message=msg}) = + Bytes{offset=off,length=len,array=msg} + +-- | Number of total bytes (transmit and receive) for the session. +repeatCount :: User -> Word64 +{-# inline repeatCount #-} +repeatCount u = u.repeatCount + +deviceGroupHierarchyLevel1 :: User -> Word64 +{-# inline deviceGroupHierarchyLevel1 #-} +deviceGroupHierarchyLevel1 u = u.deviceGroupHierarchyLevel1 + +deviceGroupHierarchyLevel2 :: User -> Word64 +{-# inline deviceGroupHierarchyLevel2 #-} +deviceGroupHierarchyLevel2 u = u.deviceGroupHierarchyLevel2 + +deviceGroupHierarchyLevel3 :: User -> Word64 +{-# inline deviceGroupHierarchyLevel3 #-} +deviceGroupHierarchyLevel3 u = u.deviceGroupHierarchyLevel3 + +deviceGroupHierarchyLevel4 :: User -> Word64 +{-# inline deviceGroupHierarchyLevel4 #-} +deviceGroupHierarchyLevel4 u = u.deviceGroupHierarchyLevel4 + +-- | Original session source IP address. +sourceIp :: User -> IP +sourceIp = U.sourceIp + diff --git a/test/Main.hs b/test/Main.hs --- a/test/Main.hs +++ b/test/Main.hs @@ -11,11 +11,13 @@ import Data.Char (ord,chr) import Data.Bytes.Types (Bytes(Bytes)) -import qualified Panos.Syslog.Traffic as Traffic -import qualified Panos.Syslog.Threat as Threat -import qualified Panos.Syslog.System as System import qualified Data.Primitive as PM import qualified GHC.Exts as Exts +import qualified Panos.Syslog.Correlation as Correlation +import qualified Panos.Syslog.System as System +import qualified Panos.Syslog.Threat as Threat +import qualified Panos.Syslog.Traffic as Traffic +import qualified Panos.Syslog.User as User import qualified Sample as S main :: IO () @@ -25,6 +27,10 @@ testA putStrLn "8.1-Traffic-B" testTrafficB + putStrLn "9.0-Traffic-A" + testTraffic_9_0_A + putStrLn "Prisma-Traffic-A" + testPrismaTrafficA putStrLn "8.1-Threat-A" testB putStrLn "8.1-Threat-B" @@ -43,8 +49,24 @@ testThreatH putStrLn "8.1-Threat-I" testThreatI + putStrLn "9.0-Threat-A" + testThreat_9_0_A + putStrLn "9.1-Threat-A" + testThreat_9_1_A + putStrLn "9.1-Threat-B" + testThreat_9_1_B + putStrLn "Prisma-Threat-A" + testPrismaThreatA + putStrLn "Prisma-Threat-B" + testPrismaThreatB + putStrLn "Prisma-Threat-C" + testPrismaThreatC putStrLn "8.1-System-A" testSystemA + putStrLn "User-A" + testUserA + putStrLn "Correlation-A" + testCorrelationA putStrLn "Finished" testA :: IO () @@ -81,6 +103,38 @@ | otherwise -> pure () Right _ -> fail "wrong log type" +testTraffic_9_0_A :: IO () +testTraffic_9_0_A = case decode S.traffic_9_0_A of + Left err -> throwIO err + Right (LogTraffic t) -> + if | Traffic.deviceName t /= bytes "NY-PAN-FW-5" -> + fail $ + "wrong device name:\nexpected: " ++ + show (bytes "NY-PAN-FW-5") ++ + "\nactually: " ++ + show (Traffic.deviceName t) + | Traffic.actionSource t /= bytes "my-policy" -> + fail $ + "wrong action source:\nexpected: " ++ + show (bytes "my-policy") ++ + "\nactually: " ++ + prettyBytes (Traffic.actionSource t) + | otherwise -> pure () + Right _ -> fail "wrong log type" + +testPrismaTrafficA :: IO () +testPrismaTrafficA = case decode S.traffic_prisma_A of + Left err -> throwIO err + Right (LogTraffic t) -> + if | Traffic.deviceName t /= bytes "The-Device-Name" -> + fail $ + "wrong device name:\nexpected: " ++ + show (bytes "NY-PAN-FW-5") ++ + "\nactually: " ++ + show (Traffic.deviceName t) + | otherwise -> pure () + Right _ -> fail "wrong log type" + testTrafficB :: IO () testTrafficB = case decode S.traffic_8_1_B of Left err -> throwIO err @@ -110,6 +164,79 @@ | otherwise -> pure () Right _ -> fail "wrong log type" +testThreat_9_0_A :: IO () +testThreat_9_0_A = case decode S.threat_9_0_A of + Left err -> throwIO err + Right (LogThreat t) -> + if | Threat.miscellaneous t /= bytes "dt.adsafeprotected.com/" -> fail $ + "wrong miscellaneous (URL):\nExpected: dt.adsafeprotected.com/\nActually: " ++ + prettyBytes (Threat.miscellaneous t) ++ "\n" + | otherwise -> pure () + Right _ -> fail "wrong log type" + +testThreat_9_1_A :: IO () +testThreat_9_1_A = case decode S.threat_9_1_A of + Left err -> throwIO err + Right (LogThreat t) -> + if | Threat.miscellaneous t /= bytes "192.0.2.17_solarwinds_zero_configuration:5986/" -> fail $ + "wrong miscellaneous (URL):\nExpected: " ++ + "192.0.2.17_solarwinds_zero_configuration:5986/\nActually: " ++ + prettyBytes (Threat.miscellaneous t) ++ "\n" + | otherwise -> pure () + Right _ -> fail "wrong log type" + +testThreat_9_1_B :: IO () +testThreat_9_1_B = case decode S.threat_9_1_B of + Left err -> throwIO err + Right (LogThreat t) -> + if | Threat.urlCategoryList t /= Just (bytes "social-networking,low-risk") -> + fail $ + "wrong url category list\nExpected:\n" + ++ + "social-networking,low-risk" + ++ + "\nActually:\n" + ++ + maybe "(empty)" prettyBytes (Threat.urlCategoryList t) + ++ + "\n" + | otherwise -> pure () + Right _ -> fail "wrong log type" + +testPrismaThreatA :: IO () +testPrismaThreatA = case decode S.threat_prisma_A of + Left err -> throwIO err + Right (LogThreat t) -> + if | Threat.miscellaneous t /= bytes "play.google.com/" -> fail $ + "wrong miscellaneous (URL):\nExpected: " ++ + "play.google.com/\nActually: " ++ + prettyBytes (Threat.miscellaneous t) ++ "\n" + | otherwise -> pure () + Right _ -> fail "wrong log type" + +testPrismaThreatB :: IO () +testPrismaThreatB = case decode S.threat_prisma_B of + Left err -> throwIO err + Right (LogThreat t) -> + if | Threat.miscellaneous t /= bytes "example.com" -> fail $ + "wrong miscellaneous (domain name):\nExpected: " ++ + "example.com\nActually: " ++ + prettyBytes (Threat.miscellaneous t) ++ "\n" + | otherwise -> pure () + Right _ -> fail "wrong log type" + +testPrismaThreatC :: IO () +testPrismaThreatC = case decode S.threat_prisma_C of + Left err -> throwIO err + Right (LogThreat t) -> + if | Threat.subtype t /= bytes "wildfire" -> fail "wrong subtype" + | Threat.miscellaneous t /= bytes "Exchange.asmx" -> fail $ + "wrong miscellaneous:\nExpected: " ++ + "Exchange.asmx\nActually: " ++ + prettyBytes (Threat.miscellaneous t) ++ "\n" + | otherwise -> pure () + Right _ -> fail "wrong log type" + testD :: IO () testD = case decode S.threat_8_1_C of Left err -> throwIO err @@ -201,6 +328,24 @@ | otherwise -> pure () Right _ -> fail "wrong log type" +testUserA :: IO () +testUserA = case decode S.user_A of + Left err -> throwIO err + Right (LogUser t) -> + if | User.subtype t /= bytes "login" -> fail $ + "wrong subtype name:\nexpected: login\nactually: " ++ + show (User.subtype t) + | User.user t /= bytes "BIGDAWG10$@EXAMPLE.COM" -> fail "wrong user" + | otherwise -> pure () + Right _ -> fail "wrong log type" + +testCorrelationA :: IO () +testCorrelationA = case decode S.correlation_A of + Left err -> throwIO err + Right (LogCorrelation t) -> + if | Correlation.objectId t /= 6005 -> fail "wrong object id" + | otherwise -> pure () + Right _ -> fail "wrong log type" bytes :: String -> Bytes bytes s = let b = pack s in Bytes b 0 (PM.sizeofByteArray b)