scan-metadata (empty) → 0.2.1.0

raw patch · 5 files changed

+199/−0 lines, 5 filesdep +aesondep +antiope-coredep +antiope-s3setup-changed

Dependencies added: aeson, antiope-core, antiope-s3, base, bytestring, containers, generic-lens, hw-aeson, hw-ip, lens, mtl, text, thyme, unliftio-core

Files

• LICENSE +20/−0
• README.md +1/−0
• Setup.hs +2/−0
• scan-metadata.cabal +41/−0
• src/Arbor/Albedo/Scan/Metadata.hs +135/−0

@@ -0,0 +1,20 @@+Copyright (c) 2019 alexeyraga++Permission is hereby granted, free of charge, to any person obtaining+a copy of this software and associated documentation files (the+"Software"), to deal in the Software without restriction, including+without limitation the rights to use, copy, modify, merge, publish,+distribute, sublicense, and/or sell copies of the Software, and to+permit persons to whom the Software is furnished to do so, subject to+the following conditions:++The above copyright notice and this permission notice shall be included+in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

@@ -0,0 +1,1 @@+# scan-metadata

@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain

@@ -0,0 +1,41 @@+cabal-version:       2.2++name:                scan-metadata+version:             0.2.1.0+synopsis:            Metadata types for Albedo Scanners+description:         Metadata types for Albedo Scanners.+-- bug-reports:+license:             MIT+license-file:        LICENSE+author:              alexeyraga+maintainer:          alexey.raga@gmail.com+-- copyright:+category:            Data+extra-source-files:  README.md++source-repository head+  type: git+  location: https://github.com/packetloop/scan-metadata++library+  exposed-modules:+    Arbor.Albedo.Scan.Metadata+  -- other-modules:+  -- other-extensions:+  build-depends:+      base >=4.7 && <=5+    , aeson+    , antiope-core >=7.0.1+    , antiope-s3 >=7.0.1+    , bytestring+    , containers+    , generic-lens >= 1.1.0.0+    , hw-aeson >=0.1.0.1+    , hw-ip >=2.3.1.0+    , lens >= 4.17.1+    , mtl+    , text+    , thyme >=0.3.5.5+    , unliftio-core+  hs-source-dirs:      src+  default-language:    Haskell2010

@@ -0,0 +1,135 @@+{-# LANGUAGE DataKinds             #-}+{-# LANGUAGE DeriveGeneric         #-}+{-# LANGUAGE DuplicateRecordFields #-}+{-# LANGUAGE OverloadedStrings     #-}+{-# LANGUAGE TypeApplications      #-}+{-# LANGUAGE GeneralizedNewtypeDeriving #-}+{-# LANGUAGE FlexibleContexts #-}+module Arbor.Albedo.Scan.Metadata+where++import Antiope.Core                          (toText, runAwsTyped)+import Antiope.Env                           (Env)+import Antiope.S3                            (S3Uri)+import Antiope.S3.Strict                     (downloadFromS3Uri)+import Control.Applicative                   ((<|>))+import Control.Lens                          (from, (&), (<&>), (^.))+import Control.Monad.Reader                  (MonadReader)+import Control.Monad.IO.Unlift               (MonadUnliftIO)+import Data.Aeson+import Data.Aeson.Types+import Data.ByteString.Lazy                  (fromStrict)+import Data.Generics.Product.Any             (the)+import Data.Generics.Product.Typed           (HasType)+import Data.Int                              (Int64)+import Data.Maybe+import Data.Thyme.Time.Core                  (UTCTime, microseconds, posixSecondsToUTCTime, toSeconds, utcTimeToPOSIXSeconds)+import GHC.Generics                          hiding (from)+import HaskellWorks.Data.Aeson+import HaskellWorks.Data.Network.Ip.Ipv4     (IpBlock)+import HaskellWorks.Data.Network.Ip.Validity (Canonical)++import qualified Data.Text as Text+++newtype Proto = Proto { unProto :: Int } deriving (Generic, Show, Eq, FromJSON, ToJSON)++data Command = Command+  { name        :: Maybe String+  , scanner     :: Maybe String+  , probe       :: Maybe String+  , probeBanner :: Maybe String -- use if zmap and zgrab use different probes(ie udp and cldap), banner-commander would prioritize probeBanner+  , probeArgs   :: Maybe String+  , port        :: Int+  , proto       :: Proto+  , destination :: Either FilePath S3Uri+  , cidr        :: IpBlock Canonical+  , deadline    :: UTCTime+  , banners     :: Bool+  } deriving (Generic, Show, Eq)++data ScanMetaData = ScanMetaData+  { startAt    :: UTCTime+  , stopAt     :: UTCTime+  , provenance :: Maybe String+  , scanCmd    :: Command+  , digest     :: String+  , port       :: Int+  , proto      :: Proto+  , cidr       :: IpBlock Canonical+  , source     :: Maybe String+  , data'      :: String+  } deriving (Generic, Show, Eq)++instance FromJSON Command where+  parseJSON = withObject "object" $ \o ->+    Command <$> o .:? "name"+            <*> o .: "scanner"+            <*> o .: "probe"+            <*> o .:? "probeBanner"+            <*> o .:? "probeArgs"+            <*> o .: "port"+            <*> (o .: "proto")+            <*> (o .:  "destination" >>= readDestination)+            <*> (o .:  "cidr"        >>= readJson "cidr")+            <*> (o .:  "deadline"    & fmap int64ToUTCTime)+            <*> (o .:? "banners" .!= False)++instance ToJSON Command where+  toJSON r = objectWithoutNulls+    [ "name"        .= (r ^. the @"name")+    , "scanner"     .= (r ^. the @"scanner")+    , "probe"       .= (r ^. the @"probe")+    , "probeBanner" .= (r ^. the @"probeBanner")+    , "probeArgs"   .= (r ^. the @"probeArgs")+    , "port"        .= (r ^. the @"port")+    , "proto"       .= (r ^. the @"proto")+    , "destination" .= either Text.pack toText (r ^. the @"destination")+    , "cidr"        .= show (r ^. the @"cidr")+    , "deadline"    .= utcTimeToInt64 (r ^. the @"deadline")+    , "banners"     .= (r ^. the @"banners")+    ]++instance FromJSON ScanMetaData where+  parseJSON = withObject "object" $ \o ->+    ScanMetaData <$> (o .: "startAt" & fmap int64ToUTCTime)+                 <*> (o .: "stopAt"  & fmap int64ToUTCTime)+                 <*> (o .:? "provenance")+                 <*> o .: "scanCmd"+                 <*> o .: "digest"+                 <*> o .: "port"+                 <*> (o .: "proto")+                 <*> (o .: "cidr"   >>= readJson "cidr")+                 <*> (o .:? "source")+                 <*> (o .: "data")++instance ToJSON ScanMetaData where+  toJSON r = objectWithoutNulls+    [ "startAt"    .= utcTimeToInt64 (r ^. the @"startAt")+    , "stopAt"     .= utcTimeToInt64 (r ^. the @"stopAt")+    , "provenance" .= (r ^. the @"provenance")+    , "scanCmd"    .= (r ^. the @"scanCmd")+    , "digest"     .= (r ^. the @"digest")+    , "port"       .= (r ^. the @"port")+    , "proto"      .= (r ^. the @"proto")+    , "cidr"       .= show (r ^. the @"cidr")+    , "source"     .= (r ^. the @"source")+    , "data"       .= (r ^. the @"data'")+    ]++readDestination :: String -> Parser (Either FilePath S3Uri)+readDestination s = (Right <$> readJson "destination (s3uri)"    s)+                <|> (Left  <$> readJson "destination (filepath)" s)++int64ToUTCTime :: Int64 -> UTCTime+int64ToUTCTime a = posixSecondsToUTCTime $ (a * 1000 * 1000) ^. from microseconds++utcTimeToInt64 :: UTCTime -> Int64+utcTimeToInt64 ut = round (toSeconds (utcTimeToPOSIXSeconds ut) :: Double)++loadScanMetadata :: (MonadUnliftIO m, MonadReader r m, HasType Env r)+    => S3Uri+    -> m (Maybe ScanMetaData)+loadScanMetadata uri = runAwsTyped $ do+    mb <- downloadFromS3Uri uri+    pure $ mb >>= (decode . fromStrict)