diff --git a/LICENSE b/LICENSE new file mode 100644 --- /dev/null +++ b/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2019 alexeyraga + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,1 @@ +# scan-metadata diff --git a/Setup.hs b/Setup.hs new file mode 100644 --- /dev/null +++ b/Setup.hs @@ -0,0 +1,2 @@ +import Distribution.Simple +main = defaultMain diff --git a/scan-metadata.cabal b/scan-metadata.cabal new file mode 100644 --- /dev/null +++ b/scan-metadata.cabal @@ -0,0 +1,41 @@ +cabal-version: 2.2 + +name: scan-metadata +version: 0.2.1.0 +synopsis: Metadata types for Albedo Scanners +description: Metadata types for Albedo Scanners. +-- bug-reports: +license: MIT +license-file: LICENSE +author: alexeyraga +maintainer: alexey.raga@gmail.com +-- copyright: +category: Data +extra-source-files: README.md + +source-repository head + type: git + location: https://github.com/packetloop/scan-metadata + +library + exposed-modules: + Arbor.Albedo.Scan.Metadata + -- other-modules: + -- other-extensions: + build-depends: + base >=4.7 && <=5 + , aeson + , antiope-core >=7.0.1 + , antiope-s3 >=7.0.1 + , bytestring + , containers + , generic-lens >= 1.1.0.0 + , hw-aeson >=0.1.0.1 + , hw-ip >=2.3.1.0 + , lens >= 4.17.1 + , mtl + , text + , thyme >=0.3.5.5 + , unliftio-core + hs-source-dirs: src + default-language: Haskell2010 diff --git a/src/Arbor/Albedo/Scan/Metadata.hs b/src/Arbor/Albedo/Scan/Metadata.hs new file mode 100644 --- /dev/null +++ b/src/Arbor/Albedo/Scan/Metadata.hs @@ -0,0 +1,135 @@ +{-# LANGUAGE DataKinds #-} +{-# LANGUAGE DeriveGeneric #-} +{-# LANGUAGE DuplicateRecordFields #-} +{-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE TypeApplications #-} +{-# LANGUAGE GeneralizedNewtypeDeriving #-} +{-# LANGUAGE FlexibleContexts #-} +module Arbor.Albedo.Scan.Metadata +where + +import Antiope.Core (toText, runAwsTyped) +import Antiope.Env (Env) +import Antiope.S3 (S3Uri) +import Antiope.S3.Strict (downloadFromS3Uri) +import Control.Applicative ((<|>)) +import Control.Lens (from, (&), (<&>), (^.)) +import Control.Monad.Reader (MonadReader) +import Control.Monad.IO.Unlift (MonadUnliftIO) +import Data.Aeson +import Data.Aeson.Types +import Data.ByteString.Lazy (fromStrict) +import Data.Generics.Product.Any (the) +import Data.Generics.Product.Typed (HasType) +import Data.Int (Int64) +import Data.Maybe +import Data.Thyme.Time.Core (UTCTime, microseconds, posixSecondsToUTCTime, toSeconds, utcTimeToPOSIXSeconds) +import GHC.Generics hiding (from) +import HaskellWorks.Data.Aeson +import HaskellWorks.Data.Network.Ip.Ipv4 (IpBlock) +import HaskellWorks.Data.Network.Ip.Validity (Canonical) + +import qualified Data.Text as Text + + +newtype Proto = Proto { unProto :: Int } deriving (Generic, Show, Eq, FromJSON, ToJSON) + +data Command = Command + { name :: Maybe String + , scanner :: Maybe String + , probe :: Maybe String + , probeBanner :: Maybe String -- use if zmap and zgrab use different probes(ie udp and cldap), banner-commander would prioritize probeBanner + , probeArgs :: Maybe String + , port :: Int + , proto :: Proto + , destination :: Either FilePath S3Uri + , cidr :: IpBlock Canonical + , deadline :: UTCTime + , banners :: Bool + } deriving (Generic, Show, Eq) + +data ScanMetaData = ScanMetaData + { startAt :: UTCTime + , stopAt :: UTCTime + , provenance :: Maybe String + , scanCmd :: Command + , digest :: String + , port :: Int + , proto :: Proto + , cidr :: IpBlock Canonical + , source :: Maybe String + , data' :: String + } deriving (Generic, Show, Eq) + +instance FromJSON Command where + parseJSON = withObject "object" $ \o -> + Command <$> o .:? "name" + <*> o .: "scanner" + <*> o .: "probe" + <*> o .:? "probeBanner" + <*> o .:? "probeArgs" + <*> o .: "port" + <*> (o .: "proto") + <*> (o .: "destination" >>= readDestination) + <*> (o .: "cidr" >>= readJson "cidr") + <*> (o .: "deadline" & fmap int64ToUTCTime) + <*> (o .:? "banners" .!= False) + +instance ToJSON Command where + toJSON r = objectWithoutNulls + [ "name" .= (r ^. the @"name") + , "scanner" .= (r ^. the @"scanner") + , "probe" .= (r ^. the @"probe") + , "probeBanner" .= (r ^. the @"probeBanner") + , "probeArgs" .= (r ^. the @"probeArgs") + , "port" .= (r ^. the @"port") + , "proto" .= (r ^. the @"proto") + , "destination" .= either Text.pack toText (r ^. the @"destination") + , "cidr" .= show (r ^. the @"cidr") + , "deadline" .= utcTimeToInt64 (r ^. the @"deadline") + , "banners" .= (r ^. the @"banners") + ] + +instance FromJSON ScanMetaData where + parseJSON = withObject "object" $ \o -> + ScanMetaData <$> (o .: "startAt" & fmap int64ToUTCTime) + <*> (o .: "stopAt" & fmap int64ToUTCTime) + <*> (o .:? "provenance") + <*> o .: "scanCmd" + <*> o .: "digest" + <*> o .: "port" + <*> (o .: "proto") + <*> (o .: "cidr" >>= readJson "cidr") + <*> (o .:? "source") + <*> (o .: "data") + +instance ToJSON ScanMetaData where + toJSON r = objectWithoutNulls + [ "startAt" .= utcTimeToInt64 (r ^. the @"startAt") + , "stopAt" .= utcTimeToInt64 (r ^. the @"stopAt") + , "provenance" .= (r ^. the @"provenance") + , "scanCmd" .= (r ^. the @"scanCmd") + , "digest" .= (r ^. the @"digest") + , "port" .= (r ^. the @"port") + , "proto" .= (r ^. the @"proto") + , "cidr" .= show (r ^. the @"cidr") + , "source" .= (r ^. the @"source") + , "data" .= (r ^. the @"data'") + ] + +readDestination :: String -> Parser (Either FilePath S3Uri) +readDestination s = (Right <$> readJson "destination (s3uri)" s) + <|> (Left <$> readJson "destination (filepath)" s) + +int64ToUTCTime :: Int64 -> UTCTime +int64ToUTCTime a = posixSecondsToUTCTime $ (a * 1000 * 1000) ^. from microseconds + +utcTimeToInt64 :: UTCTime -> Int64 +utcTimeToInt64 ut = round (toSeconds (utcTimeToPOSIXSeconds ut) :: Double) + +loadScanMetadata :: (MonadUnliftIO m, MonadReader r m, HasType Env r) + => S3Uri + -> m (Maybe ScanMetaData) +loadScanMetadata uri = runAwsTyped $ do + mb <- downloadFromS3Uri uri + pure $ mb >>= (decode . fromStrict)