packages feed

shodan-0.4.0.0: test/Main.hs

{-# LANGUAGE OverloadedStrings #-}

module Main (main) where

import           Control.Exception        (AsyncException (..), throwIO, try)
import           Control.Monad            (unless)
import           Data.Aeson               (Result (..), Value (..),
                                           eitherDecode', encode, fromJSON,
                                           object, (.=))
import qualified Data.Aeson.KeyMap        as KeyMap
import qualified Data.ByteString.Lazy     as Lazy
import qualified Data.Map.Strict          as Map
import qualified Data.Text                as Text
import qualified Data.Text.Encoding       as TextEncoding
import           DnsTests                 (dnsParsingTests)
import qualified Network.HTTP.Client      as HTTP
import qualified Network.HTTP.Types       as HTTPTypes
import qualified Network.Wai              as Wai
import           Network.Wai.Handler.Warp (testWithApplication)
import           SearchTests              (searchParsingTests)
import           Shodan

main :: IO ()
main = do
  let expected = ApiInfo 100000 (UsageLimits (-1) (-1) (-1))
        "stream-100" False True 100000 19 100000 False
  case fromJSON example of
    Success actual -> check "official example" (actual == expected)
    Error message  -> fail message
  case fromJSON (change (KeyMap.insert "future_field" Null)) of
    Success actual -> check "unknown fields" (actual == expected)
    Error message  -> fail message
  rejects "missing plan" (change (KeyMap.delete "plan"))
  rejects "null credits" (change (KeyMap.insert "query_credits" Null))
  rejects "wrong boolean type" (change (KeyMap.insert "https" (String "false")))
  rejects "fractional credits" (change (KeyMap.insert "scan_credits" (Number 1.5)))
  rejects "invalid nested limits" (change (KeyMap.insert "usage_limits" (object [])))
  rejects "non-object response" (Array mempty)
  facetTests
  countParsingTests
  dnsParsingTests
  searchParsingTests
  httpTests
  countTests
  domainHttpTests
  searchHttpTests
  putStrLn "All response parsing and HTTP tests passed."

facetTests :: IO ()
facetTests = do
  -- The organization bucket is from the official count endpoint example.
  let cases =
        [ ("official facet", "{\"value\":\"Amazon.com\",\"count\":3012386}",
            Facet (String "Amazon.com") 3012386)
        , ("numeric value", "{\"value\":443,\"count\":12}", Facet (Number 443) 12)
        , ("explicit null value", "{\"value\":null,\"count\":2}", Facet Null 2)
        , ("zero count", "{\"value\":\"KR\",\"count\":0}", Facet (String "KR") 0)
        , ("large count", "{\"value\":\"KR\",\"count\":100000000000000000000}",
            Facet (String "KR") 100000000000000000000)
        , ("unknown field", "{\"value\":\"KR\",\"count\":1,\"future\":true}",
            Facet (String "KR") 1)
        , ("null value preserved", "{\"value\":null,\"count\":1}", Facet Null 1)
        ]
  mapM_ (\(label, body, expected) ->
    check label (eitherDecode' body == Right expected)) cases
  let invalidCases =
        [ ("missing value", "{\"count\":1}")
        , ("missing count", "{\"value\":\"KR\"}")
        , ("null count", "{\"value\":\"KR\",\"count\":null}")
        , ("string count", "{\"value\":\"KR\",\"count\":\"1\"}")
        , ("fractional count", "{\"value\":\"KR\",\"count\":1.5}")
        , ("negative count", "{\"value\":\"KR\",\"count\":-1}")
        , ("non-object facet", "[]")
        , ("malformed facet JSON", "not JSON")
        , ("negative count", "{\"value\":\"KR\",\"count\":-1}")
        , ("non-object facet", "[]")
        , ("malformed facet", "not JSON")
        ]
  mapM_ (\(label, body) -> case eitherDecode' body :: Either String Facet of
    Left _  -> pure ()
    Right _ -> fail (label ++ ": unexpectedly accepted")) invalidCases
  let buckets = "[{\"value\":\"US\",\"count\":2},{\"value\":\"KR\",\"count\":1}]"
  check "facet list" (eitherDecode' buckets ==
    Right [Facet (String "US") 2, Facet (String "KR") 1])

httpTests :: IO ()
httpTests = do
  let key = "test key+&?=한글"
      application request respond = do
        check "GET method" (Wai.requestMethod request == "GET")
        check "request path" (Wai.pathInfo request == ["api-info"])
        check "encoded key" (Wai.queryString request ==
          [("key", Just (TextEncoding.encodeUtf8 key))])
        check "accept header" (lookup "Accept" (Wai.requestHeaders request) ==
          Just "application/json")
        respond (Wai.responseLBS HTTPTypes.status200 [] (encode example))
  withMockClient key application $ \client -> do
    check "client key redacted" (not (key `Text.isInfixOf` Text.pack (show client)))
    result <- getApiInfo client
    check "HTTP success" (result == (case fromJSON example of
      Success info -> Right info
      Error _      -> Left InvalidResponse))
  checkResponse "unauthorized" HTTPTypes.status401 "{\"error\":\"test-key\"}"
    (Left (HttpError 401))
  checkResponse "rate limited" HTTPTypes.status429 "not JSON"
    (Left (HttpError 429))
  checkResponse "server error" HTTPTypes.status500 "<html>unavailable</html>"
    (Left (HttpError 500))
  checkResponse "malformed JSON" HTTPTypes.status200 "not JSON"
    (Left InvalidResponse)
  checkResponse "wrong response shape" HTTPTypes.status200 "{}"
    (Left InvalidResponse)
  checkResponse "unexpected success status" HTTPTypes.status204 ""
    (Left (HttpError 204))
  let redirect _ respond = respond (Wai.responseLBS HTTPTypes.status302
        [("Location", "https://example.invalid/never-follow")] "")
  withMockClient "test-key" redirect $ \client -> do
    result <- getApiInfo client
    check "redirect not followed" (result == Left (HttpError 302))
  -- Inject an http-client transport exception without relying on external DNS.
  manager <- HTTP.newManager HTTP.defaultManagerSettings
    { HTTP.managerModifyRequest = \request -> throwIO
        (HTTP.HttpExceptionRequest request HTTP.ConnectionTimeout)
    }
  result <- getApiInfo (newClientWithManager manager "test-key")
  check "network failure" (result == Left NetworkError)
  cancellingManager <- HTTP.newManager HTTP.defaultManagerSettings
    { HTTP.managerModifyRequest = \_ -> throwIO ThreadKilled }
  cancelled <- try (getApiInfo (newClientWithManager cancellingManager "test-key"))
    :: IO (Either AsyncException (Either ShodanError ApiInfo))
  check "thread cancellation propagates" (cancelled == Left ThreadKilled)

countParsingTests :: IO ()
countParsingTests = do
  let country = [Facet (String "KR") 60, Facet (String "US") 40]
      expected = CountResult 100 (Map.fromList
        [("country", country), ("port", [Facet (Number 443) 80])])
      response = "{\"total\":100,\"facets\":{\"country\":[{\"value\":\"KR\",\"count\":60},{\"value\":\"US\",\"count\":40}],\"port\":[{\"value\":443,\"count\":80}]},\"matches\":[]}"
  check "count grouped facets" (eitherDecode' response == Right expected)
  check "count absent facets" (eitherDecode' "{\"total\":100}" ==
    Right (CountResult 100 Map.empty))
  check "count empty facets" (eitherDecode' "{\"total\":0,\"facets\":{}}" ==
    Right (CountResult 0 Map.empty))
  check "count empty bucket list" (eitherDecode'
    "{\"total\":0,\"facets\":{\"country\":[]}}" ==
    Right (CountResult 0 (Map.singleton "country" [])))
  mapM_ (\(label, body) -> case eitherDecode' body :: Either String CountResult of
    Left _  -> pure ()
    Right _ -> fail label)
    [ ("null facets", "{\"total\":1,\"facets\":null}")
    , ("array facets", "{\"total\":1,\"facets\":[]}")
    , ("non-list buckets", "{\"total\":1,\"facets\":{\"country\":{}}}")
    , ("null buckets", "{\"total\":1,\"facets\":{\"country\":null}}")
    , ("missing bucket count", "{\"total\":1,\"facets\":{\"country\":[{\"value\":\"KR\"}]}}")
    , ("negative bucket count", "{\"total\":1,\"facets\":{\"port\":[{\"value\":443,\"count\":-1}]}}")
    ]

countTests :: IO ()
countTests = do
  let key = "count-test-key"
      query = "http.title:\"한글 + & ?\" country:KR"
      application request respond = do
        check "count GET" (Wai.requestMethod request == "GET")
        check "count path" (Wai.pathInfo request == ["shodan", "host", "count"])
        check "count query encoding" (Wai.queryString request ==
          [("key", Just (TextEncoding.encodeUtf8 key)),
           ("query", Just (TextEncoding.encodeUtf8 query))])
        respond (Wai.responseLBS HTTPTypes.status200 []
          "{\"total\":19592093,\"matches\":[]}")
  withMockClient key application $ \client -> do
    actual <- countHosts client query
    check "count success" (actual == Right (CountResult 19592093 Map.empty))
  countResponse "zero count" HTTPTypes.status200 "{\"total\":0}"
    (Right (CountResult 0 Map.empty))
  countResponse "large count and unknown fields" HTTPTypes.status200
    "{\"total\":100000000000000000000,\"facets\":{},\"future\":true}"
    (Right (CountResult 100000000000000000000 Map.empty))
  mapM_ (\body -> countResponse "invalid count" HTTPTypes.status200 body
    (Left InvalidResponse))
    [ "{}", "{\"total\":null}", "{\"total\":\"12\"}"
    , "{\"total\":1.5}", "{\"total\":-1}", "[]", "invalid JSON"
    ]
  countResponse "count unauthorized" HTTPTypes.status401 "{\"error\":\"Invalid key\"}"
    (Left (HttpError 401))
  countResponse "count rate limited" HTTPTypes.status429 "not JSON"
    (Left (HttpError 429))

searchHttpTests :: IO ()
searchHttpTests = do
  let key = "search-key+&"
      query = "http.title:\"한글 + & ?\" product:nginx"
      responseBody = "{\"total\":101,\"matches\":[{\"ip_str\":\"192.0.2.1\",\"port\":443,\"product\":\"nginx\",\"asn\":\"AS65536\"}]}"
      expected = Right (SearchResult 101
        [SearchMatch "192.0.2.1" 443 (Just "nginx") Nothing (Just "AS65536") Nothing Nothing Nothing Nothing Nothing] Map.empty)
      application page request respond = do
        check "search GET" (Wai.requestMethod request == "GET")
        check "search path" (Wai.pathInfo request == ["shodan", "host", "search"])
        check "search encoded parameters" (Wai.queryString request ==
          [("key", Just (TextEncoding.encodeUtf8 key)),
           ("query", Just (TextEncoding.encodeUtf8 query)),
           ("page", Just page)])
        respond (Wai.responseLBS HTTPTypes.status200 [] responseBody)
  withMockClient key (application "1") $ \client -> do
    actual <- searchHosts client query 1
    check "first search page" (actual == expected)
  withMockClient key (application "2") $ \client -> do
    actual <- searchHosts client query 2
    check "explicit second search page" (actual == expected)
  manager <- HTTP.newManager HTTP.defaultManagerSettings
    { HTTP.managerModifyRequest = \_ -> fail "Invalid page sent an HTTP request" }
  mapM_ (\page -> do
    actual <- searchHosts (newClientWithManager manager key) query page
    check "invalid page rejected before HTTP" (actual == Left InvalidPage)) [0, -1]
  let cases =
        [ ("empty search page", HTTPTypes.status200, "{\"total\":0,\"matches\":[]}",
            Right (SearchResult 0 [] Map.empty))
        , ("search unauthorized", HTTPTypes.status401, "{\"error\":\"search-key+&\"}",
            Left (HttpError 401))
        , ("search rate limited", HTTPTypes.status429, "not JSON", Left (HttpError 429))
        , ("search invalid JSON", HTTPTypes.status200, "not JSON", Left InvalidResponse)
        , ("search invalid match", HTTPTypes.status200,
            "{\"total\":1,\"matches\":[{}]}", Left InvalidResponse)
        ]
  mapM_ (\(label, status, body, wanted) -> do
    let mock _ respond = respond (Wai.responseLBS status [] body)
    withMockClient key mock $ \client -> do
      actual <- searchHosts client query 1
      check label (actual == wanted)) cases

domainHttpTests :: IO ()
domainHttpTests = do
  let key = "dns-key+&한글"
      body = "{\"domain\":\"Example.COM\",\"tags\":[],\"data\":[],\"subdomains\":[\"www\"],\"more\":true}"
      expected = Right (DomainInfo "Example.COM" [] [] ["www"] True)
      application page request respond = do
        check "domain GET" (Wai.requestMethod request == "GET")
        check "domain path" (Wai.rawPathInfo request == "/dns/domain/Example.COM")
        check "domain parameters" (Wai.queryString request ==
          [("key", Just (TextEncoding.encodeUtf8 key)), ("page", Just page)])
        check "domain accepts JSON" (lookup "Accept" (Wai.requestHeaders request) == Just "application/json")
        respond (Wai.responseLBS HTTPTypes.status200 [] body)
  mapM_ (\(page, encoded) -> withMockClient key (application encoded) $ \client -> do
    actual <- getDomainInfo client "Example.COM" page
    check "domain selected page" (actual == expected)) [(1,"1"), (2,"2")]
  manager <- HTTP.newManager HTTP.defaultManagerSettings
    { HTTP.managerModifyRequest = \_ -> fail "Invalid DNS argument sent HTTP" }
  let client = newClientWithManager manager key
  mapM_ (\page -> do
    actual <- getDomainInfo client "example.com" page
    check "invalid DNS page" (actual == Left InvalidPage)) [0,-1]
  mapM_ (\domain -> do
    actual <- getDomainInfo client domain 1
    check "invalid DNS domain" (actual == Left InvalidDomain))
    ["", " ", ".", "example..com", "example.com.", "-bad.com", "bad-.com",
     "https://example.com", "example.com/path", "example.com?key=oops",
     "example.com#fragment", "example%2f.com", "example_com", "한글.com",
     Text.replicate 64 "a" <> ".com", Text.intercalate "." (replicate 4 (Text.replicate 63 "a"))]
  -- Longest allowed labels and complete name are accepted without modification.
  let longest = Text.intercalate "." (replicate 3 (Text.replicate 63 "a") ++ [Text.replicate 61 "b"])
  mapM_ (\domain -> do
    let mock request respond = do
          check "DNS name passed verbatim" (Wai.pathInfo request == ["dns", "domain", domain])
          respond (Wai.responseLBS HTTPTypes.status200 [] body)
    withMockClient key mock $ \validClient -> do
      actual <- getDomainInfo validClient domain 1
      check "valid DNS name" (actual == expected)) ["xn--3e0b707e.com", "a-b.example.com", longest]
  let cases =
        [(HTTPTypes.status401, "secret", Left (HttpError 401)),
         (HTTPTypes.status403, "secret", Left (HttpError 403)),
         (HTTPTypes.status404, "secret", Left (HttpError 404)),
         (HTTPTypes.status429, "secret", Left (HttpError 429)),
         (HTTPTypes.status500, "secret", Left (HttpError 500)),
         (HTTPTypes.status302, "redirect", Left (HttpError 302)),
         (HTTPTypes.status200, "not JSON", Left InvalidResponse),
         (HTTPTypes.status200, "{}", Left InvalidResponse)]
  mapM_ (\(status, responseBody, wanted) -> do
    let mock _ respond = respond (Wai.responseLBS status [("Location", "/unexpected")] responseBody)
    withMockClient key mock $ \validClient -> do
      actual <- getDomainInfo validClient "example.com" 1
      check "domain HTTP error" (actual == wanted)) cases

countResponse :: String -> HTTPTypes.Status -> Lazy.ByteString
  -> Either ShodanError CountResult -> IO ()
countResponse label status body expected = do
  let application _ respond = respond (Wai.responseLBS status [] body)
  withMockClient "count-test-key" application $ \client -> do
    actual <- countHosts client "port:443"
    check label (actual == expected)

checkResponse :: String -> HTTPTypes.Status -> Lazy.ByteString
  -> Either ShodanError ApiInfo -> IO ()
checkResponse label status body expected = do
  let application _ respond = respond (Wai.responseLBS status [] body)
  withMockClient "test-key" application $ \client -> do
    actual <- getApiInfo client
    check label (actual == expected)

-- Redirect only the test transport to a local server; production always uses
-- the fixed HTTPS endpoint. Validate the original destination before rewriting.
withMockClient :: Text.Text -> Wai.Application -> (Client -> IO ()) -> IO ()
withMockClient key application run = testWithApplication (pure application) $ \port -> do
  manager <- HTTP.newManager HTTP.defaultManagerSettings
    { HTTP.managerModifyRequest = \request -> do
        check "finite timeout" (HTTP.responseTimeout request ==
          HTTP.responseTimeoutMicro 30000000)
        -- http-client can apply the hook more than once to the same request.
        if not (HTTP.secure request) && HTTP.host request == "127.0.0.1"
            && HTTP.port request == port
          then pure request
          else do
            check "HTTPS endpoint" (HTTP.secure request &&
              HTTP.host request == "api.shodan.io" && HTTP.port request == 443)
            pure request
              { HTTP.secure = False
              , HTTP.host = "127.0.0.1"
              , HTTP.port = port
              , HTTP.proxy = Nothing
              }
    }
  run (newClientWithManager manager key)

check :: String -> Bool -> IO ()
check label passed = unless passed (fail label)

rejects :: String -> Value -> IO ()
rejects label value = case fromJSON value :: Result ApiInfo of
  Error _   -> pure ()
  Success _ -> fail (label ++ ": unexpectedly accepted")

change :: (KeyMap.KeyMap Value -> KeyMap.KeyMap Value) -> Value
change f = case example of
  Object o -> Object (f o)
  _        -> example

-- Source: https://developer.shodan.io/api, GET /api-info response example.
example :: Value
example = object
  [ "scan_credits" .= (100000 :: Integer)
  , "usage_limits" .= object
      [ "scan_credits" .= (-1 :: Integer)
      , "query_credits" .= (-1 :: Integer)
      , "monitored_ips" .= (-1 :: Integer)
      ]
  , "plan" .= String "stream-100"
  , "https" .= False
  , "unlocked" .= True
  , "query_credits" .= (100000 :: Integer)
  , "monitored_ips" .= (19 :: Integer)
  , "unlocked_left" .= (100000 :: Integer)
  , "telnet" .= False
  ]