packages feed

hs-hath-1.3.0: app/Bench.hs

-- | hath-bench: plays the H@H RPC server for a client running on this machine
-- (hs-hath or the official Java client), replays the measurements the real
-- server makes, and prints a scorecard.
--
-- The real server's quality inputs we can reproduce:
--   * servercmd speed_test / still_alive right after client_start
--   * servercmd threaded_proxy_test (the client measures someone else)
--   * other clients pulling /t/ (5 x 5 MB concurrently)
--   * readers pulling /h/ files, cold (srfetch + upstream) and warm
module Bench ( main ) where

import           Control.Concurrent       ( forkIO, threadDelay )
import           Control.Exception        ( try )

import qualified Data.ByteString          as BS
import qualified Data.ByteString.Builder  as BB
import qualified Data.ByteString.Char8    as BSC
import qualified Data.List                as List
import qualified Data.Map.Strict          as Map
import           Data.String.Interpolate  ( i )
import qualified Data.Text                as T
import           Data.Time.Clock.System   ( getSystemTime, systemSeconds )

import           Dhall                    ( embed, inject )
import qualified Dhall.Core               as Dhall

import           GHC.Clock                ( getMonotonicTime )

import           Hath.Config              ( ClientConfig(..), defaultClientConfig )
import           Hath.Protocol            ( sha1HexText )

import           Network.Connection       ( TLSSettings(TLSSettingsSimple) )
import qualified Network.HTTP.Client      as HC
import           Network.HTTP.Client.TLS  ( mkManagerSettings )
import qualified Network.HTTP.Types       as H
import           Network.TLS              ( defaultSupported )
import qualified Network.Wai              as Wai
import qualified Network.Wai.Handler.Warp as Warp

import           Options.Applicative

import           Relude

import           System.Directory         ( createDirectoryIfMissing, makeAbsolute )
import           System.Environment       ( getEnvironment )
import           System.FilePath          ( (</>) )
import           System.Process           ( CreateProcess(env), proc, readCreateProcess )

import           Text.Printf              ( printf )

import           UnliftIO                 ( pooledMapConcurrentlyN, timeout )

data Opts
  = Opts { oClientId   :: !Text
         , oClientKey  :: !Text
         , oRpcPort    :: !Int
         , oClientPort :: !Int
         , oFiles      :: !Int
         , oFileSize   :: !Int
         , oUpDelayMs  :: !Int
         , oUpKBps     :: !Int
         , oWorkDir    :: !FilePath
         , oClientHost :: !Text -- ^ where the bench reaches the client
         , oPublicHost :: !Text -- ^ where the client reaches the bench (also its rpc_server_ip)
         }

optsP :: Parser Opts
optsP
  = Opts
  <$> strOption
    (long "client-id" <> value "12345" <> showDefault <> help "Client ID of the client under test")
  <*> strOption
    (long "client-key"
     <> value "benchbenchbenchbench"
     <> showDefault
     <> help "Client key (the Java client wants 20 alphanumerics)")
  <*> option
    auto
    (long "rpc-port"
     <> value 19800
     <> showDefault
     <> help "Port of the fake RPC server, image server and speed-test target")
  <*> option
    auto
    (long "client-port"
     <> value 18443
     <> showDefault
     <> help "Port the client is told to listen on")
  <*> option
    auto
    (long "files" <> value 30 <> showDefault <> help "Gallery size for the /h/ phases")
  <*> option
    auto
    (long "file-size" <> value 300000 <> showDefault <> help "Bytes per gallery file")
  <*> option
    auto
    (long "upstream-delay-ms"
     <> value 300
     <> showDefault
     <> help "Latency the fake image server adds before its first byte")
  <*> option
    auto
    (long "upstream-kbps"
     <> value 1000
     <> showDefault
     <> help "Fake image server bandwidth in KB/s (0 = unlimited)")
  <*> strOption
    (long "work-dir"
     <> value "bench-work"
     <> showDefault
     <> help "Where certs and client configs go")
  <*> strOption
    (long "client-host"
     <> value "127.0.0.1"
     <> showDefault
     <> help "Address of the client under test")
  <*> strOption
    (long "public-host"
     <> value "127.0.0.1"
     <> showDefault
     <> help
       "Address the client uses to reach this bench; anything but 127.0.0.1 listens on all IPv4 interfaces")

data GFile = GFile { gIndex :: !Int, gId :: !Text, gBytes :: !ByteString }

data Bench
  = Bench { bOpts     :: !Opts
          , bP12      :: !ByteString
          , bFiles    :: !(Map Text GFile)
          , bRpcCalls :: !(IORef (Map Text Int))
          , bSrfetch  :: !(IORef Int)
          , bStarted  :: !(MVar ())
          }

main :: IO ()
main = do
  hSetBuffering stdout LineBuffering
  o <- execParser
    (info
       (optsP <**> helper)
       (fullDesc <> progDesc "Fake H@H RPC server that measures a local client"))
  when (oFileSize o < 64) $ die "--file-size must be at least 64"
  wd <- makeAbsolute (oWorkDir o)
  createDirectoryIfMissing True (wd </> "java" </> "data")
  p12 <- makeCert o wd
  writeClientConfigs o wd
  -- a fresh nonce per run gives new file ids, so the cold phase is really cold
  nonce <- systemSeconds <$> getSystemTime
  let files = mkGallery nonce (oFiles o) (oFileSize o)
  b <- Bench o p12 (Map.fromList [ ( gId f, f ) | f <- files ]) <$> newIORef mempty
    <*> newIORef 0
    <*> newEmptyMVar
  void
    $ forkIO
    $ Warp.runSettings
      (Warp.setHost
         (if oPublicHost o == "127.0.0.1"
            then "127.0.0.1"
            else "*4")
       $ Warp.setPort (oRpcPort o) Warp.defaultSettings)
      (app b)
  putTextLn (instructions o wd)
  takeMVar (bStarted b)
  -- the Java client only accepts normal connections once client_start has returned
  putTextLn "client_start received, starting in 3 s"
  threadDelay 3000000
  mgr <- HC.newManager
    (mkManagerSettings (TLSSettingsSimple True False False defaultSupported) Nothing)
    { HC.managerResponseTimeout = HC.responseTimeoutMicro (90 * 1000000) }
  rows <- runPhases b mgr files
  calls <- readIORef (bRpcCalls b)
  report o rows calls
  unless (all rowPassed rows) exitFailure

-- | Self-signed cert packed as PKCS#12 with the client key as password, which is what get_cert returns.
makeCert :: Opts -> FilePath -> IO ByteString
makeCert o wd = do
  environment <- getEnvironment
  let openssl args
        = void
        $ readCreateProcess
          (proc "openssl" args)
          { env = Just (( "HATH_BENCH_PASS", toString (oClientKey o) ) : environment) }
          ""
  openssl
    [ "req"
    , "-x509"
    , "-newkey"
    , "rsa:2048"
    , "-nodes"
    , "-days"
    , "7"
    , "-subj"
    , "/CN=127.0.0.1"
    , "-keyout"
    , wd </> "key.pem"
    , "-out"
    , wd </> "cert.pem"
    ]
  openssl
    [ "pkcs12"
    , "-export"
    , "-inkey"
    , wd </> "key.pem"
    , "-in"
    , wd </> "cert.pem"
    , "-out"
    , wd </> "cert.p12"
    , "-passout"
    , "env:HATH_BENCH_PASS"
    , "-name"
    , "hath.network"
    ] -- the alias the Java client looks up
  readFileBS (wd </> "cert.p12")

writeClientConfigs :: Opts -> FilePath -> IO ()
writeClientConfigs o wd = do
  writeFileText (wd </> "java" </> "data" </> "client_login") [i|#{oClientId o}-#{oClientKey o}|]
  writeFileText (wd </> "hs-hath.dhall")
    $ Dhall.pretty
    $ embed
      inject
      defaultClientConfig { clientId    = oClientId o
                          , key         = oClientKey o
                          , downloadDir = toText (wd </> "hs-download")
                          , cachePath   = toText (wd </> "hs-cache.db")
                          }

instructions :: Opts -> FilePath -> Text
instructions o wd
  = [i|hath-bench: fake RPC server on #{oPublicHost o}:#{oRpcPort o}. Start the client under test on #{oClientHost o}
(copy #{wd} there first if that is another machine):

  hs-hath:  cd #{wd} && HATH_RPC_HOST=#{oPublicHost o}:#{oRpcPort o} hs-hath --config ./hs-hath.dhall --skip-startup-verify
  Java:     cd #{wd}/java && java -jar /path/to/HentaiAtHome.jar --rpc_server_ip=#{oPublicHost o} --rpc_server_port=#{oRpcPort o}

(both configs use client id #{oClientId o}; pass --client-id/--client-key to change them)
Waiting for client_start...|]

mkGallery :: Int64 -> Int -> Int -> [ GFile ]
mkGallery nonce n size = file <$> [ 1 .. n ]
  where
    file k = GFile k fileId body
      where
        prefix = encodeUtf8 ([i|hath-bench #{nonce} #{k} |] :: Text)

        body   = prefix <> BS.replicate (size - BS.length prefix) 46

        fileId = [i|#{sha1HexText body}-#{size}-1280-1707-jpg|]

-- | RPC endpoint, fake image server (/up) and speed-test target (/t) on one port.
app :: Bench -> Wai.Application
app b req respond = case Wai.pathInfo req of
  [ "15", "rpc" ] -> rpc False
  -- 1.6.6+ (build 181): same actions, signed responses
  [ "16", "rpc" ] -> rpc True
  [ "up", fileId ] -> upstream fileId
  [ "t", sizeText, _, testKey, _ ] -> testTarget sizeText testKey
  _ -> respond $ Wai.responseLBS H.status404 [] ""
  where
    o = bOpts b

    -- Not Wai.queryString: parseQuery also splits on ';', and the Java client sends
    -- add=fileindex;xres;fileid unescaped.
    param k
      = maybe "" (H.urlDecode True . BS.drop (BS.length k + 1))
      $ find ((k <> "=") `BS.isPrefixOf`)
      $ BSC.split '&' (BS.dropWhile (== 63) (Wai.rawQueryString req))

    -- the Java client insists on a Content-Length
    rpcResponse body
      = respond
      $ Wai.responseLBS
        H.status200
        [ ( H.hContentLength, show (BS.length body) ) ]
        (fromStrict body)

    -- 1.6.6 signs replies: OK-<t>-<sha1("<t>-<key>" ++ concatMap ("\n" ++) lines)>
    okWith signed ls = do
      now <- systemSeconds <$> getSystemTime
      let sig
            = sha1HexText (encodeUtf8 (T.concat ([i|#{now}-#{oClientKey o}|] : map ("\n" <>) ls)))
          status
            = if signed
              then [i|OK-#{now}-#{sig}|]
              else "OK"
      rpcResponse (encodeUtf8 (unlines (status : ls)))

    settings
      = [ [i|rpc_server_ip=#{oPublicHost o}|]
        , [i|host=#{oClientHost o}|]
        , [i|port=#{oClientPort o}|]
          -- what the real server sends; Java stalls non-local transfers on throttle_bytes=0
        , "throttle_bytes=640000000"
        , "disable_bwm=true"
        , "disklimit_bytes=1073741824"
        , "diskremaining_bytes=1073741824"
        ]

    rpc signed = do
      let ok = okWith signed
      let act = decodeUtf8 (param "act") :: Text
      atomicModifyIORef' (bRpcCalls b) (\m -> ( Map.insertWith (+) act 1 m, () ))
      now <- systemSeconds <$> getSystemTime
      case act of
        "server_stat"
          -> ok [ "min_client_build=1", "cur_client_build=999", [i|server_time=#{now}|] ]
        "client_login" -> ok settings
        "client_settings" -> ok settings
        "get_cert" -> rpcResponse (bP12 b)
        "client_start" -> tryPutMVar (bStarted b) () >> ok []
        "srfetch" -> do
          atomicModifyIORef' (bSrfetch b) (\n -> ( n + 1, () ))
          -- add = fileindex;xres;fileid
          let fileId = decodeUtf8 (BSC.takeWhileEnd (/= ';') (param "add")) :: Text
          ok [ [i|http://#{oPublicHost o}:#{oRpcPort o}/up/#{fileId}|] ]
        _ -> ok [] -- still_alive, get_blacklist, client_stop, ...

    upstream fileId = case Map.lookup fileId (bFiles b) of
      Nothing -> respond $ Wai.responseLBS H.status404 [] ""
      Just f  -> do
        threadDelay (oUpDelayMs o * 1000)
        let body = gBytes f
        respond
          $ Wai.responseStream
            H.status200
            [ ( H.hContentType, "image/jpeg" ), ( H.hContentLength, show (BS.length body) ) ]
          $ \write flush -> forM_ (chunksOf 16384 body) $ \c -> do
            write (BB.byteString c) >> flush
            when (oUpKBps o > 0) $ threadDelay (BS.length c * 1000 `div` oUpKBps o)

    testTarget sizeText testKey = case readMaybe @Int (toString sizeText) of
      Nothing   -> respond $ Wai.responseLBS H.status400 [] ""
      Just size
        | testKey == "broken403" -> respond $ Wai.responseLBS H.status403 [] ""
        -- no Content-Length and only half the bytes: the tester sees a clean, early EOF
        | testKey == "brokenshort" -> zeros [] (size `div` 2)
        | otherwise -> zeros [ ( H.hContentLength, show size ) ] size

    zeros headers n = respond $ Wai.responseStream H.status200 headers $ \write _
      -> forM_ (chunksOf 65536 (BS.replicate n 48)) (write . BB.byteString)

chunksOf :: Int -> ByteString -> [ ByteString ]
chunksOf n bs
  | BS.null bs = []
  | otherwise
    = let
        ( a, rest ) = BS.splitAt n bs
      in 
        a : chunksOf n rest

data Sample
  = Sample { sStatus :: !(Maybe Int) -- ^ Nothing: connection error or timeout
           , sTtfb   :: !Double      -- ^ seconds until response headers
           , sTotal  :: !Double      -- ^ seconds until the body is drained
           , sBytes  :: !Int
           , sHead   :: !ByteString  -- ^ first bytes of the body
           }

-- | GET with redirects followed, timing headers and body separately.
fetch :: HC.Manager -> String -> IO Sample
fetch mgr url = do
  t0 <- getMonotonicTime
  r <- timeout (90 * 1000000) $ try @SomeException $ do
    req <- HC.parseRequest url
    HC.withResponse req mgr $ \resp -> do
      t1 <- getMonotonicTime
      let drain !n acc = HC.brRead (HC.responseBody resp) >>= \c -> if BS.null c
            then pure ( n, acc )
            else drain
              (n + BS.length c)
              (if BS.length acc < 256
                 then acc <> BS.take 256 c
                 else acc)
      ( n, headBytes ) <- drain 0 ""
      t2 <- getMonotonicTime
      pure $ Sample (Just (H.statusCode (HC.responseStatus resp))) (t1 - t0) (t2 - t0) n headBytes
  t3 <- getMonotonicTime
  pure $ case r of
    Just (Right s) -> s
    _ -> Sample Nothing (t3 - t0) (t3 - t0) 0 ""

data Row
  = Row { rowName    :: !Text
        , rowSamples :: ![ Sample ]
        , rowWall    :: !Double
        , rowPassed  :: !Bool
        , rowNote    :: !Text
        }

runPhases :: Bench -> HC.Manager -> [ GFile ] -> IO [ Row ]
runPhases b mgr files = do
  now <- systemSeconds <$> getSystemTime
  let o = bOpts b
      base = [i|https://#{oClientHost o}:#{oClientPort o}|] :: String
      sha1 = sha1HexText . encodeUtf8 @Text

      servercmd :: Text -> Text -> String
      servercmd cmd add
        = let
            sig
              = sha1 [i|hentai@home-servercmd-#{cmd}-#{add}-#{oClientId o}-#{now}-#{oClientKey o}|]
          in 
            [i|#{base}/servercmd/#{cmd}/#{add}/#{now}/#{sig}|]

      proxyTest :: Text -> String
      proxyTest k
        = servercmd
          "threaded_proxy_test"
          [i|hostname=#{oPublicHost o};protocol=http;port=#{oRpcPort o};testsize=1000000;testcount=5;testtime=#{now};testkey=#{k}|]

      speedKey = sha1 [i|hentai@home-speedtest-5000000-#{now}-#{oClientId o}-#{oClientKey o}|]

      fileUrl f
        = let
            stamp = T.take 10 (sha1 [i|#{now}-#{gId f}-#{oClientKey o}-hotlinkthis|])
          in 
            [i|#{base}/h/#{gId f}/keystamp=#{now}-#{stamp};fileindex=#{gIndex f};xres=org/#{gIndex f}.jpg|]

      -- run n requests with the given concurrency; the check sees all samples
      phase name conc urls check = do
        t0 <- getMonotonicTime
        ss <- pooledMapConcurrentlyN conc (fetch mgr) urls
        t1 <- getMonotonicTime
        let ( passed, note ) = check ss
        pure $ Row name ss (t1 - t0) passed note

      allFull size = all (\s -> sStatus s == Just 200 && sBytes s == size)
      bodyText = decodeUtf8 . sHead :: Sample -> Text
      bodyIs p ss
        = ( all (\s -> sStatus s == Just 200 && p (bodyText s)) ss
          , T.intercalate " " (bodyText <$> ss)
          )

      galleryPhase name expectFetches = do
        before <- readIORef (bSrfetch b)
        row <- phase name 4 (fileUrl <$> files) (\ss -> ( allFull (oFileSize o) ss, "" ))
        fetches <- subtract before <$> readIORef (bSrfetch b)
        pure
          row { rowPassed = rowPassed row && fetches == expectFetches
              , rowNote   = [i|srfetch=#{fetches} (want #{expectFetches})|]
              }

  -- for poking the client from a third machine (e.g. a flood-control burst)
  writeFileText (oWorkDir o </> "urls.txt") (unlines (toText . fileUrl <$> files))
  sequence
    [ phase "servercmd still_alive" 1 [ servercmd "still_alive" "" ] (bodyIs (const True))
    , phase
        "servercmd speed_test"
        2
        (replicate 2 (servercmd "speed_test" "testsize=2500000"))
        (\ss -> ( allFull 2500000 ss, "" ))
    , phase
        "servercmd unsigned"
        1
        [ [i|#{base}/servercmd/still_alive//#{now}/0|] ]
        (\ss -> ( map sStatus ss == [ Just 403 ], "want 403" ))
    , phase "proxy_test healthy" 1 [ proxyTest "ok" ] (bodyIs ("OK:5-" `T.isPrefixOf`))
    , phase "proxy_test target 403" 1 [ proxyTest "broken403" ] (bodyIs (== "OK:0-0"))
    , phase "proxy_test short body" 1 [ proxyTest "brokenshort" ] (bodyIs (== "OK:0-0"))
    , phase
        "/t/ 5x5MB concurrent"
        5
        [ [i|#{base}/t/5000000/#{now}/#{speedKey}/#{k}|] | k <- [ 1 .. 5 :: Int ] ]
        (\ss -> ( allFull 5000000 ss, "" ))
    , galleryPhase "/h/ cold (cache miss)" (length files)
    , galleryPhase "/h/ warm (cache hit)" 0
    ]

report :: Opts -> [ Row ] -> Map Text Int -> IO ()
report o rows calls = do
  printf "\nhath-bench results (client on %s:%d)\n\n" (toString (oClientHost o)) (oClientPort o)
  printf
    "%-24s %-14s %15s %15s %8s  %s\n"
    ("phase" :: String)
    ("status" :: String)
    ("ttfb p50/p95" :: String)
    ("total p50/p95" :: String)
    ("MB/s" :: String)
    ("check" :: String)
  forM_ rows $ \r -> do
    let ss       = rowSamples r
        statuses
          = unwords
            [ [i|#{maybe "err" (show @Text) s}x#{c}|] :: Text
            | ( s, c ) <- Map.toList (Map.fromListWith (+) [ ( sStatus x, 1 :: Int ) | x <- ss ])
            ]
        ms f = [i|#{msAt 0.5 f ss}/#{msAt 0.95 f ss}|] :: Text
        mbps     = fromIntegral (sum (sBytes <$> ss)) / max 1e-9 (rowWall r) / 1e6 :: Double
    printf
      "%-24s %-14s %15s %15s %8.1f  %s %s\n"
      (toString (rowName r))
      (toString statuses)
      (toString (ms sTtfb))
      (toString (ms sTotal))
      mbps
      (if rowPassed r
         then "ok"
         else "FAIL" :: String)
      (toString (rowNote r))
  printf
    "\nRPC calls seen: %s\n"
    (toString (unwords [ [i|#{a}x#{n}|] | ( a, n ) <- Map.toList calls ]))
  where
    msAt :: Double -> (Sample -> Double) -> [ Sample ] -> Int
    msAt p f ss = case sort (f <$> ss) of
      [] -> 0
      xs -> round (1000 * (xs List.!! min (length xs - 1) (floor (p * fromIntegral (length xs)))))