packages feed

memcache-0.5.0.0: test/MockServer.hs

{-# LANGUAGE OverloadedStrings #-}

-- | Mock Memcached server - just enough for testing client.
--
-- Each mock listens on 127.0.0.1 unless given another address, and on an
-- ephemeral port of its own unless given a port, so tests can run several at
-- once and never clash with a real memcached on the default port. A mock can
-- also listen on a Unix socket, at a path from 'withSocketPath'.
--
-- The client asks for the server's version on every new connection before it
-- sends anything else. A mock made by 'withMockServer' and the functions like
-- it answers that request like any other, with its next response, so a test
-- can script the reply. A mock made by 'withMockMemcached' and the functions
-- like it reports itself as memcached of a given version instead.
module MockServer (
        MockResponse(..), MockServer(..), withMockServer, withMockServerAt,
        withConcurrentMockServer, withMockMemcached, withMockMemcachedAt,
        withConcurrentMockMemcached, withBoundSocket, withBoundSocketReuseAddr,
        withFullListener, withSocketPath, canListenAt, mockServerSpec
    ) where

import           Control.Concurrent
import           Control.Monad
import qualified Data.ByteString           as B
import qualified Data.ByteString.Char8     as C
import           Data.IORef
import           Data.Maybe                (isJust)
import           Database.Memcache.Types   (Authentication (NoAuth), ServerSpec (..))
import qualified Network.Socket            as N
import qualified Network.Socket.ByteString as N
import           System.Directory          (getTemporaryDirectory, removeFile,
                                            removePathForcibly)
import           System.IO                 (hClose, openTempFile)
import           System.Timeout            (timeout)
import           Text.Read                  (readMaybe)
import           UnliftIO.Async             (concurrently_, withAsync)
import           UnliftIO.Exception         (IOException, SomeException, bracket, finally,
                                            handle, onException, throwIO)

-- | Actions the mock server can take to a request.
data MockResponse
    -- | Reply with these bytes.
    = MR B.ByteString
    -- | Close the client connection instead of replying.
    | CloseConnection
    -- | Wait this many milliseconds, then take the given action.
    | DelayMS Int MockResponse
    -- | Read the request but send nothing back.
    | Noop
    -- | Send these bytes over and over until the client closes the
    -- connection.
    | Repeat B.ByteString
    -- | Reply with these bytes, then reset the connection: close it with
    -- @SO_LINGER@ set to 0, so that once the client has read the bytes, its
    -- next read fails with @ECONNRESET@ rather than finding the connection
    -- closed. TCP only. The 'MVar' is filled once the connection is closed.
    | ReplyThenReset B.ByteString (MVar ())

-- | A running mock server.
data MockServer = MockServer
    { -- | The port the mock is listening on, or 0 on a Unix socket.
      mockPort     :: N.PortNumber
      -- | Every request received so far, oldest first, including any data
      -- block that followed the command line.
    , mockRequests :: IO [B.ByteString]
      -- | How many connections the client has closed while the mock was
      -- waiting for its next request.
    , mockClosed   :: IO Int
    }

-- | A server specification for connecting to the mock without authentication.
mockServerSpec :: MockServer -> ServerSpec
mockServerSpec server = ServerSpec "127.0.0.1" (show (mockPort server)) NoAuth

-- | Run an action with a mock Memcached server running in the background,
-- killing it once done. The server answers requests with the given responses
-- in order, one per request, across however many connections the client
-- makes; once they run out it closes the connection and stops listening.
withMockServer :: [MockResponse] -> (MockServer -> IO a) -> IO a
withMockServer = withMockServerAt loopback

-- | Like 'withMockServer', but listening on the given address instead of
-- 127.0.0.1. A port of 0 picks a free one.
withMockServerAt :: N.SockAddr -> [MockResponse] -> (MockServer -> IO a) -> IO a
withMockServerAt = serveInTurn Nothing

-- | Like 'withMockServer', but the mock reports itself as memcached of the
-- given version, such as @"1.6.24"@: it answers the first request on each
-- connection, the client's version check, with @VERSION@ and that version,
-- without using up a response. That request is still recorded.
--
-- The mock answers each connection's version check as soon as it accepts the
-- connection, even while it is serving another one, as the client can't use
-- a connection until then. It still serves the responses to one connection
-- at a time, in the order it accepted them, and once the responses run out,
-- it closes every connection and stops listening.
withMockMemcached :: B.ByteString -> [MockResponse] -> (MockServer -> IO a) -> IO a
withMockMemcached version = withMockMemcachedAt version loopback

-- | Like 'withMockServerAt', but reporting the given version, as
-- 'withMockMemcached' does.
withMockMemcachedAt :: B.ByteString -> N.SockAddr -> [MockResponse] -> (MockServer -> IO a) -> IO a
withMockMemcachedAt version = serveInTurn (Just (versionReply version))

-- | Like 'withConcurrentMockServer', but reporting the given version, as
-- 'withMockMemcached' does.
withConcurrentMockMemcached :: B.ByteString -> [[MockResponse]] -> (MockServer -> IO a) -> IO a
withConcurrentMockMemcached version = serveConcurrently (Just (versionReply version))

versionReply :: B.ByteString -> B.ByteString
versionReply version = "VERSION " <> version <> "\r\n"

-- | Serve one connection at a time on the given address, answering the first
-- request on each connection with the greeting, if there is one, and then
-- each request with the next of the responses.
serveInTurn
  :: Maybe B.ByteString -> N.SockAddr -> [MockResponse] -> (MockServer -> IO a) -> IO a
serveInTurn greeting address responses = withListener address $ \sock received closed -> do
  remaining <- newIORef responses
  -- Once the responses run out the listening socket closes, so later
  -- connection attempts are refused straight away.
  case greeting of
    Nothing -> acceptLoop sock remaining received closed
    Just reply -> greetingEach reply sock remaining received closed
  where
    -- Serve one connection at a time.
    acceptLoop sock remaining received closed = do
      pending <- readIORef remaining
      unless (null pending) $ do
        (client, _) <- N.accept sock
        serveConnection Nothing client remaining received closed
        acceptLoop sock remaining received closed

    -- Greet each connection in a thread of its own as soon as it is
    -- accepted, and queue it, then serve the queued connections one at a
    -- time. Every connection accepted is closed once the responses run out.
    greetingEach reply sock remaining received closed = do
      queue <- newChan
      accepted <- newIORef []
      let acceptor = forever $ do
            (client, _) <- N.accept sock
            atomicModifyIORef' accepted (\clients -> (client : clients, ()))
            void $ forkIO $ handle ignore $ do
              greet reply client received closed
              writeChan queue client
          serveQueued = do
            pending <- readIORef remaining
            unless (null pending) $ do
              client <- readChan queue
              serveConnection Nothing client remaining received closed
              serveQueued
      withAsync acceptor (const serveQueued)
        `finally` (readIORef accepted >>= mapM_ N.close)

    ignore :: SomeException -> IO ()
    ignore = const (return ())

-- | Like 'withMockServer', but each connection gets its own responses: the
-- n-th connection the server accepts gets the n-th list. The server answers
-- each connection in a thread of its own from the moment it accepts it, so
-- it answers one connection while another is still open. It closes a
-- connection once its responses run out, and stops listening once every
-- list has its connection. 'mockRequests' gives the requests of every
-- connection, in the order they arrived.
withConcurrentMockServer :: [[MockResponse]] -> (MockServer -> IO a) -> IO a
withConcurrentMockServer = serveConcurrently Nothing

serveConcurrently :: Maybe B.ByteString -> [[MockResponse]] -> (MockServer -> IO a) -> IO a
serveConcurrently greeting scripts = withListener loopback $ \sock received closed -> do
  -- Closing the listening socket here refuses later connection attempts
  -- straight away; closing it again on the way out does nothing.
  let serveFrom [] = N.close sock
      serveFrom (responses:rest) = do
        (client, _) <- N.accept sock
        remaining <- newIORef responses
        concurrently_ (serveConnection greeting client remaining received closed) (serveFrom rest)
  serveFrom scripts

-- | Listen on the given address and serve it with the given function in the
-- background, killing it once the action is done. The function is given the
-- listening socket, the IORef in which it records each request it reads,
-- newest first, and the IORef in which it counts the connections the client
-- closed while it waited for a request.
withListener
  :: N.SockAddr -> (N.Socket -> IORef [B.ByteString] -> IORef Int -> IO ())
  -> (MockServer -> IO a) -> IO a
withListener address serveClients action = do
  ready <- newEmptyMVar
  received <- newIORef []
  closed <- newIORef 0
  bracket
    (forkIO (serve ready received closed))
    killThread
    (const $ do
      port <- takeMVar ready
      action MockServer
        { mockPort = port
        , mockRequests = reverse <$> readIORef received
        , mockClosed = readIORef closed
        })
  where
    serve ready received closed = bracket
      (newSocket address)
      N.close
      $ \sock -> do
        N.setSocketOption sock N.ReuseAddr 1
        N.bind sock address
        N.listen sock 10
        listeningPort sock >>= putMVar ready
        serveClients sock received closed
    -- A Unix socket has no port.
    listeningPort sock = case address of
      N.SockAddrUnix{} -> return 0
      _                -> N.socketPort sock

-- | An ephemeral port on 127.0.0.1.
loopback :: N.SockAddr
loopback = N.SockAddrInet 0 (N.tupleToHostAddress (127, 0, 0, 1))

-- | Whether 'withMockServerAt' can listen on the given address. It cannot on
-- an IPv6 address where IPv6 is disabled, for example.
canListenAt :: N.SockAddr -> IO Bool
canListenAt address = handle unavailable $ bracket
  (newSocket address)
  N.close
  (\sock -> N.bind sock address >> return True)
  where
    unavailable :: IOException -> IO Bool
    unavailable = const (return False)

-- | Run an action with a socket bound to the given address but not
-- listening, so that connection attempts there are refused, and nothing else
-- can listen there in the meantime. The action is given the port, a free one
-- if the address's port is 0.
withBoundSocket :: N.SockAddr -> (N.PortNumber -> IO a) -> IO a
withBoundSocket = bindNotListening False

-- | Like 'withBoundSocket', but the socket sets SO_REUSEADDR before it binds.
-- Then the bind succeeds where an earlier mock's connections at that address
-- and port are still in TIME_WAIT, but another socket that sets SO_REUSEADDR,
-- as the mocks do, can listen there in the meantime.
withBoundSocketReuseAddr :: N.SockAddr -> (N.PortNumber -> IO a) -> IO a
withBoundSocketReuseAddr = bindNotListening True

-- | Bind a socket to the given address without listening, setting
-- SO_REUSEADDR first if asked, and run the action with its port.
bindNotListening :: Bool -> N.SockAddr -> (N.PortNumber -> IO a) -> IO a
bindNotListening reuse address action = bracket (newSocket address) N.close $ \sock -> do
  when reuse $ N.setSocketOption sock N.ReuseAddr 1
  N.bind sock address
  N.socketPort sock >>= action

-- | Run an action with a socket listening on the given address that never
-- accepts a connection, and whose queue of connections waiting to be
-- accepted is full, so that connection attempts there get no answer. The
-- action is given the port, a free one if the address's port is 0, and an
-- action that closes the socket, after which connection attempts are refused.
-- Throws instead if a connection attempt gets an answer within 100 ms.
withFullListener :: N.SockAddr -> (N.PortNumber -> IO () -> IO a) -> IO a
withFullListener address action =
  bracket (newSocket address) N.close $ \sock ->
  bracket (newSocket address) N.close $ \first ->
  bracket (newSocket address) N.close $ \second -> do
    N.bind sock address
    -- Linux queues one connection more than the backlog, so two waiting
    -- connections fill a queue of backlog 1. Linux then ignores connection
    -- attempts until the queue has room. Older kernels take a connection on
    -- a queue of backlog 0 only with SYN cookies.
    N.listen sock 1
    listening <- N.getSocketName sock
    mapM_ (`N.connect` listening) [first, second]
    -- A system that queued more would answer this attempt. Fail then, rather
    -- than let a test check something else.
    answered <- bracket (newSocket address) N.close $ \probe ->
      timeout (100 * 1000) (N.connect probe listening)
    when (isJust answered) $ throwIO (userError "the listener's queue is not full")
    port <- N.socketPort sock
    action port (N.close sock)

-- | Run an action with the path of a Unix socket in the temporary directory,
-- at which nothing exists yet, so that a mock can listen there. Whatever is
-- at the path once the action is done, such as the mock's socket, is
-- removed.
withSocketPath :: (FilePath -> IO a) -> IO a
withSocketPath = bracket freshPath removePathForcibly
  where
    -- A temporary file's name is one no other file has. Remove the file, so
    -- that a socket can be bound in its place.
    freshPath = do
      tmp <- getTemporaryDirectory
      (path, file) <- openTempFile tmp "memcache-hs.sock"
      hClose file
      removeFile path
      return path

-- | A stream socket for the given address's family: TCP, or a Unix socket.
newSocket :: N.SockAddr -> IO N.Socket
newSocket address = N.socket (addressFamily address) N.Stream N.defaultProtocol

addressFamily :: N.SockAddr -> N.Family
addressFamily N.SockAddrInet6{} = N.AF_INET6
addressFamily N.SockAddrUnix{}  = N.AF_UNIX
addressFamily _                 = N.AF_INET

-- | Answer requests on one connection until the responses run out or the
-- script closes it, then close it. If there is a greeting, the first request
-- is answered with it, without using up a response.
serveConnection
  :: Maybe B.ByteString -> N.Socket -> IORef [MockResponse] -> IORef [B.ByteString] -> IORef Int
  -> IO ()
serveConnection greeting client remaining received closed =
  handle ignore (mapM_ (\reply -> greet reply client received closed) greeting >> clientHandler)
    `finally` N.close client
  where
    ignore :: SomeException -> IO ()
    ignore = const (return ())

    clientHandler = do
      pending <- readIORef remaining
      case pending of
        [] -> return ()
        (r:rest) -> do
          recordRequest client received closed
          writeIORef remaining rest
          continue <- respond r
          when continue clientHandler

    respond r = case r of
      Noop          -> return True
      MR bytes      -> N.sendAll client bytes >> return True
      DelayMS ms r' -> threadDelay (ms * 1000) >> respond r'
      CloseConnection -> return False
      Repeat bytes  -> forever (N.sendAll client bytes)
      -- The connection is closed here, so that the MVar is filled only once
      -- it is closed. Closing it again, as serveConnection does, does
      -- nothing.
      ReplyThenReset bytes reset -> do
        N.sendAll client bytes
        N.setSockOpt client N.Linger (N.StructLinger 1 0)
        N.close client
        void (tryPutMVar reset ())
        return False

-- | Read the first request on a connection, the client's version check, and
-- answer it with the reply.
greet :: B.ByteString -> N.Socket -> IORef [B.ByteString] -> IORef Int -> IO ()
greet reply client received closed = do
  recordRequest client received closed
  N.sendAll client reply

-- | Read the next request and record it, or count the connection as closed if
-- the client has closed it. Requests and closed connections can be recorded
-- from several threads at once.
recordRequest :: N.Socket -> IORef [B.ByteString] -> IORef Int -> IO ()
recordRequest client received closed = do
  request <- recvReq client
    `onException` atomicModifyIORef' closed (\n -> (n + 1, ()))
  atomicModifyIORef' received (\requests -> (request : requests, ()))

-- | Read one request: its command line and, for storage commands, the data
-- block that follows it.
recvReq :: N.Socket -> IO B.ByteString
recvReq s = do
  line <- recvLine s []
  -- Split on spaces only, as memcached does: a raw key can contain other
  -- whitespace bytes, such as 0xa0.
  case payloadLen (filter (not . B.null) (C.split ' ' line)) of
    Just n  -> do
      payload <- recvAll s n
      trailer <- recvLine s []
      return (line <> "\n" <> payload <> trailer <> "\n")
    Nothing -> return (line <> "\n")
  where
    -- Only these carry a data block; guessing by token position instead sees
    -- a payload in commands like @stats cachedump 1 100@ and then hangs.
    payloadLen ("ms":_:size:_)      = readMaybe (C.unpack size)
    payloadLen ("set":_:_:_:size:_) = readMaybe (C.unpack size)
    payloadLen _                    = Nothing

recvAll :: N.Socket -> Int -> IO B.ByteString
recvAll _ 0 = return B.empty
recvAll socket n = do
  chunk <- N.recv socket n
  if B.null chunk
    then throwIO eof
    else (chunk <>) <$> recvAll socket (n - B.length chunk)
  where eof = userError "mock server EOF"

-- | Read up to and excluding the next @\\n@, one byte at a time so that
-- nothing past the line is consumed.
recvLine :: N.Socket -> [B.ByteString] -> IO B.ByteString
recvLine socket acc = do
  chunk <- N.recv socket 1
  if B.null chunk then throwIO (userError "mock server EOF")
  else if chunk == "\n" then return (B.concat (reverse acc))
       else recvLine socket (chunk:acc)