memcache-0.5.0.0: test/MockServer.hs
{-# LANGUAGE OverloadedStrings #-}
-- | Mock Memcached server - just enough for testing client.
--
-- Each mock listens on 127.0.0.1 unless given another address, and on an
-- ephemeral port of its own unless given a port, so tests can run several at
-- once and never clash with a real memcached on the default port. A mock can
-- also listen on a Unix socket, at a path from 'withSocketPath'.
--
-- The client asks for the server's version on every new connection before it
-- sends anything else. A mock made by 'withMockServer' and the functions like
-- it answers that request like any other, with its next response, so a test
-- can script the reply. A mock made by 'withMockMemcached' and the functions
-- like it reports itself as memcached of a given version instead.
module MockServer (
MockResponse(..), MockServer(..), withMockServer, withMockServerAt,
withConcurrentMockServer, withMockMemcached, withMockMemcachedAt,
withConcurrentMockMemcached, withBoundSocket, withBoundSocketReuseAddr,
withFullListener, withSocketPath, canListenAt, mockServerSpec
) where
import Control.Concurrent
import Control.Monad
import qualified Data.ByteString as B
import qualified Data.ByteString.Char8 as C
import Data.IORef
import Data.Maybe (isJust)
import Database.Memcache.Types (Authentication (NoAuth), ServerSpec (..))
import qualified Network.Socket as N
import qualified Network.Socket.ByteString as N
import System.Directory (getTemporaryDirectory, removeFile,
removePathForcibly)
import System.IO (hClose, openTempFile)
import System.Timeout (timeout)
import Text.Read (readMaybe)
import UnliftIO.Async (concurrently_, withAsync)
import UnliftIO.Exception (IOException, SomeException, bracket, finally,
handle, onException, throwIO)
-- | Actions the mock server can take to a request.
data MockResponse
-- | Reply with these bytes.
= MR B.ByteString
-- | Close the client connection instead of replying.
| CloseConnection
-- | Wait this many milliseconds, then take the given action.
| DelayMS Int MockResponse
-- | Read the request but send nothing back.
| Noop
-- | Send these bytes over and over until the client closes the
-- connection.
| Repeat B.ByteString
-- | Reply with these bytes, then reset the connection: close it with
-- @SO_LINGER@ set to 0, so that once the client has read the bytes, its
-- next read fails with @ECONNRESET@ rather than finding the connection
-- closed. TCP only. The 'MVar' is filled once the connection is closed.
| ReplyThenReset B.ByteString (MVar ())
-- | A running mock server.
data MockServer = MockServer
{ -- | The port the mock is listening on, or 0 on a Unix socket.
mockPort :: N.PortNumber
-- | Every request received so far, oldest first, including any data
-- block that followed the command line.
, mockRequests :: IO [B.ByteString]
-- | How many connections the client has closed while the mock was
-- waiting for its next request.
, mockClosed :: IO Int
}
-- | A server specification for connecting to the mock without authentication.
mockServerSpec :: MockServer -> ServerSpec
mockServerSpec server = ServerSpec "127.0.0.1" (show (mockPort server)) NoAuth
-- | Run an action with a mock Memcached server running in the background,
-- killing it once done. The server answers requests with the given responses
-- in order, one per request, across however many connections the client
-- makes; once they run out it closes the connection and stops listening.
withMockServer :: [MockResponse] -> (MockServer -> IO a) -> IO a
withMockServer = withMockServerAt loopback
-- | Like 'withMockServer', but listening on the given address instead of
-- 127.0.0.1. A port of 0 picks a free one.
withMockServerAt :: N.SockAddr -> [MockResponse] -> (MockServer -> IO a) -> IO a
withMockServerAt = serveInTurn Nothing
-- | Like 'withMockServer', but the mock reports itself as memcached of the
-- given version, such as @"1.6.24"@: it answers the first request on each
-- connection, the client's version check, with @VERSION@ and that version,
-- without using up a response. That request is still recorded.
--
-- The mock answers each connection's version check as soon as it accepts the
-- connection, even while it is serving another one, as the client can't use
-- a connection until then. It still serves the responses to one connection
-- at a time, in the order it accepted them, and once the responses run out,
-- it closes every connection and stops listening.
withMockMemcached :: B.ByteString -> [MockResponse] -> (MockServer -> IO a) -> IO a
withMockMemcached version = withMockMemcachedAt version loopback
-- | Like 'withMockServerAt', but reporting the given version, as
-- 'withMockMemcached' does.
withMockMemcachedAt :: B.ByteString -> N.SockAddr -> [MockResponse] -> (MockServer -> IO a) -> IO a
withMockMemcachedAt version = serveInTurn (Just (versionReply version))
-- | Like 'withConcurrentMockServer', but reporting the given version, as
-- 'withMockMemcached' does.
withConcurrentMockMemcached :: B.ByteString -> [[MockResponse]] -> (MockServer -> IO a) -> IO a
withConcurrentMockMemcached version = serveConcurrently (Just (versionReply version))
versionReply :: B.ByteString -> B.ByteString
versionReply version = "VERSION " <> version <> "\r\n"
-- | Serve one connection at a time on the given address, answering the first
-- request on each connection with the greeting, if there is one, and then
-- each request with the next of the responses.
serveInTurn
:: Maybe B.ByteString -> N.SockAddr -> [MockResponse] -> (MockServer -> IO a) -> IO a
serveInTurn greeting address responses = withListener address $ \sock received closed -> do
remaining <- newIORef responses
-- Once the responses run out the listening socket closes, so later
-- connection attempts are refused straight away.
case greeting of
Nothing -> acceptLoop sock remaining received closed
Just reply -> greetingEach reply sock remaining received closed
where
-- Serve one connection at a time.
acceptLoop sock remaining received closed = do
pending <- readIORef remaining
unless (null pending) $ do
(client, _) <- N.accept sock
serveConnection Nothing client remaining received closed
acceptLoop sock remaining received closed
-- Greet each connection in a thread of its own as soon as it is
-- accepted, and queue it, then serve the queued connections one at a
-- time. Every connection accepted is closed once the responses run out.
greetingEach reply sock remaining received closed = do
queue <- newChan
accepted <- newIORef []
let acceptor = forever $ do
(client, _) <- N.accept sock
atomicModifyIORef' accepted (\clients -> (client : clients, ()))
void $ forkIO $ handle ignore $ do
greet reply client received closed
writeChan queue client
serveQueued = do
pending <- readIORef remaining
unless (null pending) $ do
client <- readChan queue
serveConnection Nothing client remaining received closed
serveQueued
withAsync acceptor (const serveQueued)
`finally` (readIORef accepted >>= mapM_ N.close)
ignore :: SomeException -> IO ()
ignore = const (return ())
-- | Like 'withMockServer', but each connection gets its own responses: the
-- n-th connection the server accepts gets the n-th list. The server answers
-- each connection in a thread of its own from the moment it accepts it, so
-- it answers one connection while another is still open. It closes a
-- connection once its responses run out, and stops listening once every
-- list has its connection. 'mockRequests' gives the requests of every
-- connection, in the order they arrived.
withConcurrentMockServer :: [[MockResponse]] -> (MockServer -> IO a) -> IO a
withConcurrentMockServer = serveConcurrently Nothing
serveConcurrently :: Maybe B.ByteString -> [[MockResponse]] -> (MockServer -> IO a) -> IO a
serveConcurrently greeting scripts = withListener loopback $ \sock received closed -> do
-- Closing the listening socket here refuses later connection attempts
-- straight away; closing it again on the way out does nothing.
let serveFrom [] = N.close sock
serveFrom (responses:rest) = do
(client, _) <- N.accept sock
remaining <- newIORef responses
concurrently_ (serveConnection greeting client remaining received closed) (serveFrom rest)
serveFrom scripts
-- | Listen on the given address and serve it with the given function in the
-- background, killing it once the action is done. The function is given the
-- listening socket, the IORef in which it records each request it reads,
-- newest first, and the IORef in which it counts the connections the client
-- closed while it waited for a request.
withListener
:: N.SockAddr -> (N.Socket -> IORef [B.ByteString] -> IORef Int -> IO ())
-> (MockServer -> IO a) -> IO a
withListener address serveClients action = do
ready <- newEmptyMVar
received <- newIORef []
closed <- newIORef 0
bracket
(forkIO (serve ready received closed))
killThread
(const $ do
port <- takeMVar ready
action MockServer
{ mockPort = port
, mockRequests = reverse <$> readIORef received
, mockClosed = readIORef closed
})
where
serve ready received closed = bracket
(newSocket address)
N.close
$ \sock -> do
N.setSocketOption sock N.ReuseAddr 1
N.bind sock address
N.listen sock 10
listeningPort sock >>= putMVar ready
serveClients sock received closed
-- A Unix socket has no port.
listeningPort sock = case address of
N.SockAddrUnix{} -> return 0
_ -> N.socketPort sock
-- | An ephemeral port on 127.0.0.1.
loopback :: N.SockAddr
loopback = N.SockAddrInet 0 (N.tupleToHostAddress (127, 0, 0, 1))
-- | Whether 'withMockServerAt' can listen on the given address. It cannot on
-- an IPv6 address where IPv6 is disabled, for example.
canListenAt :: N.SockAddr -> IO Bool
canListenAt address = handle unavailable $ bracket
(newSocket address)
N.close
(\sock -> N.bind sock address >> return True)
where
unavailable :: IOException -> IO Bool
unavailable = const (return False)
-- | Run an action with a socket bound to the given address but not
-- listening, so that connection attempts there are refused, and nothing else
-- can listen there in the meantime. The action is given the port, a free one
-- if the address's port is 0.
withBoundSocket :: N.SockAddr -> (N.PortNumber -> IO a) -> IO a
withBoundSocket = bindNotListening False
-- | Like 'withBoundSocket', but the socket sets SO_REUSEADDR before it binds.
-- Then the bind succeeds where an earlier mock's connections at that address
-- and port are still in TIME_WAIT, but another socket that sets SO_REUSEADDR,
-- as the mocks do, can listen there in the meantime.
withBoundSocketReuseAddr :: N.SockAddr -> (N.PortNumber -> IO a) -> IO a
withBoundSocketReuseAddr = bindNotListening True
-- | Bind a socket to the given address without listening, setting
-- SO_REUSEADDR first if asked, and run the action with its port.
bindNotListening :: Bool -> N.SockAddr -> (N.PortNumber -> IO a) -> IO a
bindNotListening reuse address action = bracket (newSocket address) N.close $ \sock -> do
when reuse $ N.setSocketOption sock N.ReuseAddr 1
N.bind sock address
N.socketPort sock >>= action
-- | Run an action with a socket listening on the given address that never
-- accepts a connection, and whose queue of connections waiting to be
-- accepted is full, so that connection attempts there get no answer. The
-- action is given the port, a free one if the address's port is 0, and an
-- action that closes the socket, after which connection attempts are refused.
-- Throws instead if a connection attempt gets an answer within 100 ms.
withFullListener :: N.SockAddr -> (N.PortNumber -> IO () -> IO a) -> IO a
withFullListener address action =
bracket (newSocket address) N.close $ \sock ->
bracket (newSocket address) N.close $ \first ->
bracket (newSocket address) N.close $ \second -> do
N.bind sock address
-- Linux queues one connection more than the backlog, so two waiting
-- connections fill a queue of backlog 1. Linux then ignores connection
-- attempts until the queue has room. Older kernels take a connection on
-- a queue of backlog 0 only with SYN cookies.
N.listen sock 1
listening <- N.getSocketName sock
mapM_ (`N.connect` listening) [first, second]
-- A system that queued more would answer this attempt. Fail then, rather
-- than let a test check something else.
answered <- bracket (newSocket address) N.close $ \probe ->
timeout (100 * 1000) (N.connect probe listening)
when (isJust answered) $ throwIO (userError "the listener's queue is not full")
port <- N.socketPort sock
action port (N.close sock)
-- | Run an action with the path of a Unix socket in the temporary directory,
-- at which nothing exists yet, so that a mock can listen there. Whatever is
-- at the path once the action is done, such as the mock's socket, is
-- removed.
withSocketPath :: (FilePath -> IO a) -> IO a
withSocketPath = bracket freshPath removePathForcibly
where
-- A temporary file's name is one no other file has. Remove the file, so
-- that a socket can be bound in its place.
freshPath = do
tmp <- getTemporaryDirectory
(path, file) <- openTempFile tmp "memcache-hs.sock"
hClose file
removeFile path
return path
-- | A stream socket for the given address's family: TCP, or a Unix socket.
newSocket :: N.SockAddr -> IO N.Socket
newSocket address = N.socket (addressFamily address) N.Stream N.defaultProtocol
addressFamily :: N.SockAddr -> N.Family
addressFamily N.SockAddrInet6{} = N.AF_INET6
addressFamily N.SockAddrUnix{} = N.AF_UNIX
addressFamily _ = N.AF_INET
-- | Answer requests on one connection until the responses run out or the
-- script closes it, then close it. If there is a greeting, the first request
-- is answered with it, without using up a response.
serveConnection
:: Maybe B.ByteString -> N.Socket -> IORef [MockResponse] -> IORef [B.ByteString] -> IORef Int
-> IO ()
serveConnection greeting client remaining received closed =
handle ignore (mapM_ (\reply -> greet reply client received closed) greeting >> clientHandler)
`finally` N.close client
where
ignore :: SomeException -> IO ()
ignore = const (return ())
clientHandler = do
pending <- readIORef remaining
case pending of
[] -> return ()
(r:rest) -> do
recordRequest client received closed
writeIORef remaining rest
continue <- respond r
when continue clientHandler
respond r = case r of
Noop -> return True
MR bytes -> N.sendAll client bytes >> return True
DelayMS ms r' -> threadDelay (ms * 1000) >> respond r'
CloseConnection -> return False
Repeat bytes -> forever (N.sendAll client bytes)
-- The connection is closed here, so that the MVar is filled only once
-- it is closed. Closing it again, as serveConnection does, does
-- nothing.
ReplyThenReset bytes reset -> do
N.sendAll client bytes
N.setSockOpt client N.Linger (N.StructLinger 1 0)
N.close client
void (tryPutMVar reset ())
return False
-- | Read the first request on a connection, the client's version check, and
-- answer it with the reply.
greet :: B.ByteString -> N.Socket -> IORef [B.ByteString] -> IORef Int -> IO ()
greet reply client received closed = do
recordRequest client received closed
N.sendAll client reply
-- | Read the next request and record it, or count the connection as closed if
-- the client has closed it. Requests and closed connections can be recorded
-- from several threads at once.
recordRequest :: N.Socket -> IORef [B.ByteString] -> IORef Int -> IO ()
recordRequest client received closed = do
request <- recvReq client
`onException` atomicModifyIORef' closed (\n -> (n + 1, ()))
atomicModifyIORef' received (\requests -> (request : requests, ()))
-- | Read one request: its command line and, for storage commands, the data
-- block that follows it.
recvReq :: N.Socket -> IO B.ByteString
recvReq s = do
line <- recvLine s []
-- Split on spaces only, as memcached does: a raw key can contain other
-- whitespace bytes, such as 0xa0.
case payloadLen (filter (not . B.null) (C.split ' ' line)) of
Just n -> do
payload <- recvAll s n
trailer <- recvLine s []
return (line <> "\n" <> payload <> trailer <> "\n")
Nothing -> return (line <> "\n")
where
-- Only these carry a data block; guessing by token position instead sees
-- a payload in commands like @stats cachedump 1 100@ and then hangs.
payloadLen ("ms":_:size:_) = readMaybe (C.unpack size)
payloadLen ("set":_:_:_:size:_) = readMaybe (C.unpack size)
payloadLen _ = Nothing
recvAll :: N.Socket -> Int -> IO B.ByteString
recvAll _ 0 = return B.empty
recvAll socket n = do
chunk <- N.recv socket n
if B.null chunk
then throwIO eof
else (chunk <>) <$> recvAll socket (n - B.length chunk)
where eof = userError "mock server EOF"
-- | Read up to and excluding the next @\\n@, one byte at a time so that
-- nothing past the line is consumed.
recvLine :: N.Socket -> [B.ByteString] -> IO B.ByteString
recvLine socket acc = do
chunk <- N.recv socket 1
if B.null chunk then throwIO (userError "mock server EOF")
else if chunk == "\n" then return (B.concat (reverse acc))
else recvLine socket (chunk:acc)