diff --git a/Changelog.md b/Changelog.md --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,10 @@ +### 0.5.3.0 + +- Handlers *makeSubrequestFull* and *makeSubrequestFullWithRead* no longer throw + errors on responses with HTTP statuses other than *2xx*. +- Implemented forwarding responses downstream to the client with function + *contentFromFullResponse* and content handler *fromFullResponse*. + ### 0.5.2.0 - Added handlers to extract data from lists of values in the Prometheus module. diff --git a/NgxExport/Tools/Prometheus.hs b/NgxExport/Tools/Prometheus.hs --- a/NgxExport/Tools/Prometheus.hs +++ b/NgxExport/Tools/Prometheus.hs @@ -136,13 +136,13 @@ -- sendfile on; -- -- map $status $inc_cnt_4xx { --- default 0; --- \'~^4(?:\\d){2}\' 1; +-- default 0; +-- \'~^4(?:\\d){2}\' 1; -- } -- -- map $status $inc_cnt_5xx { --- default 0; --- \'~^5(?:\\d){2}\' 1; +-- default 0; +-- \'~^5(?:\\d){2}\' 1; -- } -- -- map_to_range_index $hs_request_time $request_time_bucket diff --git a/NgxExport/Tools/Subrequest.hs b/NgxExport/Tools/Subrequest.hs --- a/NgxExport/Tools/Subrequest.hs +++ b/NgxExport/Tools/Subrequest.hs @@ -29,6 +29,10 @@ ,extractStatusFromFullResponse ,extractHeaderFromFullResponse ,extractBodyFromFullResponse + -- * Forwarding full response data to the client + -- $forwardingFullResponse + ,notForwardableResponseHeaders + ,contentFromFullResponse ) where import NgxExport @@ -44,7 +48,9 @@ import qualified Data.ByteString.Lazy.Char8 as C8L import qualified Data.Text.Encoding as T import qualified Data.Binary as Binary -import Data.CaseInsensitive (mk, original) +import Data.HashSet (HashSet) +import qualified Data.HashSet as HS +import Data.CaseInsensitive (FoldCase (foldCase), mk, original) import Data.Aeson import Data.Maybe import Control.Arrow @@ -219,10 +225,10 @@ return SubrequestConf {..} where maybeEmpty = fmap $ maybe "" T.encodeUtf8 -subrequest :: (Response L.ByteString -> L.ByteString) -> SubrequestConf -> - IO L.ByteString -subrequest fromResponse SubrequestConf {..} = do - req <- parseUrlThrow srUri +subrequest :: (Response L.ByteString -> L.ByteString) -> + (String -> IO Request) -> SubrequestConf -> IO L.ByteString +subrequest fromResponse parseUrlF SubrequestConf {..} = do + req <- parseUrlF srUri let req' = if B.null srMethod then req else req { method = srMethod } @@ -244,12 +250,12 @@ setTimeout _ = undefined subrequestBody :: SubrequestConf -> IO L.ByteString -subrequestBody = subrequest responseBody +subrequestBody = subrequest responseBody parseUrlThrow type FullResponse = (Int, [(ByteString, L.ByteString)], L.ByteString) subrequestFull :: SubrequestConf -> IO L.ByteString -subrequestFull = handleAll . subrequest buildResponse +subrequestFull = handleAll . subrequest buildResponse parseRequest where handleAll = handle $ \e -> return $ Binary.encode @FullResponse $ case fromException e of Just (HttpExceptionRequest _ c) -> @@ -473,7 +479,8 @@ -- The same as 'makeSubrequest' except it returns a binary encoded response data -- whose parts must be extracted by handlers made of -- 'extractStatusFromFullResponse', 'extractHeaderFromFullResponse', and --- 'extractBodyFromFullResponse'. Exported on the Nginx level by handler +-- 'extractBodyFromFullResponse'. It also does not throw errors when HTTP status +-- of the response is not /2xx/. Exported on the Nginx level by handler -- /makeSubrequestFull/. makeSubrequestFull :: ByteString -- ^ Subrequest configuration @@ -489,7 +496,8 @@ -- The same as 'makeSubrequestWithRead' except it returns a binary encoded -- response data whose parts must be extracted by handlers made of -- 'extractStatusFromFullResponse', 'extractHeaderFromFullResponse', and --- 'extractBodyFromFullResponse'. Exported on the Nginx level by handler +-- 'extractBodyFromFullResponse'. It also does not throw errors when HTTP status +-- of the response is not /2xx/. Exported on the Nginx level by handler -- /makeSubrequestFullWithRead/. makeSubrequestFullWithRead :: ByteString -- ^ Subrequest configuration @@ -546,4 +554,89 @@ (\(_, _, a) -> a) . (Binary.decode @FullResponse) . L.fromStrict ngxExportYY 'extractBodyFromFullResponse + +-- $forwardingFullResponse +-- +-- Data encoded in the full response can be translated to 'ContentHandlerResult' +-- and forwarded downstream to the client in directive /haskell_content/. +-- Handler __/fromFullResponse/__ performs such a translation. Not all response +-- headers are allowed being forwarded downstream, and thus the handler deletes +-- response headers with names listed in set 'notForwardableResponseHeaders' as +-- well as all headers with names starting with /X-Accel-/ before sending the +-- response to the client. The set of not forwardable response headers can be +-- customized in function 'contentFromFullResponse'. +-- +-- Let's forward responses in location /\/full/ when argument /proxy/ in the +-- client request's URI is equal to /yes/. +-- +-- ==== File /nginx.conf/: forward responses from location /\/full/ +-- @ +-- if ($arg_proxy = yes) { +-- haskell_content fromFullResponse $hs_subrequest; +-- break; +-- } +-- @ +-- +-- ==== A simple test +-- +-- > $ curl -D- 'http://localhost:8010/full/?a=Value&p=8020&proxy=yes' +-- > HTTP/1.1 200 OK +-- > Server: nginx/1.17.9 +-- > Date: Fri, 24 Jul 2020 13:14:33 GMT +-- > Content-Type: application/octet-stream +-- > Content-Length: 37 +-- > Connection: keep-alive +-- > Subrequest-Header: This is response from subrequest +-- > +-- > In backend, Custom-Header is 'Value' + +-- | Default set of not forwardable response headers. +-- +-- HTTP response headers that won't be forwarded to the client in handler +-- /fromFullResponse/. The set contains /Connection/, /Content-Length/, /Date/, +-- /Keep-Alive/, /Last-Modified/, /Server/, /Transfer-Encoding/, and +-- /Content-Type/ headers (the latter gets reset in the handler's result value). +-- If this set is not satisfactory, then handler /fromFullResponse/ must be +-- replaced with a custom handler based on 'contentFromFullResponse' with a +-- customized set of not forwardable response headers. +notForwardableResponseHeaders :: HashSet HeaderName +notForwardableResponseHeaders = HS.fromList $ + map mk ["Connection" + ,"Content-Length" + ,"Content-Type" + ,"Date" + ,"Keep-Alive" + ,"Last-Modified" + ,"Server" + ,"Transfer-Encoding" + ,"X-Pad" + ] + +-- | Translates encoded response to 'ContentHandlerResult'. +-- +-- The translated data can be forwarded to the client by a simple handler based +-- on this function in directive /haskell_content/. Handler /fromFullResponse/ +-- forwards the response to the client after deleting headers listed in set +-- 'notForwardableResponseHeaders' and headers with names starting with +-- /X-Accel-/. +contentFromFullResponse + :: HashSet HeaderName -- ^ Set of not forwardable response headers + -> Bool -- ^ Do not forward /X-Accel-.../ response headers + -> ByteString -- ^ Encoded HTTP response + -> ContentHandlerResult +contentFromFullResponse headersToDelete deleteXAccel v = + let (st, hs, b) = Binary.decode @FullResponse $ L.fromStrict v + hs' = map (first mk) hs + ct = L.toStrict $ fromMaybe "" $ lookup (mk "Content-Type") hs' + hs'' = filter (\(n, _) -> not $ + mk n `HS.member` headersToDelete || + deleteXAccel && + foldCase "X-Accel-" `B.isPrefixOf` foldCase n + ) hs + in (b, ct, st, map (second L.toStrict) hs'') + +fromFullResponse :: ByteString -> ContentHandlerResult +fromFullResponse = contentFromFullResponse notForwardableResponseHeaders True + +ngxExportHandler 'fromFullResponse diff --git a/ngx-export-tools-extra.cabal b/ngx-export-tools-extra.cabal --- a/ngx-export-tools-extra.cabal +++ b/ngx-export-tools-extra.cabal @@ -1,5 +1,5 @@ name: ngx-export-tools-extra -version: 0.5.2.1 +version: 0.5.3.0 synopsis: More extra tools for Nginx haskell module description: More extra tools for .