diff --git a/Changelog.md b/Changelog.md --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,7 @@ +### 0.5.2.0 + +- Added handlers to extract data from lists of values in the Prometheus module. + ### 0.5.1.0 - Added gauge metrics type to Prometheus counters. diff --git a/NgxExport/Tools/Prometheus.hs b/NgxExport/Tools/Prometheus.hs --- a/NgxExport/Tools/Prometheus.hs +++ b/NgxExport/Tools/Prometheus.hs @@ -18,12 +18,15 @@ module NgxExport.Tools.Prometheus ( - -- *Exporters + -- * Exporters -- $exporters - -- *Utilities + -- * Utilities + -- *** Scaling functions scale ,scale1000 + -- *** Converting lists of values to counters + -- $convertingListsOfValuesToCounters ) where import NgxExport @@ -32,6 +35,7 @@ import Data.Map.Strict (Map) import qualified Data.Map.Strict as M import Data.ByteString (ByteString) +import qualified Data.ByteString.Char8 as C8 import qualified Data.ByteString.Lazy as L import qualified Data.ByteString.Lazy.Char8 as C8L import Data.IORef @@ -40,8 +44,14 @@ import qualified Data.Text.Encoding as T import Data.Aeson import Data.Maybe +import Data.Function +import Data.List +import Data.Char import Data.Word +import Data.Array.ST hiding (range) import Control.Arrow +import Control.Monad +import Control.Monad.ST import System.IO.Unsafe import GHC.Generics import Safe @@ -256,20 +266,20 @@ -- scaled with /scale1000/ and must be converted back. -- -- Handler /toPrometheusMetrics/ expects 4 fields: the name of the --- /counter set identifier/: in our example there is only one counter set --- /main/, predefined variables /cnt_collection/ and /cnt_histograms/ from --- /nginx-custom-counters-module/, and a list of additional counters: in our --- example there are two additional counters /cnt_uptime/ and --- /cnt_uptime_reload/ which are also defined in +-- /counter set identifier/ — in our example there is only one counter +-- set /main/, predefined variables /cnt_collection/ and /cnt_histograms/ from +-- /nginx-custom-counters-module/, and a list of additional counters — in +-- our example there are three additional counters /cnt_stub_status_active/, +-- /cnt_uptime/, and /cnt_uptime_reload/ which are also defined in -- /nginx-custom-counters-module/. -- -- To fulfill histogram description in Prometheus, the /sum/ value must be -- provided. Histogram sums are not supported in /nginx-custom-counters-module/, -- and therefore they must be declared in separate counters. In this example -- there are two histograms collecting request durations and the number of sent --- bytes: accordingly, there are two sum counters: /hst_request_time_sum/ and --- /hst_bytes_sent_sum/. As request durations may last milliseconds while being --- shown in seconds, they must be scaled with /scale1000/. +-- bytes, and accordingly, there are two sum counters: /hst_request_time_sum/ +-- and /hst_bytes_sent_sum/. As request durations may last milliseconds while +-- being shown in seconds, they must be scaled with /scale1000/. -- -- To further ensure histogram validity, it is important to have the last bucket -- in a histogram labeled as /\"+Inf\"/. This is achieved automatically when @@ -333,9 +343,9 @@ -- -- Run some requests and look at the metrics again. -- --- > $ for in in {1..20} ; do curl -D- 'http://localhost:8010/' & done +-- > $ for i in {1..20} ; do curl -D- 'http://localhost:8010/' & done -- > ... --- > $ for in in {1..30} ; do curl -D- 'http://localhost:8010/1' & done +-- > $ for i in {1..30} ; do curl -D- 'http://localhost:8010/1' & done -- > ... -- > $ curl 'http://127.0.0.1:8010/404' -- > ... @@ -523,4 +533,272 @@ in C8L.pack $ show $ scale 1000 v' ngxExportYY 'scale1000 + +-- $convertingListsOfValuesToCounters +-- +-- This module has limited support for extracting data from lists of values. +-- Normally, variables from Nginx upstream module such as /upstream_status/, +-- /upstream_response_time/ and others contain lists of values separated by +-- commas and semicolons. With handler __/statusLayout/__, numbers of /2xx/, +-- /3xx/, /4xx/ and /5xx/ responses from backends can be collected in a +-- comma-separated list. Handlers __/cumulativeValue/__ and +-- __/cumulativeFPValue/__ can be used to count cumulative integer and floating +-- point numbers from lists of values. +-- +-- Let's add checking upstream statuses and cumulative response times from all +-- servers in an upstream into the original file /nginx.conf/ from the previous +-- example. +-- +-- ==== File /nginx.conf/: checking upstream statuses and response times +-- @ +-- upstream backends { +-- server 127.0.0.1:8030 max_fails=0; +-- server 127.0.0.1:8040 max_fails=0; +-- } +-- @ +-- @ +-- server { +-- listen 8030; +-- server_name backend1; +-- +-- location \/ { +-- echo_sleep 0.5; +-- echo_status 404; +-- echo \"Backend1 Ok\"; +-- } +-- } +-- +-- server { +-- listen 8040; +-- server_name backend2; +-- +-- location \/ { +-- echo_status 504; +-- echo \"Backend2 Ok\"; +-- } +-- } +-- @ +-- +-- Here we added upstream /backends/ with two virtual servers that will play +-- the role of backends. One of them will wait for half a second and return +-- HTTP status /404/, while the other will return HTTP status /504/ immediately. +-- Both servers are tagged with /max_fails=0/ to prevent blacklisting them. +-- +-- We also have to add counters and mappings. +-- +-- @ +-- map $hs_upstream_status $inc_cnt_u_4xx { +-- default 0; +-- \'~^(?:(?:\\d+),){2}(?P\\\d+)\' $m_status; +-- } +-- +-- map $hs_upstream_status $inc_cnt_u_5xx { +-- default 0; +-- \'~^(?:(?:\\d+),){3}(?P\\\d+)\' $m_status; +-- } +-- +-- map_to_range_index $hs_u_response_time $u_response_time_bucket +-- 0.005 +-- 0.01 +-- 0.05 +-- 0.1 +-- 0.5 +-- 1.0 +-- 5.0 +-- 10.0 +-- 30.0 +-- 60.0; +-- @ +-- @ +-- haskell_run __/statusLayout/__ $hs_upstream_status $upstream_status; +-- counter $__/cnt_u_4xx/__ inc $inc_cnt_u_4xx; +-- counter $__/cnt_u_5xx/__ inc $inc_cnt_u_5xx; +-- +-- haskell_run ! $hs_u_response_times $upstream_response_time; +-- haskell_run __/cumulativeFPValue/__ $hs_u_response_time $hs_u_response_times; +-- +-- histogram $__/hst_u_response_time/__ 11 $u_response_time_bucket; +-- haskell_run scale1000 $hs_u_response_time_scaled $hs_u_response_time; +-- counter $__/hst_u_response_time_sum/__ inc $hs_u_response_time_scaled; +-- @ +-- +-- So many new variables require a bigger hash table to store them. +-- +-- @ +-- variables_hash_max_size 4096; +-- @ +-- +-- And finally, we have to update counters declarations in +-- /simpleService_prometheusConf/ and add location /\/backends/ in the main +-- server. +-- +-- @ +-- haskell_run___/service simpleService_prometheusConf/__ $hs_prometheus_conf +-- \'__/PrometheusConf/__ +-- { __/pcMetrics/__ = fromList +-- [(\"cnt_4xx\", \"Number of responses with 4xx status\") +-- ,(\"cnt_5xx\", \"Number of responses with 5xx status\") +-- ,(\"__/cnt_u_4xx/__\" +-- ,\"Number of responses from upstreams with 4xx status\") +-- ,(\"__/cnt_u_5xx/__\" +-- ,\"Number of responses from upstreams with 5xx status\") +-- ,(\"cnt_stub_status_active\", \"Active requests\") +-- ,(\"cnt_uptime\", \"Nginx master uptime\") +-- ,(\"cnt_uptime_reload\", \"Nginx master uptime after reload\") +-- ,(\"hst_request_time\", \"Request duration\") +-- ,(\"__/hst_u_response_time/__\" +-- ,\"Response time from all servers in a single upstream\") +-- ] +-- , __/pcGauges/__ = [\"cnt_stub_status_active\"] +-- , __/pcScale1000/__ = [\"hst_request_time_sum\" +-- ,\"__/hst_u_response_time_sum/__\" +-- ] +-- }\'; +-- @ +-- @ +-- location \/backends { +-- error_page 404 \@status404; +-- proxy_intercept_errors on; +-- proxy_pass http:\/\/backends; +-- } +-- +-- location \@status404 { +-- echo_sleep 0.2; +-- echo \"Caught 404\"; +-- } +-- @ +-- +-- We are going to additionally increase response time by /0.2/ seconds when a +-- backend server responds with HTTP status /404/. +-- +-- ==== A simple test +-- +-- After restart of Nginx. +-- +-- > $ for i in {1..20} ; do curl -D- 'http://localhost:8010/backends' & done +-- > ... +-- +-- > $ curl -s 'http://127.0.0.1:8020/metrics' +-- > # HELP cnt_4xx Number of responses with 4xx status +-- > # TYPE cnt_4xx counter +-- > cnt_4xx 11.0 +-- > # HELP cnt_5xx Number of responses with 5xx status +-- > # TYPE cnt_5xx counter +-- > cnt_5xx 9.0 +-- > # HELP cnt_stub_status_active Active requests +-- > # TYPE cnt_stub_status_active gauge +-- > cnt_stub_status_active 1.0 +-- > # HELP cnt_u_4xx Number of responses from upstreams with 4xx status +-- > # TYPE cnt_u_4xx counter +-- > cnt_u_4xx 11.0 +-- > # HELP cnt_u_5xx Number of responses from upstreams with 5xx status +-- > # TYPE cnt_u_5xx counter +-- > cnt_u_5xx 9.0 +-- > # HELP cnt_uptime Nginx master uptime +-- > # TYPE cnt_uptime counter +-- > cnt_uptime 63.0 +-- > # HELP cnt_uptime_reload Nginx master uptime after reload +-- > # TYPE cnt_uptime_reload counter +-- > cnt_uptime_reload 63.0 +-- > # HELP hst_bytes_sent +-- > # TYPE hst_bytes_sent histogram +-- > hst_bytes_sent_bucket{le="0"} 0 +-- > hst_bytes_sent_bucket{le="10"} 0 +-- > hst_bytes_sent_bucket{le="100"} 0 +-- > hst_bytes_sent_bucket{le="1000"} 20 +-- > hst_bytes_sent_bucket{le="10000"} 20 +-- > hst_bytes_sent_bucket{le="+Inf"} 20 +-- > hst_bytes_sent_count 20 +-- > hst_bytes_sent_sum 4032.0 +-- > # HELP hst_bytes_sent_err +-- > # TYPE hst_bytes_sent_err counter +-- > hst_bytes_sent_err 0.0 +-- > # HELP hst_request_time Request duration +-- > # TYPE hst_request_time histogram +-- > hst_request_time_bucket{le="0.005"} 9 +-- > hst_request_time_bucket{le="0.01"} 9 +-- > hst_request_time_bucket{le="0.05"} 9 +-- > hst_request_time_bucket{le="0.1"} 9 +-- > hst_request_time_bucket{le="0.5"} 9 +-- > hst_request_time_bucket{le="1.0"} 20 +-- > hst_request_time_bucket{le="5.0"} 20 +-- > hst_request_time_bucket{le="10.0"} 20 +-- > hst_request_time_bucket{le="30.0"} 20 +-- > hst_request_time_bucket{le="60.0"} 20 +-- > hst_request_time_bucket{le="+Inf"} 20 +-- > hst_request_time_count 20 +-- > hst_request_time_sum 7.721 +-- > # HELP hst_request_time_err +-- > # TYPE hst_request_time_err counter +-- > hst_request_time_err 0.0 +-- > # HELP hst_u_response_time Response time from all servers in a single upstream +-- > # TYPE hst_u_response_time histogram +-- > hst_u_response_time_bucket{le="0.005"} 9 +-- > hst_u_response_time_bucket{le="0.01"} 9 +-- > hst_u_response_time_bucket{le="0.05"} 9 +-- > hst_u_response_time_bucket{le="0.1"} 9 +-- > hst_u_response_time_bucket{le="0.5"} 13 +-- > hst_u_response_time_bucket{le="1.0"} 20 +-- > hst_u_response_time_bucket{le="5.0"} 20 +-- > hst_u_response_time_bucket{le="10.0"} 20 +-- > hst_u_response_time_bucket{le="30.0"} 20 +-- > hst_u_response_time_bucket{le="60.0"} 20 +-- > hst_u_response_time_bucket{le="+Inf"} 20 +-- > hst_u_response_time_count 20 +-- > hst_u_response_time_sum 5.519 +-- > # HELP hst_u_response_time_err +-- > # TYPE hst_u_response_time_err counter +-- > hst_u_response_time_err 0.0 +-- +-- Counters look good. Numbers of visiting backend servers are almost equal (11 +-- and 9), the sum of cumulative response times from backends is approximately 5 +-- seconds, while the sum of all requests durations is approximately 7 seconds +-- which corresponds to 11 visits to location /\@status404/ and the sleep time +-- /0.2/ seconds that was added there. Notice that upstream response times will +-- be updated on entering /any/ location in the main server as the histogram and +-- its sum counter were declared on the server level. To update the histogram on +-- entering location /\/backends/ only, it is possible to move the declarations +-- inside this location, however in this case there will be no updates in +-- location /\@status404/ as there is no way to declare the same histogram more +-- than once in a single counter set. Another solution would be putting line +-- +-- @ +-- set $u_response_time_bucket unavailable; +-- @ +-- +-- into all other locations of the main server (i.e. /\//, /\/1/, and /\/404/): +-- in this case only counter /hst_u_response_time_err/ will be updated when +-- entering these locations. + +extractValues :: ByteString -> [ByteString] +extractValues = filter ((&&) <$> not . C8.null <*> isDigit . C8.head) + . C8.splitWith ((&&) <$> not . isDigit <*> (/= '.')) + +statusLayout :: ByteString -> L.ByteString +statusLayout = C8L.pack . intercalate "," . map show . statuses + where statuses s = runST $ do + a <- newArray bs 0 :: ST s (STUArray s Int Int) + mapM_ (uncurry $ writeStatus a) $ toPairs s + getElems a + toPairs = map (subtract (ord '0') . ord . C8.head . head &&& length) + . groupBy ((==) `on` C8.head) + . sort + . extractValues + writeStatus a i = when (i >= lb && i <= ub) . writeArray a i + bs@(lb, ub) = (2, 5) + +ngxExportYY 'statusLayout + +cumulativeValue' :: (Num a, Read a) => ByteString -> a +cumulativeValue' = foldr ((+) . (read . C8.unpack)) 0 . extractValues + +cumulativeValue :: ByteString -> L.ByteString +cumulativeValue = C8L.pack . show . cumulativeValue' @Int + +ngxExportYY 'cumulativeValue + +cumulativeFPValue :: ByteString -> L.ByteString +cumulativeFPValue = C8L.pack . show . cumulativeValue' @Double + +ngxExportYY 'cumulativeFPValue diff --git a/ngx-export-tools-extra.cabal b/ngx-export-tools-extra.cabal --- a/ngx-export-tools-extra.cabal +++ b/ngx-export-tools-extra.cabal @@ -1,5 +1,5 @@ name: ngx-export-tools-extra -version: 0.5.1.0 +version: 0.5.2.0 synopsis: More extra tools for Nginx haskell module description: More extra tools for . @@ -43,6 +43,7 @@ , case-insensitive , containers , unordered-containers + , array , enclosed-exceptions , snap-core , snap-server