diff --git a/Changelog.md b/Changelog.md --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,9 @@ +### 0.5.0.0 + +- Added module *NgxExport.Tools.Prometheus* to convert custom counters from + [*nginx-custom-counters-module*](https://github.com/lyokha/nginx-custom-counters-module) + to Prometheus metrics. + ### 0.4.1.0 - Handler *renderEDETemplateFromFreeValue* was moved from the example to the diff --git a/NgxExport/Tools/Prometheus.hs b/NgxExport/Tools/Prometheus.hs new file mode 100644 --- /dev/null +++ b/NgxExport/Tools/Prometheus.hs @@ -0,0 +1,506 @@ +{-# LANGUAGE TemplateHaskell, DeriveGeneric, RecordWildCards #-} +{-# LANGUAGE TypeApplications, TupleSections, OverloadedStrings #-} + +----------------------------------------------------------------------------- +-- | +-- Module : NgxExport.Tools.Prometheus +-- Copyright : (c) Alexey Radkov 2020 +-- License : BSD-style +-- +-- Maintainer : alexey.radkov@gmail.com +-- Stability : experimental +-- Portability : non-portable (requires Template Haskell) +-- +-- Prometheus metrics from the more extra tools collection for +-- . +-- +----------------------------------------------------------------------------- + + +module NgxExport.Tools.Prometheus ( + -- *Exporters + -- $exporters + + -- *Utilities + scale + ,scale1000 + ) where + +import NgxExport +import NgxExport.Tools + +import Data.Map.Strict (Map) +import qualified Data.Map.Strict as M +import Data.ByteString (ByteString) +import qualified Data.ByteString.Lazy as L +import qualified Data.ByteString.Lazy.Char8 as C8L +import Data.IORef +import Data.Text (Text) +import qualified Data.Text as T +import qualified Data.Text.Encoding as T +import Data.Aeson +import Data.Maybe +import Data.Word +import Control.Arrow +import System.IO.Unsafe +import GHC.Generics +import Safe + +type ServerName = Text +type MetricsName = Text +type MetricsHelp = Text +type MetricsLabel = Text +type CounterValue = Word64 +type MetricsData = Map MetricsName CounterValue +type HistogramData = Map MetricsName (MetricsLabel, Double) +type MetricsToLabelMap = Map MetricsName MetricsLabel + +data PrometheusConf = + PrometheusConf { pcMetrics :: Map MetricsName MetricsHelp + , pcScale1000 :: [MetricsName] + } deriving Read + +data HistogramLayout = + HistogramLayout { range :: MetricsToLabelMap + , cnt :: (MetricsName, MetricsLabel) + , err :: (MetricsName, MetricsLabel) + } deriving Generic + +instance FromJSON HistogramLayout + +type AllCounters = Map ServerName MetricsData +type AllHistogramsLayout = Map ServerName (Map MetricsName HistogramLayout) +type AllOtherCounters = MetricsData + +type AllMetrtics = + (ServerName, AllCounters, AllHistogramsLayout, AllOtherCounters) + +data MetricsType = Counter Double + | Histogram HistogramData + +type PrometheusMetrics = Map MetricsName (MetricsHelp, MetricsType) + +conf :: IORef (Maybe PrometheusConf) +conf = unsafePerformIO $ newIORef Nothing +{-# NOINLINE conf #-} + +-- $exporters +-- +-- This module is aimed to convert custom counters from +-- [nginx-custom-counters-module](https://github.com/lyokha/nginx-custom-counters-module) +-- to Prometheus metrics. For this, it exposes three exporters: +-- __/prometheusConf/__ which is an 'ignitionService' in terms of module +-- "NgxExport.Tools", __/toPrometheusMetrics/__ to convert /custom counters/ to +-- Prometheus metrics, and __/scale1000/__: a small utility to convert small +-- floating point numbers to integers by multiplying them by /1000/ (this fits +-- well for dealing with request durations, for instance). +-- +-- The module makes use of a few custom data types which are not exported while +-- still needed when writing Nginx configurations. In the following example they +-- are used in configurations of /simpleService_prometheusConf/ and +-- /toPrometheusMetrics/. +-- +-- ==== File /test_tools_extra_prometheus.hs/ +-- @ +-- module TestToolsExtraPrometheus where +-- +-- import NgxExport.Tools.Prometheus () +-- @ +-- +-- The file does not contain any significant declarations as we are going to use +-- only the exporters. +-- +-- ==== File /nginx.conf/ +-- @ +-- user nobody; +-- worker_processes 2; +-- +-- events { +-- worker_connections 1024; +-- } +-- +-- http { +-- default_type application\/octet-stream; +-- sendfile on; +-- +-- map $status $inc_cnt_4xx { +-- default 0; +-- \'~^4(?:\\d){2}\' 1; +-- } +-- +-- map $status $inc_cnt_5xx { +-- default 0; +-- \'~^5(?:\\d){2}\' 1; +-- } +-- +-- map_to_range_index $hs_request_time $request_time_bucket +-- 0.005 +-- 0.01 +-- 0.05 +-- 0.1 +-- 0.5 +-- 1.0 +-- 5.0 +-- 10.0 +-- 30.0 +-- 60.0; +-- +-- map_to_range_index $hs_bytes_sent $bytes_sent_bucket +-- 0 +-- 10 +-- 100 +-- 1000 +-- 10000; +-- +-- haskell load \/var\/lib\/nginx\/test_tools_extra_prometheus.so; +-- +-- haskell_run_service __/simpleService_prometheusConf/__ $hs_prometheus_conf +-- \'__/PrometheusConf/__ +-- { __/pcMetrics/__ = fromList +-- [(\"cnt_4xx\", \"Number of responses with 4xx status\") +-- ,(\"cnt_5xx\", \"Number of responses with 5xx status\") +-- ,(\"cnt_uptime\", \"Nginx master uptime\") +-- ,(\"cnt_uptime_reload\", \"Nginx master uptime after reload\") +-- ,(\"hst_request_time\", \"Request duration\") +-- ] +-- , __/pcScale1000/__ = [\"hst_request_time_sum\"] +-- }'; +-- +-- haskell_var_empty_on_error $hs_prom_metrics; +-- +-- counters_survive_reload on; +-- +-- server { +-- listen 8010; +-- server_name __/main/__; +-- error_log \/tmp\/nginx-test-haskell-error.log; +-- access_log \/tmp\/nginx-test-haskell-access.log; +-- +-- counter $cnt_4xx inc $inc_cnt_4xx; +-- counter $cnt_5xx inc $inc_cnt_5xx; +-- +-- \# cache $request_time and $bytes_sent +-- haskell_run ! $hs_request_time $request_time; +-- haskell_run ! $hs_bytes_sent $bytes_sent; +-- +-- histogram $hst_request_time 11 $request_time_bucket; +-- haskell_run __/scale1000/__ $hs_request_time_scaled $hs_request_time; +-- counter $hst_request_time_sum inc $hs_request_time_scaled; +-- +-- histogram $hst_bytes_sent 6 $bytes_sent_bucket; +-- counter $hst_bytes_sent_sum inc $hs_bytes_sent; +-- +-- location \/ { +-- echo_sleep 0.5; +-- echo Ok; +-- } +-- +-- location \/1 { +-- echo_sleep 1.0; +-- echo Ok; +-- } +-- +-- location \/404 { +-- return 404; +-- } +-- } +-- +-- server { +-- listen 8020; +-- server_name stats; +-- +-- location \/ { +-- haskell_run __/toPrometheusMetrics/__ $hs_prom_metrics +-- '[\"__/main/__\" +-- ,__/$cnt_collection/__ +-- ,__/$cnt_histograms/__ +-- ,{\"cnt_uptime\": $cnt_uptime +-- ,\"cnt_uptime_reload\": $cnt_uptime_reload +-- } +-- ]'; +-- +-- if ($hs_prom_metrics = \'\') { +-- return 503; +-- } +-- +-- echo -n $hs_prom_metrics; +-- } +-- +-- location \/counters { +-- echo $cnt_collection; +-- } +-- +-- location \/histograms { +-- echo $cnt_histograms; +-- } +-- +-- location \/uptime { +-- echo "$cnt_uptime ($cnt_uptime_reload)"; +-- } +-- } +-- } +-- @ +-- +-- Type /PrometheusConf/ contains fields /pcMetrics/ and /pcScale1000/. The +-- latter is a map from metrics names to help messages: this can be used to +-- bind small descriptions to the metrics as /nginx-custom-counters-module/ +-- does not provide such functionality. Setting descriptions to counters is +-- optional. Field /pcScale1000/ contains a list of counters that were scaled +-- with /scale1000/ and must be converted back. +-- +-- Handler /toPrometheusMetrics/ expects 4 fields: the name of the +-- /counter set id/: in our example there is only one counter set /main/, +-- predefined variables /cnt_collection/ and /cnt_histograms/ from +-- /nginx-custom-counters-module/, and a list of additional counters: in our +-- example there are two additional counters /cnt_uptime/ and +-- /cnt_uptime_reload/ which are also defined in +-- /nginx-custom-counters-module/. +-- +-- To fulfill histogram description in Prometheus, the /sum/ value must be +-- provided. Histogram sums are not supported in /nginx-custom-counters-module/, +-- and therefore they must be declared in separate counters. In this example +-- there are two histograms collecting request durations and sent bytes: +-- accordingly, there are two sum counters: /hst_request_time_sum/ and +-- /hst_bytes_sent_sum/. As request durations may last milliseconds while being +-- shown in seconds, they must be scaled with /scale1000/. +-- +-- To further ensure histogram validity, it is important to have the last bucket +-- in a histogram labeled as /\"+Inf\"/. This is achieved automatically when +-- the number of range boundaries in directive /map_to_range_index/ is less by +-- one than the number in the corresponding histogram declaration: in this +-- example, the map for /request_time_bucket/ has 10 range boundaries while +-- histogram /hst_request_time/ has 11 buckets, the map for /bytes_sent_bucket/ +-- has 5 range boundaries while histogram /hst_bytes_sent/ has 6 buckets. +-- +-- ==== A simple test +-- +-- Let's look at the metrics right after starting Nginx. +-- +-- > $ curl -s 'http://localhost:8020/' +-- > # HELP cnt_4xx Number of responses with 4xx status +-- > # TYPE cnt_4xx counter +-- > cnt_4xx 0.0 +-- > # HELP cnt_5xx Number of responses with 5xx status +-- > # TYPE cnt_5xx counter +-- > cnt_5xx 0.0 +-- > # HELP cnt_uptime Nginx master uptime +-- > # TYPE cnt_uptime counter +-- > cnt_uptime 8.0 +-- > # HELP cnt_uptime_reload Nginx master uptime after reload +-- > # TYPE cnt_uptime_reload counter +-- > cnt_uptime_reload 8.0 +-- > # HELP hst_bytes_sent +-- > # TYPE hst_bytes_sent histogram +-- > hst_bytes_sent_bucket{le="0"} 0 +-- > hst_bytes_sent_bucket{le="10"} 0 +-- > hst_bytes_sent_bucket{le="100"} 0 +-- > hst_bytes_sent_bucket{le="1000"} 0 +-- > hst_bytes_sent_bucket{le="10000"} 0 +-- > hst_bytes_sent_bucket{le="+Inf"} 0 +-- > hst_bytes_sent_count 0 +-- > hst_bytes_sent_sum 0.0 +-- > # HELP hst_bytes_sent_err +-- > # TYPE hst_bytes_sent_err counter +-- > hst_bytes_sent_err 0.0 +-- > # HELP hst_request_time Request duration +-- > # TYPE hst_request_time histogram +-- > hst_request_time_bucket{le="0.005"} 0 +-- > hst_request_time_bucket{le="0.01"} 0 +-- > hst_request_time_bucket{le="0.05"} 0 +-- > hst_request_time_bucket{le="0.1"} 0 +-- > hst_request_time_bucket{le="0.5"} 0 +-- > hst_request_time_bucket{le="1.0"} 0 +-- > hst_request_time_bucket{le="5.0"} 0 +-- > hst_request_time_bucket{le="10.0"} 0 +-- > hst_request_time_bucket{le="30.0"} 0 +-- > hst_request_time_bucket{le="60.0"} 0 +-- > hst_request_time_bucket{le="+Inf"} 0 +-- > hst_request_time_count 0 +-- > hst_request_time_sum 0.0 +-- > # HELP hst_request_time_err +-- > # TYPE hst_request_time_err counter +-- > hst_request_time_err 0.0 +-- +-- Run some requests and look at the metrics again. +-- +-- > $ for in in {1..20} ; do curl -D- 'http://localhost:8010/' & done +-- > ... +-- > $ for in in {1..30} ; do curl -D- 'http://localhost:8010/1' & done +-- > ... +-- > $ curl 'http://127.0.0.1:8010/404' +-- > ... +-- +-- > $ curl -s 'http://localhost:8020/' +-- > # HELP cnt_4xx Number of responses with 4xx status +-- > # TYPE cnt_4xx counter +-- > cnt_4xx 1.0 +-- > # HELP cnt_5xx Number of responses with 5xx status +-- > # TYPE cnt_5xx counter +-- > cnt_5xx 0.0 +-- > # HELP cnt_uptime Nginx master uptime +-- > # TYPE cnt_uptime counter +-- > cnt_uptime 371.0 +-- > # HELP cnt_uptime_reload Nginx master uptime after reload +-- > # TYPE cnt_uptime_reload counter +-- > cnt_uptime_reload 371.0 +-- > # HELP hst_bytes_sent +-- > # TYPE hst_bytes_sent histogram +-- > hst_bytes_sent_bucket{le="0"} 0 +-- > hst_bytes_sent_bucket{le="10"} 0 +-- > hst_bytes_sent_bucket{le="100"} 0 +-- > hst_bytes_sent_bucket{le="1000"} 51 +-- > hst_bytes_sent_bucket{le="10000"} 51 +-- > hst_bytes_sent_bucket{le="+Inf"} 51 +-- > hst_bytes_sent_count 51 +-- > hst_bytes_sent_sum 9458.0 +-- > # HELP hst_bytes_sent_err +-- > # TYPE hst_bytes_sent_err counter +-- > hst_bytes_sent_err 0.0 +-- > # HELP hst_request_time Request duration +-- > # TYPE hst_request_time histogram +-- > hst_request_time_bucket{le="0.005"} 1 +-- > hst_request_time_bucket{le="0.01"} 1 +-- > hst_request_time_bucket{le="0.05"} 1 +-- > hst_request_time_bucket{le="0.1"} 1 +-- > hst_request_time_bucket{le="0.5"} 13 +-- > hst_request_time_bucket{le="1.0"} 44 +-- > hst_request_time_bucket{le="5.0"} 51 +-- > hst_request_time_bucket{le="10.0"} 51 +-- > hst_request_time_bucket{le="30.0"} 51 +-- > hst_request_time_bucket{le="60.0"} 51 +-- > hst_request_time_bucket{le="+Inf"} 51 +-- > hst_request_time_count 51 +-- > hst_request_time_sum 40.006 +-- > # HELP hst_request_time_err +-- > # TYPE hst_request_time_err counter +-- > hst_request_time_err 0.0 + +prometheusConf :: PrometheusConf -> Bool -> IO L.ByteString +prometheusConf = ignitionService $ \a -> do + atomicWriteIORef conf $ Just a + return "" + +ngxExportSimpleServiceTyped + 'prometheusConf ''PrometheusConf SingleShotService + +toPrometheusMetrics' :: PrometheusConf -> AllMetrtics -> PrometheusMetrics +toPrometheusMetrics' PrometheusConf {..} (srv, cnts, hs, ocnts) = + let toValues = M.mapWithKey + (\k v -> (if k `elem` pcScale1000 + then (/ 1000) + else id + ) $ fromIntegral v + ) + cnts' = maybe M.empty toValues $ M.lookup srv cnts + hs' = M.lookup srv hs + (cntsH, cntsC) = + if maybe True M.null hs' + then (M.empty, M.map Counter cnts') + else let hs'' = fromJust hs' + rs = M.keys &&& M.foldr labeledRange M.empty $ hs'' + cntsH' = M.filterWithKey (hCounter rs) cnts' + cntsC' = cnts' `M.difference` cntsH' + cntsH'' = M.mapWithKey + (\k -> toHistogram cntsH' k . range) hs'' + in (M.map Histogram cntsH'', M.map Counter cntsC') + cntsA = cntsH `M.union` cntsC `M.union` M.map Counter (toValues ocnts) + in M.mapWithKey (\k v -> case M.lookup k pcMetrics of + Nothing -> ("", v) + Just h -> (h, v) + ) cntsA + where labeledRange = M.union . M.filter (not . T.null) . range + hCounter (ks, ts) k = const $ + k `M.member` ts || + headDef False + (dropWhile not $ + foldr (\v -> + let v1 = v `T.append` "_sum" + v2 = v `T.append` "_cnt" + in ((k == v1 || k == v2) :) + ) [] ks + ) + toHistogram cs hk rs = + let ranges = M.mapWithKey + (\k l -> case M.lookup k cs of + Just v -> (l, v) + Nothing -> (l, 0.0) + ) rs + sums = let v1 = hk `T.append` "_sum" + v2 = hk `T.append` "_cnt" + withZeroLabel = return . ("",) + in M.fromList $ + map (second fromJust) $ filter (isJust . snd) + [(v1, M.lookup v1 cs >>= withZeroLabel) + ,(v2, M.lookup v2 cs >>= withZeroLabel) + ] + in ranges `M.union` sums + +showPrometheusMetrics :: PrometheusMetrics -> L.ByteString +showPrometheusMetrics = L.fromStrict . T.encodeUtf8 . M.foldlWithKey + (\a k (h, m) -> T.concat [a, "# HELP ", k, " ", h, "\n" + , "# TYPE ", k, " ", showType m, "\n" + ,case m of + Histogram h' -> fst $ + M.foldlWithKey (showHistogram k) + ("", 0.0) h' + Counter v -> + T.concat [k, " ", T.pack $ show v, "\n"] + ] + ) "" + where showType (Counter _) = "counter" + showType (Histogram _) = "histogram" + showHistogram k a@(t, n) c (l, v) = + if T.null l + then if k `T.append` "_sum" == c + then (T.concat [t, c, " ", T.pack $ show v, "\n"] + ,n + ) + else if k `T.append` "_cnt" == c + then (T.concat [t, k, "_count " + ,T.pack $ + show (round v :: Word64) + ,"\n" + ] + ,n + ) + else a + else let n' = n + v + in (T.concat [t, k, "_bucket{le=\"", l, "\"} " + ,T.pack $ show (round n' :: Word64) + ,"\n" + ] + ,n' + ) + +toPrometheusMetrics :: ByteString -> IO L.ByteString +toPrometheusMetrics v = do + let cs = fromJust $ readFromByteStringAsJSON @AllMetrtics v + pc <- readIORef conf + return $ case pc of + Just c -> showPrometheusMetrics $ toPrometheusMetrics' c cs + Nothing -> "" + +ngxExportIOYY 'toPrometheusMetrics + +-- | Multiplies a floating point value by a factor. +-- +-- Returns an integer value as the result of rounding the scaled floating point +-- value. +scale + :: Int -- ^ Factor + -> Double -- ^ Floating point value + -> Int +scale n = round . (fromIntegral n *) + +-- | Multiplies a floating point value by /1000/. +-- +-- The floating point value gets read from a 'ByteString'. Throws an exception +-- on conversion failure which results in returning an empty string. +scale1000 + :: ByteString -- ^ Floating point value + -> L.ByteString +scale1000 v = let v' = fromJust $ readFromByteString @Double v + in C8L.pack $ show $ scale 1000 v' + +ngxExportYY 'scale1000 + diff --git a/NgxExport/Tools/Subrequest.hs b/NgxExport/Tools/Subrequest.hs --- a/NgxExport/Tools/Subrequest.hs +++ b/NgxExport/Tools/Subrequest.hs @@ -115,7 +115,7 @@ -- -- location \/ { -- haskell_run_async __/makeSubrequest/__ $hs_subrequest --- \'{\"uri\": \"http:\/\/127.0.0.1:8020\/proxy\", +-- \'{\"uri\": \"http:\/\/127.0.0.1:8010\/proxy\", -- \"headers\": [[\"Custom-Header\", \"$arg_a\"]]}\'; -- -- if ($hs_subrequest = \'\') { @@ -128,7 +128,8 @@ -- } -- -- location \/proxy { --- internal; +-- allow 127.0.0.1; +-- deny all; -- proxy_pass http:\/\/backend; -- } -- diff --git a/ngx-export-tools-extra.cabal b/ngx-export-tools-extra.cabal --- a/ngx-export-tools-extra.cabal +++ b/ngx-export-tools-extra.cabal @@ -1,5 +1,5 @@ name: ngx-export-tools-extra -version: 0.4.1.1 +version: 0.5.0.0 synopsis: More extra tools for Nginx haskell module description: More extra tools for . @@ -13,7 +13,7 @@ copyright: 2019-2020 Alexey Radkov category: Network build-type: Simple -cabal-version: >= 1.8 +cabal-version: 1.20 source-repository head type: git @@ -29,6 +29,7 @@ manual: True library + default-language: Haskell2010 build-depends: base >= 4.8 && < 5 , template-haskell >= 2.11.0.0 , bytestring >= 0.10.0.0 @@ -47,6 +48,7 @@ , snap-server , text , time + , safe if flag(EDE) if flag(ExperimentalEDE) @@ -60,6 +62,7 @@ , trifecta <= 2 exposed-modules: NgxExport.Tools.Aggregate + , NgxExport.Tools.Prometheus , NgxExport.Tools.Subrequest if flag(EDE)