packages feed

ekg-rrd (empty) → 0.2.0.5

raw patch · 6 files changed

+348/−0 lines, 6 filesdep +HUnitdep +basedep +directorysetup-changed

Dependencies added: HUnit, base, directory, ekg-core, ekg-rrd, mtl, process, test-framework, test-framework-hunit, text, time, unordered-containers

Files

+ LICENSE view
@@ -0,0 +1,30 @@+Copyright (c) 2014, David Turner++All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++    * Redistributions of source code must retain the above copyright+      notice, this list of conditions and the following disclaimer.++    * Redistributions in binary form must reproduce the above+      copyright notice, this list of conditions and the following+      disclaimer in the documentation and/or other materials provided+      with the distribution.++    * Neither the name of David Turner nor the names of other+      contributors may be used to endorse or promote products derived+      from this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ ekg-rrd.cabal view
@@ -0,0 +1,44 @@+name:                ekg-rrd+version:             0.2.0.5+synopsis:            Passes ekg statistics to rrdtool+description:         Simple API for passing ekg monitoring statistics to a round-robin database (RRD) using rrdtool.+homepage:            https://bitbucket.org/davecturner/ekg-rrd+license:             BSD3+license-file:        LICENSE+author:              David Turner+maintainer:          dct25-dkefo@mythic-beasts.com+copyright:           David Turner 2014+category:            System+build-type:          Simple+cabal-version:       >=1.10++library+  exposed-modules:      System.Metrics.RRDTool+                      , System.Metrics.RRDTool.Internals+  build-depends:       base >=4.7 && <4.8+                      , text == 1.2.*+                      , unordered-containers == 0.2.*+                      , process == 1.2.*+                      , ekg-core == 0.1.*+                      , directory == 1.2.*+                      , time == 1.4.*+                      , mtl  == 2.1.*+  hs-source-dirs:      src+  default-language:    Haskell2010+  ghc-options: -Wall++test-suite test+  type: exitcode-stdio-1.0+  main-is: Main.hs+  hs-source-dirs: test+  default-language:   Haskell2010+  build-depends:      base+                    , ekg-rrd+                    , HUnit+                    , ekg-core+                    , test-framework+                    , test-framework-hunit+                    , text+                    , time+                    , unordered-containers+  ghc-options: -Wall -with-rtsopts=-T
+ src/System/Metrics/RRDTool.hs view
@@ -0,0 +1,104 @@+{-# LANGUAGE OverloadedStrings #-}++{- | Simple API for passing ekg monitoring statistics to a round-robin database+(RRD) using `rrdtool`. -}++module System.Metrics.RRDTool+  ( -- * Basic Types+    IntervalSeconds+  , SourceValue++  -- * Data Sources+  , DataSource(..)+  , DataSourceType(..)+  , gcSources++  -- * Round Robin Archives+  , ConsolidationFunction(..)+  , RoundRobinArchive(..)++  -- * Round Robin Databases+  , RoundRobinDatabase+  , newRRD+  , rrdStore++  -- * Monitoring Thread+  , MonitorThread+  , runMonitor+  , killMonitor+  ) where++import Control.Concurrent+import Control.Exception+import Control.Monad+import Data.Maybe+import Data.Time+import System.Directory+import System.Metrics+import System.Process++import qualified Data.HashMap.Strict as HM++import System.Metrics.RRDTool.Internals++{- | Create a new 'RoundRobinDatabase'. Creates the database file on disk if+not already present.  Metrics must subsequently be registered in the store+associated with the 'RoundRobinDatabase', which can be obtained with+'rrdStore'. Throws an 'IOException' if there was a problem running `rrdtool`.+-}+newRRD+  :: FilePath -- ^ The name of the database file.+  -> IntervalSeconds -- ^ The update interval in seconds.+  -> Maybe FilePath -- ^ A path to the `rrdtool` binary, or 'Nothing' to search `$PATH` for `rrdtool`.+  -> [RoundRobinArchive] -- ^ Round-robin archives that define how data is stored.+  -> [DataSource] -- ^ Data sources to define in the database.+  -> IO RoundRobinDatabase+newRRD dbPath step maybeToolPath archives sources = do+  store <- newStore++  let rrd = RoundRobinDatabase+        { rrdToolPath = fromMaybe "rrdtool" maybeToolPath+        , rrdFilePath = dbPath+        , rrdArchives = archives+        , rrdSources  = HM.fromList [(dsMetric source, source) | source <- sources]+        , rrdStore    = store+        , rrdStep     = step+        }++  rrdExists <- doesFileExist $ rrdFilePath rrd+  unless rrdExists $ runTool rrd $ createRRDArgs rrd+  return rrd++runTool :: RoundRobinDatabase -> [String] -> IO ()+runTool = callProcess . rrdToolPath++{- | The monitoring thread for a round robin database, which can be created+with 'runMonitor' and killed with 'killMonitor'. -}+data MonitorThread = MonitorThread+  { killMonitor :: IO () -- ^ Kills the monitoring thread.+  }++{- | Run a monitoring thread for the given database, which samples the metrics+and updates the database at the chosen frequency. Failed updates are ignored.+-}+runMonitor :: RoundRobinDatabase -> IO MonitorThread+runMonitor rrd = do+  joinVar <- newEmptyMVar+  monitorThreadId <- mask $ \restore -> forkIO $ monitorThread restore `finally` putMVar joinVar ()+  return MonitorThread+    { killMonitor = killThread monitorThreadId >> takeMVar joinVar+    }+  where+  monitorThread restore = forever $ do+    threadDelay $ rrdStep rrd * 1000000+    void $ forkIO $ restore rrdUpdate++  ignoreIOException :: IOException -> IO ()+  ignoreIOException = const $ return ()++  rrdUpdate = do+    now <- getCurrentTime+    sample <- sampleAll $ rrdStore rrd+    case updateRRDArgs rrd sample now of+      Nothing -> return ()+      Just args -> handle ignoreIOException $ runTool rrd args
+ src/System/Metrics/RRDTool/Internals.hs view
@@ -0,0 +1,159 @@+{-# LANGUAGE OverloadedStrings #-}++module System.Metrics.RRDTool.Internals where++import Control.Monad+import Control.Monad.Writer+import Data.Int+import Data.List+import Data.Time+import Data.Ord+import Data.Word+import System.Metrics+import Text.Printf++import qualified Data.HashMap.Strict as HM+import qualified Data.Text as T++{- | Intervals in seconds (e.g. heartbeats and step size) -}+type IntervalSeconds = Int++{- | Counters, gauges etc. in `ekg` have this type -}+type SourceValue = Int64++{- | Types of data source in a round-robin database. -}+data DataSourceType+  {- | Metrics whose current value is tracked. This is appropriate for+'System.Metrics.Gauge.Gauge' metrics and creates a `GAUGE` DS. -}+  = DsGauge+    {- | Metrics that are monotonically increasing and whose rate-of-change is+tracked. This is appropriate for 'System.Metrics.Counter.Counter' metrics and+creates a `DERIVE` DS. -}+  | DsDerive+  deriving (Eq, Show, Bounded, Enum)++{- | A data source in a round-robin database. -}+data DataSource = DataSource+  { dsName      :: T.Text -- ^ The name of the metric in the database. Maximum 19 characters and contains letters, digits and underscores only.+  , dsMetric    :: T.Text -- ^ The name of the metric in the `ekg` 'Store'.+  , dsType      :: DataSourceType -- ^ The type of the metric.+  , dsHeartBeat :: IntervalSeconds -- ^ After this length of time with no updates, consider the value to be \'unknown\'.+  , dsMin       :: Maybe SourceValue -- ^ Values less than this should be interpreted as \'unknown\'.+  , dsMax       :: Maybe SourceValue -- ^ Values greater than this should be interpreted as \'unknown\'.+  } deriving (Show, Eq)++{- | Pre-defined data sources for tracking GHC's GC metrics. See 'registerGcMetrics' or 'GHC.Stats.GCStats' for more details.+     Clients must run 'registerGcMetrics' to register these metrics once the RRD is created. -}+gcSources+  :: IntervalSeconds -- ^ The heartbeat for these metrics, in seconds.+  -> [DataSource]+gcSources heartBeat = gcCounters ++ gcGauges+  where+  gcCounters = map (\(metric, name) -> DataSource name metric DsDerive heartBeat (Just 0) Nothing)+      [("rts.gc.bytes_allocated", "bytes_allocated")+      ,("rts.gc.num_gcs", "num_gcs")+      ,("rts.gc.num_bytes_usage_samples", "num_bytes_usage_sam")+      ,("rts.gc.cumulative_bytes_used", "cumulative_bytes_us")+      ,("rts.gc.bytes_copied", "bytes_copied")+      ,("rts.gc.mutator_cpu_ms", "mutator_cpu_ms")+      ,("rts.gc.mutator_wall_ms", "mutator_wall_ms")+      ,("rts.gc.gc_cpu_ms", "gc_cpu_ms")+      ,("rts.gc.gc_wall_ms", "gc_wall_ms")+      ,("rts.gc.cpu_ms", "cpu_ms")+      ,("rts.gc.wall_ms", "wall_ms")+      ]++  gcGauges = map (\(metric, name) -> DataSource name metric DsGauge heartBeat (Just 0) Nothing)+      [("rts.gc.max_bytes_used", "max_bytes_used")+      ,("rts.gc.current_bytes_used", "current_bytes_used")+      ,("rts.gc.current_bytes_slop", "current_bytes_slop")+      ,("rts.gc.max_bytes_slop", "max_bytes_slop")+      ,("rts.gc.peak_megabytes_allocated", "peak_megabytes_allo")+      ]++{- | Defines how multiple primary data points (PDPs) are turned into a+consolidated data point (CDP) for storage in a 'RoundRobinArchive'. -}+data ConsolidationFunction+  = CFLast -- ^ Keep the last point only.+  | CFAverage -- ^ Take the mean of the points.+  | CFMin -- ^ Take the minimum point.+  | CFMax -- ^ Take the maximum point.+  deriving (Show, Eq)++{- | A sequence of consolidated data points (CDPs). -}+data RoundRobinArchive = RoundRobinArchive+  { rraCf :: ConsolidationFunction -- ^ How to consolidate PDPs to get each CDP.+  , rraXff :: Double -- ^ The \'Xfiles factor\' - if more than this proportion of PDPs is unknown then the CDP is unknown.+  , rraPdpCount :: Int -- ^ The number of PDPs to use to calculate each CDP.+  , rraRecordCount :: Int -- ^ The number of CDPs to store in the archive.+  } deriving (Show, Eq)++{- | A round-robin database (RRD), which is a file on disk that stores+     time series data from a number of sources. -}+data RoundRobinDatabase = RoundRobinDatabase+  { rrdToolPath :: FilePath+  , rrdFilePath :: FilePath+  , rrdSources  :: HM.HashMap T.Text DataSource+  , rrdArchives :: [RoundRobinArchive]+  , rrdStore    :: Store -- ^ Get the 'Store' associated with this 'RoundRobinDatabase' in order to register metrics.+  , rrdStep     :: IntervalSeconds+  }++defineDataSource :: DataSource -> String+defineDataSource ds = printf "DS:%s:%s:%d:%s:%s"+  (T.unpack $ dsName ds)+  (case dsType ds of+    DsDerive -> "DERIVE"+    DsGauge  -> "GAUGE" :: String)+  (dsHeartBeat ds)+  (showMaybeValue $ dsMin ds)+  (showMaybeValue $ dsMax ds)++showMaybeValue :: Maybe SourceValue -> String+showMaybeValue = maybe "U" (showWord64 . fromIntegral)++defineRoundRobinArchive :: RoundRobinArchive -> String+defineRoundRobinArchive rra = printf "RRA:%s:%f:%d:%d"+  (case rraCf rra of+    CFAverage -> "AVERAGE"+    CFMin -> "MIN"+    CFMax -> "MAX"+    CFLast -> "LAST" :: String)+  (rraXff rra)+  (rraPdpCount rra)+  (rraRecordCount rra)++createRRDArgs :: RoundRobinDatabase -> [String]+createRRDArgs rrd = execWriter $ do+  tell ["create", rrdFilePath rrd, "--no-overwrite", "-s", show $ rrdStep rrd]+  forM_ (sortBy (comparing dsName) $ HM.elems $ rrdSources rrd)+    $ tell . return . defineDataSource+  forM_ (rrdArchives rrd) $ tell . return . defineRoundRobinArchive++updateRRDArgs :: RoundRobinDatabase -> HM.HashMap T.Text Value -> UTCTime -> Maybe [String]+updateRRDArgs rrd sample now =+  let updateSpec = HM.elems $ HM.intersectionWith (,) sample $ rrdSources rrd+  in if null updateSpec+        then Nothing+        else Just $ execWriter $ do+          tell ["update", rrdFilePath rrd]+          tell ["-t", intercalate ":" $ map (T.unpack . dsName . snd) updateSpec]+          tell [epochTimeFromUTCTime now ++ concatMap (formatValue . fst) updateSpec]++epoch :: UTCTime+epoch = UTCTime (fromGregorian 1970 1 1) 0++epochTime :: UTCTime -> Integer+epochTime = floor . flip diffUTCTime epoch++epochTimeFromUTCTime :: UTCTime -> String+epochTimeFromUTCTime = show . epochTime++formatValue :: Value -> String+formatValue = (':':) . showMaybeValue . getMaybeValue+  where getMaybeValue (Counter n) = Just n+        getMaybeValue (Gauge n)   = Just n+        getMaybeValue _           = Nothing++showWord64 :: Word64 -> String+showWord64 = show
+ test/Main.hs view
@@ -0,0 +1,9 @@+module Main (main) where++import qualified Tests.System.Metrics.RRDTool.Tests (tests)+import Test.Framework++main :: IO ()+main = defaultMain+  [ Tests.System.Metrics.RRDTool.Tests.tests+  ]