packages feed

kenshou-cli-0.1.0.0: src/Kenshou/Cli/Diagnose.hs

module Kenshou.Cli.Diagnose
  ( diagnoseCommand,
  )
where

import Control.Exception (SomeException, bracket, displayException, try)
import Data.Aeson
import Data.ByteString.Lazy qualified as LazyByteString
import Data.Foldable (traverse_)
import Data.List (isPrefixOf, isSuffixOf, sort)
import Data.Text (Text)
import Data.Text qualified as Text
import Data.Text.IO qualified as Text.IO
import Data.Time (getCurrentTime)
import Data.Word (Word64)
import Hasql.Connection qualified as Connection
import Hasql.Connection.Settings qualified as Settings
import Kenshou.Core.Cli
import Kenshou.Diagnose.Document
import Kenshou.Diagnose.Leak
import Kenshou.Diagnose.LockGraph (buildGraph, renderDot)
import Kenshou.Diagnose.Postgres
import Kenshou.Diagnose.Profile
import Kenshou.Diagnose.Render
import Kenshou.Diagnose.Stall.Classify (classify)
import Kenshou.Diagnose.Stall.Types
import Kenshou.Diagnose.Threads (dumpThreads)
import Options.Applicative hiding (Success)
import System.Directory (doesDirectoryExist, doesFileExist, listDirectory)
import System.Exit (ExitCode (..))
import System.FilePath (takeFileName, (</>))
import System.IO (stderr)
import System.Process (callProcess)

data DiagnoseOptions = DiagnoseLeak LeakOptions | DiagnoseStall StallOptions | DiagnoseProfile ProfileOptions

data LeakOptions = LeakOptions
  { runDirectory :: !FilePath,
    policy :: !(Maybe InputSource),
    seed :: !Word64,
    output :: !(Maybe FilePath),
    json :: !Bool
  }

data StallOptions = StallOptions
  { runDirectory :: !(Maybe FilePath),
    live :: !Bool,
    connection :: !(Maybe Text),
    reclassify :: !Bool,
    dotOutput :: !(Maybe FilePath),
    output :: !(Maybe FilePath),
    json :: !Bool
  }

data ProfileOptions = ProfileOptions
  { scenario :: !Text,
    modeName :: !Text,
    classes :: !String,
    breakdown :: !Char,
    intervalSeconds :: !Double,
    eventlogMaxBytes :: !Word64,
    worker :: !(Maybe Text),
    outputRoot :: !FilePath,
    knobs :: ![String],
    dimensions :: ![String]
  }

diagnoseCommand :: CliCommand
diagnoseCommand = CliCommand "diagnose" "Diagnose leaks, stalls, and profiles" Analysis False (runDiagnose <$> diagnoseParser)

diagnoseParser :: Parser DiagnoseOptions
diagnoseParser = hsubparser (command "leak" (info (DiagnoseLeak <$> leakParser) (progDesc "Re-judge sampled series for resource leaks")) <> command "stall" (info (DiagnoseStall <$> stallParser) (progDesc "Render or capture a concurrency stall")) <> command "profile" (info (DiagnoseProfile <$> profileParser) (progDesc "Run a scenario under heap or event-log profiling")))

leakParser :: Parser LeakOptions
leakParser =
  LeakOptions
    <$> strArgument (metavar "RUN_DIR")
    <*> optional (parseInputSource <$> strOption (long "policy" <> metavar "FILE" <> help "Leak policy file, or - for stdin"))
    <*> option auto (long "seed" <> metavar "N" <> value 0 <> showDefault)
    <*> optional (strOption (long "out" <> metavar "FILE"))
    <*> switch (long "json")

stallParser :: Parser StallOptions
stallParser =
  StallOptions
    <$> optional (strArgument (metavar "RUN_DIR"))
    <*> switch (long "live" <> help "Capture PostgreSQL and local threads now")
    <*> optional (Text.pack <$> strOption (long "connection" <> metavar "CONNECTION"))
    <*> switch (long "reclassify")
    <*> optional (strOption (long "dot" <> metavar "FILE"))
    <*> optional (strOption (long "out" <> metavar "FILE"))
    <*> switch (long "json")

profileParser :: Parser ProfileOptions
profileParser =
  ProfileOptions
    <$> (Text.pack <$> strArgument (metavar "SCENARIO"))
    <*> (Text.pack <$> strOption (long "mode" <> metavar "closure-type|info-table|eventlog|profiled"))
    <*> strOption (long "classes" <> metavar "CLASSES" <> value "gu" <> showDefault)
    <*> option (eitherReader parseBreakdown) (long "breakdown" <> metavar "c|r|d|y" <> value 'c' <> showDefault)
    <*> option auto (long "interval-s" <> metavar "SECONDS" <> value 10 <> showDefault)
    <*> option auto (long "eventlog-max-bytes" <> metavar "BYTES" <> value 536870912 <> showDefault)
    <*> optional (Text.pack <$> strOption (long "worker" <> metavar "ROLE"))
    <*> strOption (long "out" <> metavar "DIR" <> value ".dev/profiles" <> showDefault)
    <*> many (strOption (long "set" <> metavar "NAME=VALUE"))
    <*> many (strOption (long "dim" <> metavar "NAME=VALUE"))

runDiagnose :: DiagnoseOptions -> CliEnv -> IO ExitCode
runDiagnose (DiagnoseLeak options) _ = runLeak options
runDiagnose (DiagnoseStall options) _ = runStall options
runDiagnose (DiagnoseProfile options) _ = runProfile options

runLeak :: LeakOptions -> IO ExitCode
runLeak options = do
  attempted <- try @SomeException do
    spec <- loadPolicy options.policy
    analyseRunDirectory options.runDirectory spec options.seed
  case attempted of
    Left err -> diagnostic "leak" (Text.pack (displayException err)) >> pure (ExitFailure 4)
    Right (Left err) -> diagnostic "leak" (Text.pack (show err)) >> pure (ExitFailure 4)
    Right (Right report) -> do
      now <- getCurrentTime
      let document = Diagnosis LeakDiagnosis (Text.pack (takeFileName options.runDirectory)) "offline" now (Generator "kenshou-diagnose" "0.1.0.0" "theil-sen+moving-block-bootstrap/1") (toJSON report)
      emit options.json options.output document (renderLeakReport report)
      pure case report.verdict of
        Stable -> ExitSuccess
        LeakSuspected -> ExitFailure 1
        InsufficientData -> ExitFailure 3

loadPolicy :: Maybe InputSource -> IO LeakSpec
loadPolicy Nothing = pure defaultLeakSpec
loadPolicy (Just source) = do
  bytes <- readInputSource source
  either (ioError . userError) pure (eitherDecodeStrict' bytes)

runStall :: StallOptions -> IO ExitCode
runStall options
  | options.live = runLiveStall options
  | otherwise = case options.runDirectory of
      Nothing -> diagnostic "stall" "provide RUN_DIR, or use --live --connection CONNECTION" >> pure (ExitFailure 2)
      Just runDirectory -> do
        attempted <- try @SomeException (loadStallReports runDirectory)
        case attempted of
          Left err -> diagnostic "stall" (Text.pack (displayException err)) >> pure (ExitFailure 4)
          Right reports -> do
            let rendered = fmap (if options.reclassify then reclassified else id) reports
            traverse_ (writeDot options.dotOutput) (take 1 rendered)
            if null rendered
              then Text.IO.putStrLn "no stall diagnoses" >> pure ExitSuccess
              else do
                now <- getCurrentTime
                let body = toJSON rendered
                    document = Diagnosis StallDiagnosis (Text.pack (takeFileName runDirectory)) "offline" now (Generator "kenshou-diagnose" "0.1.0.0" "stall-render-v1") body
                emit options.json options.output document (Text.intercalate "\n" (fmap renderStallReport rendered))
                pure (ExitFailure 1)

runLiveStall :: StallOptions -> IO ExitCode
runLiveStall options = case options.connection of
  Nothing -> diagnostic "stall" "--live requires --connection CONNECTION" >> pure (ExitFailure 2)
  Just connectionString -> do
    attempted <- try @SomeException $ bracket (acquire connectionString) Connection.release \connection -> do
      postgres <- capturePostgres connection 2
      threads <- dumpThreads False
      now <- getCurrentTime
      let graph = buildGraph postgres.activity postgres.locks
          snapshot = StallSnapshot 0 [] threads (Just postgres) graph [] Null (SpinEvidence 0 (maybe 0 (.callsPerSecond) postgres.statementRate))
          (primary, secondary, reasons) = classify snapshot
      pure (StallReport now 1 primary secondary reasons snapshot)
    case attempted of
      Left err -> diagnostic "stall" (Text.pack (displayException err)) >> pure (ExitFailure 4)
      Right report -> do
        writeDot options.dotOutput report
        let document = Diagnosis StallDiagnosis "live" "live" report.detectedAt (Generator "kenshou-diagnose" "0.1.0.0" "live-stall-v1") (toJSON report)
        emit options.json options.output document (renderStallReport report)
        pure if report.classification == Unknown then ExitSuccess else ExitFailure 1

acquire :: Text -> IO Connection.Connection
acquire connectionString = Connection.acquire (Settings.connectionString connectionString <> Settings.applicationName "kenshou-diagnose-live") >>= either (ioError . userError . show) pure

loadStallReports :: FilePath -> IO [StallReport]
loadStallReports runDirectory = do
  exists <- doesDirectoryExist runDirectory
  if not exists then ioError (userError ("run directory does not exist: " <> runDirectory)) else pure ()
  let diagnosisDirectory = runDirectory </> "diagnosis"
  diagnosisExists <- doesDirectoryExist diagnosisDirectory
  if not diagnosisExists
    then pure []
    else do
      names <- sort <$> listDirectory diagnosisDirectory
      traverse (loadOne . (diagnosisDirectory </>)) [name | name <- names, "stall-" `isPrefixOf` name, ".json" `isSuffixOf` name]
  where
    loadOne path = do
      decoded <- eitherDecodeFileStrict' path :: IO (Either String Diagnosis)
      document <- either (ioError . userError) pure decoded
      case fromJSON document.body of
        Error err -> ioError (userError (path <> ": " <> err))
        Success report -> pure report

reclassified :: StallReport -> StallReport
reclassified report =
  let (primary, secondary, reasons) = classify report.snapshot
   in StallReport report.detectedAt report.captureNumber primary secondary reasons report.snapshot

writeDot :: Maybe FilePath -> StallReport -> IO ()
writeDot Nothing _ = pure ()
writeDot (Just path) report = Text.IO.writeFile path (renderDot report.snapshot.graph)

runProfile :: ProfileOptions -> IO ExitCode
runProfile options = case parseMode options of
  Left message -> diagnostic "profile" message >> pure (ExitFailure 2)
  Right mode -> do
    let arguments = concatMap (\assignment -> ["--set", assignment]) options.knobs <> concatMap (\assignment -> ["--dim", assignment]) options.dimensions
        request = ProfileRequest mode options.scenario arguments options.outputRoot options.intervalSeconds options.eventlogMaxBytes options.worker
    attempted <- try @SomeException (runProfileSession request >>= postprocessProfile)
    case attempted of
      Left err -> diagnostic "profile" (Text.pack (displayException err)) >> pure (ExitFailure 4)
      Right report -> Text.IO.putStr (renderProfileReport report) >> pure (toExitCode report.exitCode)

postprocessProfile :: ProfileReport -> IO ProfileReport
postprocessProfile report = do
  let tool = ".dev/bin/eventlog2html"
      eventlog = report.sessionDirectory </> "kenshou.eventlog"
  available <- doesFileExist tool
  eventlogExists <- doesFileExist eventlog
  if available && eventlogExists then callProcess tool [eventlog] else pure ()
  pure report

parseMode :: ProfileOptions -> Either Text ProfileMode
parseMode options = case options.modeName of
  "closure-type" -> Right ClosureType
  "info-table" -> Right InfoTable
  "eventlog" -> Right (Eventlog options.classes)
  "profiled" -> Right (Profiled options.breakdown)
  other -> Left ("unknown profile mode " <> other)

parseBreakdown :: String -> Either String Char
parseBreakdown [breakdown] | breakdown `elem` ("crdy" :: String) = Right breakdown
parseBreakdown _ = Left "breakdown must be c, r, d, or y"

emit :: (ToJSON value) => Bool -> Maybe FilePath -> value -> Text -> IO ()
emit json output document rendered = do
  let bytes = encode document <> "\n"
  traverse_ (\path -> LazyByteString.writeFile path bytes) output
  case (json, output) of
    (True, Nothing) -> LazyByteString.putStr bytes
    (True, Just _) -> pure ()
    (False, _) -> Text.IO.putStr rendered

diagnostic :: Text -> Text -> IO ()
diagnostic commandName message = Text.IO.hPutStrLn stderr ("kenshou diagnose " <> commandName <> ": " <> message)

toExitCode :: Int -> ExitCode
toExitCode 0 = ExitSuccess
toExitCode code = ExitFailure code