hledger-flow-0.21.0: src/Hledger/Flow/ReportReview.hs
{-# LANGUAGE DeriveAnyClass #-}
{-# LANGUAGE DeriveGeneric #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
{-# LANGUAGE ScopedTypeVariables #-}
module Hledger.Flow.ReportReview
( Review (..),
ReviewCategory (..),
ReviewItem (..),
ReviewFile,
ReviewKind (..),
captureReportInputs,
countTransferAccounts,
countUnknownTransactions,
invalidateReportReview,
loadReportReview,
reviewFile,
reviewHasData,
saveReportReview,
startReportRun,
)
where
import Control.Exception (IOException, try)
import Data.Aeson (FromJSON, ToJSON, decodeFileStrict', encodeFile)
import Data.Char (isDigit, isSpace)
import Data.List (nub, sortOn)
import Data.Maybe (mapMaybe)
import Data.Ord (Down (..))
import qualified Data.Text as T
import qualified Data.Text.IO as T
import Data.Time.Clock (UTCTime)
import GHC.Generics (Generic)
import Hledger.Flow.InputSnapshot (InputSnapshot, inputsUnchanged, snapshotInputs)
import Hledger.Flow.Cache (cacheDir, initializeCache)
import Hledger.Flow.LedgerSelection (journalInputs)
import Hledger.Flow.PathHelpers (AbsFile, RelFile, withStagedOutput, writeTimestampMarker)
import Hledger.Flow.Types (BaseDir, HledgerConf, HledgerInfo, hledgerConfPath)
import Path
import qualified Path.IO as PathIO
import qualified System.Directory as D
import qualified System.FilePath as F
import System.IO.Error (catchIOError, isDoesNotExistError)
import Text.Read (readMaybe)
data ReviewKind = UnknownTransactions | TransferBalances
deriving (Eq, Generic, Show, FromJSON, ToJSON)
data ReviewFile = ReviewFile ReviewKind RelFile Integer
deriving (Generic, FromJSON, ToJSON)
data ReviewItem = ReviewItem
{ itemKind :: ReviewKind,
itemPath :: FilePath,
itemYear :: Integer,
itemCount :: Maybe Int
}
deriving (Show)
data ReviewCategory = ReviewCategory {reviewCount :: Maybe Int, reviewFiles :: [FilePath]}
deriving (Show)
data Review = Review
{ reviewUnknowns :: ReviewCategory,
reviewTransfers :: ReviewCategory,
reviewItems :: [ReviewItem]
}
deriving (Show)
data ReportInputs = ReportInputs
{ runStarted :: UTCTime,
runAggregate :: AbsFile,
runConfig :: Maybe AbsFile,
runInputs :: InputSnapshot
}
deriving (Generic, FromJSON, ToJSON)
data Manifest = Manifest ReportInputs [ReviewFile]
deriving (Generic, FromJSON, ToJSON)
manifestFile :: BaseDir -> AbsFile
manifestFile base = cacheDir base </> [relfile|reports.json|]
invalidateReportReview :: BaseDir -> IO ()
invalidateReportReview base =
PathIO.removeFile (manifestFile base) `catchIOError` \err ->
if isDoesNotExistError err then return () else ioError err
startReportRun :: BaseDir -> IO UTCTime
startReportRun base = do
initializeCache base
invalidateReportReview base
writeTimestampMarker (cacheDir base </> [relfile|report-started|])
configPath :: Maybe HledgerConf -> IO (Maybe AbsFile)
configPath = traverse ((>>= parseAbsFile) . D.canonicalizePath . hledgerConfPath)
captureReportInputs :: HledgerInfo -> UTCTime -> AbsFile -> Maybe HledgerConf -> [[AbsFile]] -> IO (Either T.Text ReportInputs)
captureReportInputs info started aggregate conf roots = do
discovered <- traverse (journalInputs info) (nub roots)
case sequence discovered of
Left err -> return (Left err)
Right paths -> do
result <- try $ do
config <- configPath conf
snapshot <- snapshotInputs (maybe id (:) config (concat paths))
return (ReportInputs started aggregate config snapshot)
return (either (Left . T.pack . show) Right (result :: Either IOException ReportInputs))
reviewFile :: BaseDir -> AbsFile -> Maybe ReviewFile
reviewFile base file = do
kind <- case toFilePath (filename file) of
"unknown-transactions.txt" -> Just UnknownTransactions
"transfer-balance.txt" -> Just TransferBalances
_ -> Nothing
rel <- stripProperPrefix base file
year <- readMaybe (F.dropTrailingPathSeparator (toFilePath (dirname (parent file))))
return (ReviewFile kind rel year)
saveReportReview :: BaseDir -> ReportInputs -> [ReviewFile] -> IO ()
saveReportReview base inputs files =
withStagedOutput (manifestFile base) (\stage -> encodeFile (toFilePath stage) (Manifest inputs files))
loadReportReview :: BaseDir -> AbsFile -> Maybe HledgerConf -> IO (Maybe Review)
loadReportReview base aggregate conf = do
result <- try $ do
saved <- decodeFileStrict' (toFilePath (manifestFile base))
case saved of
Nothing -> return Nothing
Just (Manifest inputs files) -> do
currentConfig <- configPath conf
if runConfig inputs /= currentConfig || runAggregate inputs /= aggregate
then return Nothing
else do
current <- inputsUnchanged (runInputs inputs) (runStarted inputs)
if current then Just . summarize <$> traverse readReview files else return Nothing
return (either (const Nothing :: IOException -> Maybe Review) id result)
where
readReview (ReviewFile kind rel year) = do
text <- T.readFile (toFilePath (base </> rel))
let body = T.unlines (drop 2 (T.lines text))
count = if T.null (T.strip body) then Nothing else countFor kind body
return (ReviewItem kind (toFilePath rel) year count)
summarize items =
let ordered = sortOn (\item -> (Down (itemYear item), itemPath item)) items
in Review (category UnknownTransactions ordered) (category TransferBalances ordered) ordered
category kind items =
let relevant = filter ((== kind) . itemKind) items
in ReviewCategory (sum <$> traverse itemCount relevant) (map itemPath relevant)
countFor :: ReviewKind -> T.Text -> Maybe Int
countFor UnknownTransactions = countUnknownTransactions
countFor TransferBalances = countTransferAccounts
reviewHasData :: ReviewKind -> T.Text -> Bool
reviewHasData kind text = case filter (not . T.null) (map T.strip (T.lines text)) of
[] -> False
[line] | csvHeader kind line -> False
_ -> countFor kind text /= Just 0
csvHeader :: ReviewKind -> T.Text -> Bool
csvHeader UnknownTransactions = T.isPrefixOf "\"txnidx\",\"date\","
csvHeader TransferBalances = T.isPrefixOf "\"account\","
countUnknownTransactions :: T.Text -> Maybe Int
countUnknownTransactions text
| null rows = Just 0
| not (null headers) && all known rows = Just (length headers)
| otherwise = Nothing
where
rows = filter (not . T.null . T.strip) (T.lines text)
headers = filter dateHeader rows
known row = dateHeader row || maybe False (isSpace . fst) (T.uncons row) || ";" `T.isPrefixOf` row
dateHeader row = case T.unpack (T.take 10 row) of
[a,b,c,d,'-',e,f,'-',g,h] ->
all isDigit [a,b,c,d,e,f,g,h]
&& maybe True (\(end, _) -> isSpace end || end == '=') (T.uncons (T.drop 10 row))
_ -> False
countTransferAccounts :: T.Text -> Maybe Int
countTransferAccounts text = case filter (not . T.null) (map T.strip (T.lines text)) of
[] -> Just 0
title : table | "Balance changes in " `T.isPrefixOf` title ->
case break divider table of
(heading, _ : body)
| any (\line -> "||" `T.isInfixOf` line || "║" `T.isInfixOf` line) heading ->
let rows = filter (not . T.isPrefixOf "└") body
in if all accountRow rows then Just (length rows) else Nothing
_ -> Nothing
_ -> Nothing
where
divider row = "╞" `T.isPrefixOf` row || ("=" `T.isPrefixOf` row && "++" `T.isInfixOf` row)
accountRow row = case mapMaybe (\sep -> nonemptyField sep row) ["||", "║"] of
[_] -> True
_ -> False
nonemptyField sep row =
let (label, rest) = T.breakOn sep row
in if T.null rest || T.null (T.strip (T.dropWhile (== '│') label)) then Nothing else Just label