hledger-flow-0.21.0: src/Hledger/Flow/Status/Gather.hs
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
-- | Collect a ledger's status from disk. Read-only and silent.
module Hledger.Flow.Status.Gather
( classifyDir,
gatherStatus,
gatherStatusWithConfig,
accountName,
cachedStatement,
hookExists,
mainRulesCandidates,
)
where
import Control.Applicative ((<|>))
import Control.Monad (forM, when)
import Data.Char (toLower)
import qualified Data.List as List
import qualified Data.Map.Strict as Map
import qualified Data.Set as Set
import Data.Maybe (catMaybes, fromMaybe, isJust, isNothing)
import qualified Data.Text as T
import qualified Data.Text.IO as T
import Data.Time.Clock (UTCTime)
import Hledger.Flow.BaseDir (findBaseDir, relativeToBase')
import Hledger.Flow.Common (firstExistingFile, groupValuesBy, hledgerFileArgument, runProcessCaptured)
import Hledger.Flow.HledgerConfig (findConfigConflicts)
import Hledger.Flow.Import.ImportHelpers (defaultRulesCandidates, extractImportDirs, rulesFileCandidates)
import Hledger.Flow.Import.IncludeFiles (includesNeedUpdate)
import Hledger.Flow.Import.Inventory (classifyGeneratedOutputs, findImportFiles, generatedOutput, isManualJournal, managedIncludeFiles, stageFiles)
import Hledger.Flow.Import.Types (ImportDirs (..))
import Hledger.Flow.Init.Example (directivesAreStarter, journalIsCommentsOnly, sampleAccountDir, starterOwner, untouchedExample)
import Hledger.Flow.LedgerSelection
import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelFile, pathOccupied)
import Hledger.Flow.ReportReview (loadReportReview, reviewItems)
import Hledger.Flow.StatementScan (StatementYear (..), inferStatementYear, isDelimitedFile, readDelimitedStatement)
import Hledger.Flow.Status
import Hledger.Flow.Types (BaseDir, HledgerConf, HledgerInfo (..))
import Path
import qualified Path.IO as PathIO
import qualified System.Directory as Directory
import System.Exit (ExitCode (..))
import qualified System.FilePath as FilePath
import qualified System.Info as Sys
import System.IO.Error (tryIOError)
-- | Is this folder inside a ledger, empty, or something else?
classifyDir :: AbsDir -> IO Situation
classifyDir dir = do
found <- findBaseDir dir
case found of
Just base -> return (InLedger base)
Nothing -> do
(dirs, files) <- PathIO.listDirRel dir
let names = map (FilePath.dropTrailingPathSeparator . toFilePath) dirs ++ map toFilePath files
return (if all ("." `List.isPrefixOf`) names then EmptyDir else Elsewhere)
gatherStatus :: Maybe HledgerInfo -> BaseDir -> IO LedgerStatus
gatherStatus hledger = gatherStatusWithConfig hledger Nothing
gatherStatusWithConfig :: Maybe HledgerInfo -> Maybe HledgerConf -> BaseDir -> IO LedgerStatus
gatherStatusWithConfig hledger conf base = do
let root = base </> [reldir|import|]
files <- findImportFiles root
let statements = stageFiles root [reldir|1-in|] files
manualJournals = filter (isManualJournal root) files
groups = Map.toAscList (groupValuesBy (accountName root) statements)
accounts <- forM groups (uncurry (accountStatus base))
conflicts <- maybe (pure []) (\hl -> findConfigConflicts hl conf) hledger
selection <- selectLedger [] hledger base
let defaultJournal = base </> [relfile|all-years.journal|]
allYears <- PathIO.doesFileExist defaultJournal
let selected = aggregateJournal selection
(orphans, journals) <- classifyGeneratedOutputs root files statements
-- A newly initialised, empty ledger has no indexes to reconcile yet.
reconciliation <- if (allYears || isJust selected) && null orphans
then includesNeedUpdate base root files journals allYears
else return (not (null orphans) || not (null (managedIncludeFiles root files)))
needsPrices <- case (hledger, selected) of
(Just info, Just journal) | not reconciliation -> journalNeedsPrices info journal
_ -> return False
review <- maybe (pure Nothing) (\journal -> loadReportReview base journal conf) selected
owner <- starterOwner
examplePresent <- either (const True) id <$> tryIOError (pathOccupied (sampleAccountDir base owner))
let personal = [reldir|personal|]
alternate = if FilePath.equalFilePath (toFilePath owner) (toFilePath personal) then [reldir|me|] else personal
firstOwner = if examplePresent then alternate else owner
example <- case review of
Just current | null (reviewItems current) -> untouchedExample base statements
_ -> return Nothing
manualWork <- case manualJournals of
[] -> pure False
-- A wrapper can contribute journal entries of its own.
_ | not (null statements) || maybe False (/= defaultJournal) selected -> pure True
_ -> either (const True) not <$> tryIOError (onlyStarterJournals base manualJournals)
misplaced <- traverse (inspectMisplaced base) (List.sortOn mpPath (misplacedFiles root files))
return
LedgerStatus
{ lsBaseDir = base,
lsHledger = hledger,
lsLedgerFile = selectedLedgerFile selection,
lsAggregateJournal = selected,
lsHledgerConf = conf,
lsConfigConflicts = conflicts,
lsAccounts = accounts,
lsMisplaced = misplaced,
lsManualJournals = length manualJournals,
lsHasManualWork = manualWork,
lsAllYearsExists = allYears,
lsNeedsReconciliation = reconciliation,
lsNeedsPrices = needsPrices,
lsReview = review,
lsFirstStatementOwner = firstOwner,
lsUntouchedExample = example
}
-- The starter ledger contains declarations and comments only.
onlyStarterJournals :: BaseDir -> [AbsFile] -> IO Bool
onlyStarterJournals base [] = directivesAreStarter base
onlyStarterJournals base (file : rest) = do
commentOnly <- journalIsCommentsOnly file
if commentOnly then onlyStarterJournals base rest else return False
inspectMisplaced :: BaseDir -> Misplaced -> IO Misplaced
inspectMisplaced base item = do
evidence <- case mpYear item of
UnknownYear | isDelimitedFile (mpPath item), Just path <- parseRelFile (mpPath item) -> do
scanned <- tryIOError (readDelimitedStatement Nothing (base </> path))
return $ case scanned of
Right (Right statement) -> inferStatementYear statement
_ -> UnknownYear
known -> return known
let suggestion = case evidence of
SingleYear year -> FilePath.joinPath
[if part == "<year>" then show year else part | part <- FilePath.splitDirectories (mpSuggestion item)]
_ -> mpSuggestion item
enriched = item {mpYear = evidence, mpSuggestion = suggestion}
occupied <- case placementDestination enriched of
Nothing -> return False
Just path -> either (const True) id <$> tryIOError (pathOccupied (base </> path))
return enriched {mpDestinationExists = occupied}
-- Ask hledger to resolve includes and parse the journal. If it cannot read
-- the journal, status can still report what it found in the import tree.
journalNeedsPrices :: HledgerInfo -> AbsFile -> IO Bool
journalNeedsPrices info journal = do
commodities <- query ["commodities", "--used"]
case T.lines <$> commodities of
Just (_ : _ : _) -> (== Just "") . fmap T.strip <$> query ["prices"]
_ -> return False
where
query args = do
let executable = T.pack (toFilePath (hlPath info))
fileArgs = ["--no-conf", "--file", hledgerFileArgument info (toFilePath journal)]
result <- tryIOError (runProcessCaptured (executable, args ++ fileArgs, Nothing))
return $ case result of
Right (ExitSuccess, output, _) -> Just output
_ -> Nothing
accountName :: AbsDir -> AbsFile -> T.Text
accountName root file = case stripProperPrefix root file of
Just rel -> T.intercalate "/" (map T.pack (take 3 (FilePath.splitDirectories (toFilePath rel))))
Nothing -> T.pack (toFilePath file)
accountStatus :: BaseDir -> T.Text -> [AbsFile] -> IO AccountStatus
accountStatus base name statements = do
dirs <- case statements of
s : _ -> either (ioError . userError . T.unpack) return (extractImportDirs (base </> [reldir|import|]) s)
[] -> ioError (userError ("No statements for " ++ T.unpack name))
let relative = relativeToBase' (toFilePath base) . toFilePath
defaults <- defaultRulesCandidates dirs
rulesFile <- firstExistingFile defaults
preprocess <- hookFile (accountDir dirs) [relfile|preprocess|]
construct <- hookFile (accountDir dirs) [relfile|construct|]
directives <- firstExistingFile [base </> [relfile|directives.journal|]]
sources <- forM statements $ \s -> do
cached <- cachedStatement s
candidates <- rulesFileCandidates (fromMaybe s cached) dirs
rules <- firstExistingFile candidates
return (s, cached, rules, candidates)
let resolved = [rules | (_, _, rules, _) <- sources]
withoutRules = length (filter isNothing resolved)
(otherRules, targetOccupied) <- if withoutRules > 0 && isNothing construct
then do
other <- tryIOError (mainRulesCandidates (accountDir dirs) (concat [xs | (_, _, _, xs) <- sources]))
occupied <- case defaults of
target : _ -> either (const True) id <$> tryIOError (pathOccupied target)
_ -> return False
return (either (const []) id other, occupied)
else return ([], False)
-- Statements in an account commonly share one rules file.
ruleTimes <- case construct of
Just _ -> return Map.empty
Nothing -> traverse (tryIOError . latestRuleChange) $
Map.fromSet id (Set.fromList (catMaybes resolved))
journals <- forM sources $ \(s, cached, rules, _) -> do
let inputs = s : catMaybes [cached, preprocess, construct <|> directives]
journal <- generatedOutput [reldir|3-journal|] ".journal" s
journalState inputs (rules >>= (`Map.lookup` ruleTimes)) journal
return
AccountStatus
{ asAccount = name,
asStatements = length statements,
asYears = List.nub (List.sort (map yearOf statements)),
asRulesFile = relative <$> rulesFile,
asRulesCandidates = map relative defaults,
asWithoutRules = withoutRules,
asOtherRules = map relative otherRules,
asRulesTargetOccupied = targetOccupied,
asPreprocess = isJust preprocess,
asConstruct = isJust construct,
asJournalsMissing = length (filter (== Missing) journals),
asJournalsOutdated = length (filter (== Outdated) journals)
}
where
yearOf s = read (FilePath.dropTrailingPathSeparator (toFilePath (dirname (parent s))))
data JournalState = Current | Outdated | Missing deriving (Eq)
-- hledger interprets each include path relative to the file containing it.
latestRuleChange :: AbsFile -> IO UTCTime
latestRuleChange = visit Set.empty . toFilePath
where
visit ancestors file = do
key <- Directory.canonicalizePath file
when (Set.member key ancestors) $ ioError (userError ("cyclic rules include: " ++ file))
modified <- Directory.getModificationTime file
children <- ruleIncludes file
times <- mapM (visit (Set.insert key ancestors)) children
return (maximum (modified : times))
ruleIncludes :: FilePath -> IO [FilePath]
ruleIncludes file = do
contents <- T.readFile file
return [FilePath.takeDirectory file FilePath.</> T.unpack (T.strip path)
| line <- T.lines (T.dropWhile (== '\xFEFF') contents),
Just path <- [T.stripPrefix "include " line]]
mainRulesCandidates :: AbsDir -> [AbsFile] -> IO [AbsFile]
mainRulesCandidates dir recognized = do
(_, files) <- PathIO.listDir dir
let local = List.sort [file | file <- files, map toLower (FilePath.takeExtension (toFilePath file)) == ".rules"]
canonical = Directory.canonicalizePath . toFilePath
recognizedTargets <- Set.fromList <$> traverse canonical recognized
children <- concat <$> traverse (ruleIncludes . toFilePath) local
childTargets <- Set.fromList <$> traverse Directory.canonicalizePath children
pairs <- traverse (\file -> (,) file <$> canonical file) local
return [file | (file, target) <- pairs, Set.notMember target (Set.union recognizedTargets childTargets)]
journalState :: [AbsFile] -> Maybe (Either IOError UTCTime) -> AbsFile -> IO JournalState
journalState inputs rulesTime journal = do
exists <- PathIO.doesFileExist journal
if exists
then do
-- Keep filesystem precision so a rule edited just after import is noticed.
newer <- tryIOError $ do
written <- PathIO.getModificationTime journal
times <- mapM PathIO.getModificationTime inputs
return (any (> written) times || maybe False (either (const True) (> written)) rulesTime)
return (if either (const True) id newer then Outdated else Current)
else return Missing
-- | Import reads this statement's cached CSV when present, even without a hook.
cachedStatement :: AbsFile -> IO (Maybe AbsFile)
cachedStatement statement = do
csv <- generatedOutput [reldir|2-preprocessed|] ".csv" statement
firstExistingFile [csv]
-- | Same hook names import looks for.
hookExists :: AbsDir -> RelFile -> IO Bool
hookExists dir hook = isJust <$> hookFile dir hook
hookFile :: AbsDir -> RelFile -> IO (Maybe AbsFile)
hookFile dir hook = do
let base = dir </> hook
alternatives <-
if Sys.os == "mingw32"
then mapM (`addExtension` base) [".ps1", ".exe"]
else return []
firstExistingFile (base : alternatives)