packages feed

hledger-flow-0.21.0: src/Hledger/Flow/RulesTemplate.hs

{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}

-- | A starter hledger rules file built from a statement's header row.
module Hledger.Flow.RulesTemplate
  ( splitHeader,
    looksLikeHeader,
    templateFields,
    rulesTemplate,
    rulesTemplateFor,
    rulesStarterFor,
    runRules,
  )
where

import Data.Char (isAlpha, isAlphaNum, isDigit, toLower)
import qualified Data.List as List
import Data.Maybe (fromMaybe)
import qualified Data.Set as Set
import qualified Data.Text as T
import Hledger.Flow.BaseDir (relativeToBase')
import Hledger.Flow.DocHelpers (DocLink (..), docURL)
import Hledger.Flow.Common (homeDirs, nativeShell, tildePath)
import Hledger.Flow.Import.ImportHelpers (defaultRulesCandidates, extractImportDirs)
import Hledger.Flow.Import.Inventory (findImportFiles, stageFiles)
import Hledger.Flow.Init.Example (isSampleStatement)
import Hledger.Flow.PathHelpers (AbsFile, writeNewUtf8File)
import Hledger.Flow.StatementScan
import Hledger.Flow.Status (flowCommand)
import Hledger.Flow.Status.Gather (accountName, cachedStatement)
import Hledger.Flow.Types (BaseDir, HledgerConf)
import Path
import qualified Path.IO as PathIO
import qualified Data.Text.IO as T
import System.IO (Newline (..), nativeNewline)
import System.IO.Error (tryIOError, isAlreadyExistsError)

-- | hledger field names for the columns: obvious date, description and amount
-- columns are named for hledger; the rest keep a normalised version of their name.
templateFields :: [T.Text] -> [T.Text]
templateFields names = dedupe (zipWith pick [0 ..] normalised)
  where
    normalised = zipWith normalise [1 :: Int ..] names
    normalise i n =
      let t = T.intercalate "_" (filter (not . T.null) (T.splitOn "_" (T.map (\c -> if isAlphaNum c then toLower c else '_') n)))
       in if T.null t then "field" <> T.pack (show i) else t
    indexed = zip [0 :: Int ..] normalised
    dateIx = fst <$> List.find (("date" `T.isInfixOf`) . snd) indexed
    descIx = fst <$> List.find (\(i, n) -> Just i /= dateIx && any (`T.isInfixOf` n) ["desc", "payee", "narrat", "detail"]) indexed
    amountIx = case [i | (i, n) <- indexed, "amount" `T.isInfixOf` n, Just i /= dateIx, Just i /= descIx] of
      [i] -> Just i
      _ -> fst <$> List.find ((== "amount") . snd) indexed
    pick i n
      | Just i == dateIx = "date"
      | Just i == descIx = "description"
      | Just i == amountIx = "amount"
      | isSpecialField n = "bank_" <> n
      | otherwise = n
    dedupe = snd . List.mapAccumL unique Set.empty
      where
        unique seen name = (Set.insert chosen seen, chosen)
          where
            chosen = available name (2 :: Int)
            available candidate suffix
              | Set.notMember candidate seen = candidate
              | otherwise = available (name <> "_" <> T.pack (show suffix)) (suffix + 1)

-- | Names hledger gives a meaning of its own, such as balance (assertions) or status.
isSpecialField :: T.Text -> Bool
isSpecialField n =
  n `elem` ["status", "code", "comment", "balance", "currency", "date2", "account", "amount", "date", "description"]
    || (T.dropWhileEnd isDigit n `elem` ["account", "amount", "balance", "comment", "currency"] && T.any isDigit n)

notCsv :: FilePath -> T.Text
notCsv path =
  T.pack path <> " isn't a CSV file, so there's no header row to build rules from. "
    <> "A preprocess script can convert it to CSV first: "
    <> docURL PreprocessScript

-- | The starter rules file for a statement, from its contents.
rulesTemplate :: T.Text -> T.Text -> FilePath -> T.Text -> Either T.Text T.Text
rulesTemplate bank account path contents
  | not (isDelimitedFile path) = Left (notCsv path)
  | otherwise = renderTemplate bank account path (parseRuleSample path contents)

renderTemplate :: T.Text -> T.Text -> FilePath -> Either T.Text DelimitedStatement -> Either T.Text T.Text
renderTemplate bank account path result = case result of
  Left problem -> Left (T.pack path <> ": " <> problem <> " Write the fields line yourself: " <> docURL RulesFiles)
  Right statement -> case statementHeader statement of
    Nothing -> headerlessTemplate bank account path statement
    Just names ->
      let fields = templateFields names
       in Right . T.unlines $
        [ "# Starter rules for " <> bank <> "/" <> account <> ", made by hledger-flow from the header row of",
          "# " <> T.pack path <> ". Check every line, then run: hledger-flow import",
          "# hledger's CSV rules reference: https://hledger.org/hledger.html#csv",
          "",
          "skip 1"
        ]
          ++ separatorRule (statementSeparator statement)
          ++ [ "fields " <> T.intercalate ", " fields,
               ""
             ]
          ++ [ "# Columns prefixed with bank_ would otherwise mean something to hledger.\n"
                 <> "# For example, rename bank_balance to balance to check balance assertions.\n"
               | any (\f -> maybe False isSpecialField (T.stripPrefix "bank_" f)) fields
             ]
          ++ [
               "# Make the date format match your statement, e.g. %d/%m/%Y",
               "# date-format %Y-%m-%d",
               "",
               "# Separate debit and credit columns? Name them amount-out and amount-in",
               "# in the fields line instead of amount.",
               "",
               "account1 assets:" <> bank <> ":" <> account,
               "account2 expenses:unknown"
             ]
separatorRule :: Char -> [T.Text]
separatorRule sep = ["separator " <> (if sep == '\t' then "TAB" else T.singleton sep) | sep /= ',']

headerlessTemplate :: T.Text -> T.Text -> FilePath -> DelimitedStatement -> Either T.Text T.Text
headerlessTemplate bank account path statement = case statementRows statement of
  [] -> Left ("No data rows in " <> T.pack path <> ".")
  sample : _ ->
    let columns = zip [1 :: Int ..] (List.transpose (statementRows statement))
        dateCandidates = [(n, cells) | (n, column) <- columns, Just cells <- [traverse parseDateCell column]]
        dateChoice = unique dateCandidates
        amountChoice = unique [n | (n, column) <- columns, all isSignedDecimal column]
        descriptionChoice = unique
          [ n | (n, column) <- columns, all (T.any isAlpha) column, Set.size (Set.fromList column) > 1
          ]
        field n
          | Just n == fmap fst dateChoice = "date"
          | Just n == amountChoice = "amount"
          | Just n == descriptionChoice = "description"
          | otherwise = columnName n
        dateRule = case dateChoice of
          Just (n, cells) -> case inferDateFormat cells of
            Just "%Y-%m-%d" -> []
            Just layout -> ["date-format " <> layout]
            Nothing -> ["# date-format (set the order and separator for " <> sampleValue n sample <> ")"]
          Nothing -> ["# Rename the transaction date column to date, then set date-format if needed."]
     in Right . T.unlines $
          [ "# Starter rules for " <> bank <> "/" <> account <> ", made by hledger-flow from",
            "# " <> T.pack path <> ", which has no header row.",
            "# The first row, by column:"
          ]
            ++ concat (zipWith commentValue [1 :: Int ..] sample)
            ++ ["# Check every line, then run: hledger-flow import", ""]
            ++ separatorRule (statementSeparator statement)
            ++ ["fields " <> T.intercalate ", " (map (field . fst) columns)]
            ++ dateRule
            ++ ["# Rename the amount column to amount." | amountChoice == Nothing]
            ++ ["# Rename the payee column to description." | descriptionChoice == Nothing]
            ++ ["", "account1 assets:" <> bank <> ":" <> account, "account2 expenses:unknown"]
  where
    unique [one] = Just one
    unique _ = Nothing
    columnName n = "column" <> T.pack (show n)
    sampleValue n = T.filter (/= '\n') . T.intercalate " " . take 1 . drop (n - 1)
    commentValue n value = case T.splitOn "\n" value of
      first : rest -> ("#   " <> columnName n <> " =" <> if T.null first then "" else " " <> first)
        : map ("#     " <>) rest
      [] -> []

-- | The first default rules candidate and the starter for the newest statement,
-- skipping init's sample statement when the account also has real ones.
rulesStarterFor :: BaseDir -> T.Text -> IO (Either T.Text (AbsFile, T.Text))
rulesStarterFor base requested = do
  let account = T.dropWhileEnd (== '/') (fromMaybe requested (T.stripPrefix "import/" requested))
      root = base </> [reldir|import|]
  files <- findImportFiles root
  let inputs = stageFiles root [reldir|1-in|] files
      found = filter ((== account) . accountName root) inputs
      statements = case filter (not . isSampleStatement) found of
        [] -> found
        real -> real
      known = List.nub (map (accountName root) inputs)
  case reverse statements of
    [] -> return . Left $
      "No statements found for " <> account <> ". Accounts with statements: "
        <> (if null known then "none" else T.intercalate ", " known) <> "."
    newest : _ -> case extractImportDirs root newest of
      Left problem -> return (Left problem)
      Right dirs -> do
        candidates <- defaultRulesCandidates dirs
        case candidates of
          target : _ -> do
            source <- fromMaybe newest <$> cachedStatement newest
            let path = relativeToBase' (toFilePath base) (toFilePath source)
                (bank, acct) = case T.splitOn "/" account of
                  [_, b, a] -> (b, a)
                  _ -> (account, account)
            if isDelimitedFile path
              then fmap (fmap (\template -> (target, template)) . renderTemplate bank acct path) (readRuleSample source)
              else return (Left (notCsv path))
          [] -> return (Left ("No rules file target for " <> account <> "."))

rulesTemplateFor :: BaseDir -> T.Text -> IO (Either T.Text T.Text)
rulesTemplateFor base = fmap (fmap snd) . rulesStarterFor base

runRules :: Maybe HledgerConf -> BaseDir -> T.Text -> IO ()
runRules conf base account = do
  result <- rulesStarterFor base account
  (target, template) <- either (ioError . userError . T.unpack) return result
  cwd <- PathIO.getCurrentDir
  homes <- homeDirs
  let path = toFilePath target
      shown = T.pack (if cwd == base then relativeToBase' (toFilePath base) path else tildePath homes path)
      command sub args = flowCommand nativeShell homes cwd base conf sub args
      -- Match the native line endings of status --rules-template on stdout.
      fileTemplate = case nativeNewline of
        LF -> template
        CRLF -> T.replace "\n" "\r\n" template
  written <- tryIOError (writeNewUtf8File target fileTemplate)
  case written of
    Left err -> do
      occupied <- PathIO.doesFileExist target
      let problem
            | occupied || isAlreadyExistsError err =
                shown <> " already exists. Delete it first or preview a fresh starter with: "
                  <> command "status" ["--rules-template", account]
            | otherwise = "Could not create " <> shown <> ": " <> T.pack (show err)
      ioError (userError (T.unpack problem))
    Right () -> do
      T.putStrLn ("Created " <> shown)
      T.putStrLn ("Edit it: check the fields line and the account names. Then run: " <> command "import" [])