hledger-flow-0.20.1: src/Hledger/Flow/Import/CSVImport.hs
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
module Hledger.Flow.Import.CSVImport
( importCSVs,
)
where
import Control.Concurrent.STM
import Control.Monad
import Data.Maybe (fromMaybe)
import qualified Data.Text as T
import Hledger.Flow.DateTime (timeAction)
import Hledger.Flow.Environment (importSettings, renderSettings)
import Hledger.Flow.BaseDir (effectiveRunDir, relativeToBase)
import Hledger.Flow.Common
import Hledger.Flow.Import.Incremental
import Hledger.Flow.Import.Conversion
import Hledger.Flow.Import.ImportHelpers
import Hledger.Flow.Import.IncludeFiles (writeIncludesFromInventory, writeToplevelAllYearsInclude)
import Hledger.Flow.Import.Inventory
import Hledger.Flow.Import.Types
import Hledger.Flow.Logging
import Hledger.Flow.PathHelpers (AbsDir, AbsFile, isGlobMetacharacter)
import Hledger.Flow.RuntimeOptions
import qualified Hledger.Flow.Types as FlowTypes
import qualified Path
import qualified Path.IO as PathIO
import System.FilePath (dropTrailingPathSeparator)
import qualified Data.List as List
import Hledger.Flow.Status (plural)
import Hledger.Flow.Status.Gather (accountName)
import Prelude hiding (putStrLn, take, writeFile)
type FileWasGenerated = Bool
importCSVs :: RuntimeOptions -> IO ()
importCSVs opts = withConsoleLogging $ \ch -> do
homes <- homeDirs
when (showOptions opts) $ do
info <- hledgerInfo opts
channelOutLn ch (T.unlines (renderSettings id (importSettings homes info opts)))
logVerbose opts ch $ "Base directory: " <> T.pack (tildePath homes (dropTrailingPathSeparator (Path.toFilePath (baseDir opts))))
(journals, diff) <- timeAction $ importCSVs' opts ch
let importRoot = baseDir opts Path.</> [Path.reldir|import|]
effectiveDir = effectiveRunDir (baseDir opts) (importRunDir opts)
scope = T.pack (dropTrailingPathSeparator (relativeToBase opts (Path.toFilePath effectiveDir)))
accounts = List.nub (map (accountName importRoot . fst) journals)
when (verbose opts) $ channelErr ch "\n"
channelOutLn ch (importSummary scope (length journals) (length (filter snd journals)) accounts)
logVerbose opts ch ("Import finished in " <> showDuration diff)
-- | One line saying what import did.
importSummary :: T.Text -> Int -> Int -> [T.Text] -> T.Text
importSummary scope total generated accounts
| total == 0 = "No statements to import in " <> scope <> "."
| generated == total = "Imported " <> plural total "statement" <> " for " <> accountsText <> "."
| otherwise =
"Imported " <> T.pack (show generated) <> " of " <> plural total "statement" <> " for " <> accountsText
<> " (" <> T.pack (show (total - generated)) <> " already up to date)."
where
accountsText
| length accounts <= 3 = T.intercalate ", " accounts
| otherwise = plural (length accounts) "account"
importCSVs' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> IO [(AbsFile, FileWasGenerated)]
importCSVs' opts ch = do
let effectiveDir = effectiveRunDir (baseDir opts) (importRunDir opts)
importRoot = baseDir opts Path.</> [Path.reldir|import|]
selected = inImportScope effectiveDir (fromMaybe 0 (importStartYear opts))
let startYearMsg = maybe "" (\y -> " from " <> T.pack (show y) <> " onwards") (importStartYear opts)
let relative = T.pack . relativeToBase opts . Path.toFilePath
logVerbose opts ch $ "Looking for statements" <> startYearMsg <> " in " <> relative effectiveDir
<> " (import/<owner>/<bank>/<account>/1-in/<year>/)"
(inventory, diff) <- timeAction $ findImportFiles importRoot
let inputFiles = filter selected (stageFiles importRoot [Path.reldir|1-in|] inventory)
fileCount = length inputFiles
collisions <- conflictingOutputs inputFiles
unless (null collisions) $
errExit 1 ch ("Multiple input files map to the same output:\n" <>
T.intercalate "\n" [relative output <> " <- " <> T.intercalate ", " (map relative sources)
| (output, sources) <- collisions])
logVerbose opts ch $ "Found " <> plural fileCount "statement" <> " in " <> showDuration diff
-- Glob escaping depends on the hledger version. Check it before any worker
-- writes output; conversions reuse this result. directives.journal and
-- cached CSVs share these paths' base and statement names.
when (any (any isGlobMetacharacter . Path.toFilePath) inputFiles) $ void (hledgerVersion opts)
let run tracking = parAwareActions opts (map (extractAndImport tracking opts ch importRoot) inputFiles)
outcomes <- case importPolicy opts of
RunScripts -> withIncrementalRun opts ch fileCount (run . Just)
SkipUnchangedScripts -> withIncrementalRun opts ch fileCount (run . Just)
_ -> run Nothing
let importedJournals = [(journal, generated) | (journal, generated, _) <- outcomes]
cached = if or [refresh | (_, _, refresh) <- outcomes] then Nothing else Just inventory
((refreshed, orphans, journalsOnDisk), journalFindTime) <- timeAction $ reconciliationInventory importRoot cached
forM_ (filter selected orphans) $ \file -> do
target <- moveToTrash file
channelOutLn ch $ "Moved orphaned output " <> relative file <> " to " <> relative target
withLogBlock opts True ch "Include files" $ \_ -> do
(_, writeIncludeTime1) <- timeAction $ writeIncludesFromInventory opts ch importRoot refreshed journalsOnDisk
(_, writeIncludeTime2) <- timeAction $ writeToplevelAllYearsInclude opts
let includeGenTime = journalFindTime + writeIncludeTime1 + writeIncludeTime2
logVerbose opts ch $ "Updated include files for " <> plural (length journalsOnDisk) "generated journal" <> " in " <> showDuration includeGenTime
return importedJournals
extractAndImport :: Maybe IncrementalRun -> RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsDir -> AbsFile -> IO (AbsFile, Bool, Bool)
extractAndImport tracking opts parent root inputFile =
withLogBlock opts (sequential opts) parent ("Statement " <> T.pack (relativeToBase opts (Path.toFilePath inputFile))) $ \ch ->
case extractImportDirs root inputFile of
Right importDirs -> case tracking of
Just run -> importStatement run opts ch importDirs inputFile
Nothing -> do
(output, generated) <- importCSV opts ch importDirs inputFile
pure (output, generated, generated)
Left errorMessage -> do
errExit 1 ch errorMessage
importCSV :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> IO (AbsFile, FileWasGenerated)
importCSV opts ch importDirs srcFile = do
csvOut <- generatedOutput [Path.reldir|2-preprocessed|] ".csv" srcFile
(csvFile, preprocessHappened) <- preprocessIfNeeded opts ch importDirs srcFile csvOut
journalOut <- generatedOutput [Path.reldir|3-journal|] ".journal" csvFile
shouldImport <-
if onlyNewFiles opts && not preprocessHappened
then needsRegeneration csvFile journalOut
else return True
PathIO.createDirIfMissing True (Path.parent journalOut)
if shouldImport
then do
hook <- findHook opts ch "construct script" (accountDir importDirs Path.</> [Path.relfile|construct|])
case hook of
Just script -> withStagedOutput journalOut (customConstruct opts ch script importDirs csvFile journalOut)
Nothing -> hledgerImport opts ch importDirs csvFile journalOut
logVerbose opts ch ("Wrote " <> T.pack (relativeToBase opts (Path.toFilePath journalOut)))
else logNewFileSkip opts ch "import" journalOut
return (journalOut, shouldImport)
preprocessIfNeeded :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO (AbsFile, Bool)
preprocessIfNeeded opts ch dirs src csvOut = do
hook <- findHook opts ch "preprocess script" (accountDir dirs Path.</> [Path.relfile|preprocess|])
case hook of
Nothing -> do
effective <- savedInput opts ch src csvOut
return (effective, False)
Just script -> do
regenerate <- if onlyNewFiles opts then needsRegeneration src csvOut else return True
if regenerate
then do
withStagedOutput csvOut (preprocess opts ch script dirs src csvOut)
logVerbose opts ch ("Wrote " <> T.pack (relativeToBase opts (Path.toFilePath csvOut)))
else logNewFileSkip opts ch "preprocess" csvOut
return (csvOut, regenerate)