packages feed

hledger-flow-0.20.1: src/Hledger/Flow/Import/Conversion.hs

{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
module Hledger.Flow.Import.Conversion
  ( findHook, preprocess, savedInput, logNewFileSkip, hledgerImport, customConstruct
  , hookDirArgs, RulesPaths(..), convertRules, constructWith, rulesArguments, constructArguments, pinnedConfig
  ) where

import Control.Concurrent.STM
import Control.Monad
import qualified Data.Text as T
import Hledger.Flow.DateTime (timeAction)
import Hledger.Flow.BaseDir (relativeToBase)
import Hledger.Flow.Common
import Hledger.Flow.DocHelpers (DocLink (..), docURL)
import Hledger.Flow.Import.ImportHelpers
import Hledger.Flow.Import.Types
import Hledger.Flow.Logging
import Hledger.Flow.PathHelpers (AbsFile, isGlobMetacharacter)
import Hledger.Flow.RuntimeOptions
import qualified Hledger.Flow.Types as FlowTypes
import qualified Path
import qualified Path.IO as PathIO
import System.Environment (getEnv)
import System.Exit (ExitCode (..))
import Control.Concurrent.Async (concurrently)
import qualified Data.Text.IO as T
import System.IO (hClose, stderr)
import qualified System.Info as Sys
import qualified System.Process as Process
import System.FilePath (takeExtension, (</>))
import Prelude hiding (putStrLn, take, writeFile)

findHook :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> AbsFile -> IO (Maybe AbsFile)
findHook opts ch what base = do
  alternatives <- if Sys.os == "mingw32"
    then mapM (`Path.addExtension` base) [".ps1", ".exe"]
    else return []
  findFirstFile opts ch what (base : alternatives)

hookCommand :: AbsFile -> [T.Text] -> IO (T.Text, [T.Text])
hookCommand script args
  | Sys.os == "mingw32" && takeExtension path == ".ps1" = do
      root <- getEnv "SystemRoot"
      let shell = root </> "System32" </> "WindowsPowerShell" </> "v1.0" </> "powershell.exe"
      -- Windows PowerShell -File treats construct's literal "-" as an option.
      -- Invoke the script with literal strings and propagate its exit status.
      let command = "[Console]::OutputEncoding = New-Object Text.UTF8Encoding($false); & "
            <> T.unwords (map powerShellLiteral (T.pack path : args))
            <> "; if (!$?) { if ($LASTEXITCODE) { exit $LASTEXITCODE }; exit 1 }"
      return (T.pack shell, ["-NoProfile", "-NonInteractive", "-OutputFormat", "Text", "-Command", command])
  | otherwise = return (T.pack path, args)
  where
    path = Path.toFilePath script

logNewFileSkip :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> AbsFile -> IO ()
logNewFileSkip opts ch logIdentifier absTarget =
  Control.Monad.when (onlyNewFiles opts) $ do
    let relativeTarget = relativeToBase opts (Path.toFilePath absTarget)
    logVerbose opts ch $
      "Skipping " <> logIdentifier <> ": " <> T.pack relativeTarget <> " is at least as new as its source"

-- | Run the preprocess script, writing to the staged file. The command shown names the final
-- output instead, so a pasted command writes where hledger-flow does.
preprocess :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
preprocess opts ch script dirs src csvOut staged = do
  let args out = map (T.pack . Path.toFilePath) [src, out] ++ hookDirArgs dirs
      relSrc = relativeToBase opts (Path.toFilePath src)
      external = External (relativeHook opts script) ("preprocess " <> T.pack relSrc)
        [T.pack (Path.toFilePath script) : args csvOut] []
  (cmd, hookArgs) <- hookCommand script (args staged)
  void $ runExternal opts ch external True (cmd, hookArgs, Nothing)

relativeHook :: RuntimeOptions -> AbsFile -> T.Text
relativeHook opts = T.pack . relativeToBase opts . Path.toFilePath

savedInput :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> AbsFile -> IO AbsFile
savedInput opts ch src csvOut = do
  cached <- verboseTestFile opts ch "preprocessed CSV from an earlier run" csvOut
  if cached
    then do
      stale <- needsRegeneration src csvOut
      when stale $ ioError $ userError $
        relativeToBase opts (Path.toFilePath src) ++ " is newer than its cached CSV "
          ++ relativeToBase opts (Path.toFilePath csvOut) ++ ". Restore the preprocess script to regenerate it,"
          ++ " or move the cached CSV aside to use the source directly. Then run hledger-flow import."
      return csvOut
    else return src

hledgerImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()
hledgerImport opts ch importDirs csvFile journalOut = do
  let csvSrc = Path.toFilePath csvFile
  candidates <- rulesFileCandidates csvFile importDirs
  maybeRulesFile <- findFirstFile opts ch "rules file" candidates
  let relCSV = relativeToBase opts csvSrc
  case maybeRulesFile of
    Just rf -> do
      let directives = directivesFile opts
      exists <- PathIO.doesFileExist directives
      let paths = RulesPaths csvFile rf (if exists then Just directives else Nothing)
      stdOut <- convertRules opts ch (hledgerExecutable opts) paths paths (hledgerConfArgs opts) pure
      writeGeneratedText (Path.toFilePath journalOut) stdOut
    Nothing ->
      do
        let candidatesTxt = T.intercalate "\n" $ map (T.pack . relativeToBase opts . Path.toFilePath) candidates
        let msg =
              "I couldn't find an hledger rules file while trying to import\n" <> T.pack relCSV
                <> "\n\nI will happily use the first rules file I can find from any one of these "
                <> T.pack (show (length candidates)) <> " files:\n" <> candidatesTxt
                <> "\n\nHere is a bit of documentation about rules files that you may find helpful:\n"
                <> docURL RulesFiles
        errExit 1 ch msg

data RulesPaths = RulesPaths AbsFile AbsFile (Maybe AbsFile)

-- The journal cache uses these same argument builders with stable path labels.
pinnedConfig :: Maybe T.Text -> [T.Text]
pinnedConfig = maybe ["--no-conf"] (\path -> ["--conf", path])

rulesArguments :: [T.Text] -> T.Text -> T.Text -> Maybe T.Text -> [T.Text]
rulesArguments conf input rules directives =
  conf ++ ["print", "--explicit"] ++ maybe [] (\path -> ["--file", path]) directives
       ++ ["--file", input, "--rules-file", rules]

constructArguments :: [T.Text] -> T.Text -> [T.Text]
constructArguments conf out = conf ++ ["print", "--ignore-assertions", "--file", "-", "--output-file", out]

convertRules :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> RulesPaths -> RulesPaths -> [T.Text]
             -> (T.Text -> IO T.Text) -> IO T.Text
convertRules opts ch executable original@(RulesPaths csv rules _) actual config translate = do
  let shown = T.pack . relativeToBase opts . Path.toFilePath
      hledger = T.pack (Path.toFilePath (hledgerExecutable opts))
      args conf (RulesPaths input rf directives) = do
        let paths = input : maybe [] pure directives
        fileArg <- if any (any isGlobMetacharacter . Path.toFilePath) paths
          then hledgerFileArgument <$> hledgerInfo opts else pure T.pack
        pure $ rulesArguments conf (fileArg (Path.toFilePath input)) (T.pack (Path.toFilePath rf))
                 (fileArg . Path.toFilePath <$> directives)
  command <- args (hledgerConfArgs opts) original
  invocation <- args config actual
  let ext = External "hledger" ("convert " <> shown csv <> " with the rules in " <> shown rules)
              [hledger : command] ["Check the skip, fields and separator lines in " <> shown rules <> ".", "Docs: " <> docURL RulesFiles]
  announceExternal opts ch ext
  ((status,out,err),duration) <- timeAction $ runProcessCaptured (T.pack (Path.toFilePath executable),invocation,Nothing)
  diagnostic <- translate err
  diagnosticOut <- if status == ExitSuccess then pure out else translate out
  checkExternal opts ch ext False duration (status,diagnosticOut,diagnostic)
  pure out

hookDirArgs :: ImportDirs b -> [T.Text]
hookDirArgs dirs = map (T.concat . importDirLines) [bankDir dirs, accountDir dirs, ownerDir dirs]

-- | Pipe the construct script's output through hledger into the staged journal. The command
-- shown names the final journal instead, so a pasted command writes where hledger-flow does.
customConstruct :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
customConstruct opts ch = constructWith opts ch (hledgerExecutable opts) (hledgerConfArgs opts) pure

constructWith :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> [T.Text] -> (T.Text -> IO T.Text) -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
constructWith opts ch executable config translate constructScript dirs csvFile journalOut staged = do
  let csvSrc = Path.toFilePath csvFile
      relScript = relativeHook opts constructScript
      relSrc = T.pack (relativeToBase opts csvSrc)
      constructArgs = [T.pack csvSrc, "-"] ++ hookDirArgs dirs
      hledger = T.pack (Path.toFilePath (hledgerExecutable opts))
      hledgerArgs conf = constructArguments conf . T.pack . Path.toFilePath
      shown = [T.pack (Path.toFilePath constructScript) : constructArgs, hledger : hledgerArgs (hledgerConfArgs opts) journalOut]
      constructExt = External relScript ("construct a journal from " <> relSrc) shown []
      hledgerExt = External "hledger" ("read the journal " <> relScript <> " constructed from " <> relSrc) shown []
  (cmd, hookArgs) <- hookCommand constructScript constructArgs
  let hook = (Process.proc (T.unpack cmd) (map T.unpack hookArgs))
        { Process.std_in = Process.CreatePipe, Process.std_out = Process.CreatePipe, Process.std_err = Process.CreatePipe }
  announceExternal opts ch constructExt
  Process.withCreateProcess hook $ \input output errors handle -> do
    Just stdinPipe <- return input
    Just stdoutPipe <- return output
    Just stderrPipe <- return errors
    hClose stdinPipe
    (((hledgerStatus, converterOut, converterErr), duration), hookErr) <- concurrently
      (timeAction (runProcessCaptured (T.pack (Path.toFilePath executable), hledgerArgs config staged, Just stdoutPipe)))
      (T.hGetContents stderrPipe)
    mappedErr <- translate converterErr
    let converted = (hledgerStatus, converterOut, mappedErr)
    status <- Process.waitForProcess handle
    color <- useColor stderr
    -- When hledger stops reading early, the script fails with a broken pipe, so hledger's error
    -- comes first, along with anything the script said.
    let scriptNote = case status of
          ExitFailure code -> [relScript <> " also failed, with exit code " <> T.pack (show code) <> "."]
          ExitSuccess -> []
        scriptOutput = outputSection color (verbose opts) ("Error output from " <> relScript) hookErr
    when (hledgerStatus /= ExitSuccess) $
      checkExternal opts ch hledgerExt {extHints = scriptNote ++ scriptOutput} True duration converted
    checkExternal opts ch constructExt True duration (status, "", hookErr)
    checkExternal opts ch hledgerExt True duration converted