packages feed

hledger-flow-0.20.1: src/Hledger/Flow/Import/Incremental.hs

{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
{-# LANGUAGE TupleSections #-}

module Hledger.Flow.Import.Incremental
  ( IncrementalRun, withIncrementalRun, importStatement ) where

import Control.Concurrent.STM (TChan)
import Control.Exception (finally)
import Control.Monad (unless, when)
import Data.IORef
import qualified Data.Map.Strict as Map
import Data.Maybe (maybeToList)
import qualified Data.Set as Set
import qualified Data.Text as T
import Hledger.Flow.BaseDir (relativeToBase)
import Hledger.Flow.Common (directivesFile, writeStagedText)
import Hledger.Flow.DocHelpers
import Hledger.Flow.Cache (initializeCache)
import Hledger.Flow.Import.Cache
import Hledger.Flow.Import.Conversion
import Hledger.Flow.Import.Dependencies
import Hledger.Flow.Import.Fingerprint
import Hledger.Flow.Import.Inventory (generatedOutput)
import Hledger.Flow.Import.Snapshot
import Hledger.Flow.Import.Types
import Hledger.Flow.Logging
import Hledger.Flow.PathHelpers (AbsFile, withStagedOutput)
import Hledger.Flow.RuntimeOptions
import Hledger.Flow.Types (LogMessage)
import qualified Path as P
import qualified Path.IO as P
import qualified System.Directory as D
import System.IO.Error (tryIOError)
import qualified System.Info as Sys

-- Keep one path only when every occurrence of a reason names that file.
data Notices = Notices !(Set.Set AbsFile) !(Map.Map T.Text (Maybe T.Text))
data IncrementalRun = IncrementalRun SnapshotRun (IO (ReuseDecision Converter)) (IORef Notices)

withIncrementalRun :: RuntimeOptions -> TChan LogMessage -> Int -> (IncrementalRun -> IO a) -> IO a
withIncrementalRun opts ch count action = withSnapshotRun (baseDir opts) $ \snapshots -> do
  reasons <- newIORef (Notices Set.empty Map.empty)
  probe <- newConverterProbe opts
  when (count > 0) $ do
    initialized <- tryIOError $ initializeCache (baseDir opts)
    either (channelErrLn ch . ("Could not initialize incremental cache: " <>) . T.pack . show) pure initialized
  finally (action (IncrementalRun snapshots probe reasons)) $ do
    Notices affected warnings <- readIORef reasons
    unless (Map.null warnings) $ do
      let scope | Set.size affected == count = "all selected statements"
                | otherwise = T.pack (show (Set.size affected)) <> " of " <> T.pack (show count) <> " selected statements"
          describe (why,path) = why <> maybe "" (": " <>) path
      channelErrLn ch $ "Incremental reuse unavailable for " <> scope <> ": "
        <> T.intercalate ", " (map describe (Map.toAscList warnings)) <> ". See " <> docURL IncrementalImportFallback

data HookInputs = HookInputs AbsFile FileObservation AbsFile FileObservation
  deriving (Eq, Show)

observeHook :: AbsFile -> AbsFile -> IO (ReuseDecision HookInputs)
observeHook script source = checkedIO "could not check script inputs" (P.toFilePath script) $ do
  runnable <- if Sys.os == "mingw32" then pure True else D.executable <$> D.getPermissions (P.toFilePath script)
  if runnable
    then Reusable <$> (HookInputs script <$> observeFile script <*> pure source <*> observeFile source)
    else pure $ Fallback (OrdinaryInputError "script is not executable" (pathText script))

hookInputsKey :: ImportDirs P.Abs -> FilePath -> T.Text -> HookInputs -> StageInputs
hookInputsKey dirs cwd stage (HookInputs script s input i) =
  [("stage",stage),("working directory",T.pack cwd),("hook",pathText script),
   ("hook digest",digestHex (observationDigest s)),("input",pathText input),
   ("input digest",digestHex (observationDigest i))] ++ map ("argument",) (hookDirArgs dirs)

pathText :: AbsFile -> T.Text
pathText = T.pack . P.toFilePath

data JournalInputs = JournalInputs
  { engine :: Converter
  , copiedInputs :: SnapshotInput
  , recordedInputs :: StageInputs
  , copiedDigests :: Map.Map AbsFile Digest
  , liveConstruct :: Maybe HookInputs
  } deriving (Eq, Show)

importStatement :: IncrementalRun -> RuntimeOptions -> TChan LogMessage -> ImportDirs P.Abs -> AbsFile -> IO (AbsFile,Bool,Bool)
importStatement (IncrementalRun snapshots probe reasons) opts ch dirs source = do
  cwd <- D.getCurrentDirectory
  record <- recordPath (baseDir opts) source
  old <- readRecord record
  intermediate <- generatedOutput [P.reldir|2-preprocessed|] ".csv" source
  let hook name = findHook opts ch name (accountDir dirs P.</> (if name == "preprocess" then [P.relfile|preprocess|] else [P.relfile|construct|]))
      report OrdinaryInputError{} = pure ()
      report problem = do
        let reason = reasonScope problem
            path = case problem of
              FileReason _ file _ -> Just (T.pack (relativeToBase opts file))
              _ -> Nothing
            detail = reasonDetail problem
            common a b = if a == b then a else Nothing
        atomicModifyIORef' reasons $ \(Notices files summaries) ->
          (Notices (Set.insert source files) (Map.insertWith common reason path summaries), ())
        logVerbose opts ch $ T.intercalate " " (filter (not . T.null) ([reason] ++ maybeToList path ++ [detail]))
      reuseStage stage inputs target prior = do
        miss <- stageMiss inputs target prior
        case miss of
          Nothing -> pure True
          Just why -> logVerbose opts ch (stage <> ": " <> why) >> pure False
      changed = FileReason "dependency changed during conversion" (P.toFilePath source) ""
      skipScripts = importPolicy opts == SkipUnchangedScripts
  pre <- hook "preprocess"
  (csv,preRecord,preRan) <- case pre of
    Nothing -> do
      input <- savedInput opts ch source intermediate
      pure (input,Nothing,False)
    Just script -> do
      before <- observeHook script source
      let key = hookInputsKey dirs cwd "preprocess" <$> before
          prior = old >>= preprocessRecord
      hit <- case key of
        Reusable k | skipScripts -> reuseStage "Preprocess" k intermediate prior
        _ -> do
          unless skipScripts $ logVerbose opts ch "Running preprocess script in run-scripts mode."
          pure False
      if hit then do
        logVerbose opts ch "Skipped unchanged preprocess script."
        pure (intermediate,prior,False)
      else do
        previous <- digestIfPresent intermediate
        digest <- installOutput True intermediate (preprocess opts ch script dirs source intermediate)
        when (Just digest == previous) $ logVerbose opts ch "Preprocessing ran; the CSV is unchanged."
        after <- observeHook script source
        selected <- hook "preprocess"
        usable <- case key of
          Fallback reason -> when skipScripts (report reason) >> pure Nothing
          Reusable k | selected == pre && after == before -> pure (Just (StageRecord k digest))
                     | otherwise -> when skipScripts (report changed) >> pure Nothing
        pure (intermediate,usable,True)
  output <- generatedOutput [P.reldir|3-journal|] ".journal" csv
  let discover = do
        candidate <- hook "construct"
        context <- probe
        case context of
          Fallback r -> pure (Fallback r)
          Reusable converter -> checkedIO "could not verify journal inputs" (P.toFilePath csv) $ do
            config <- discoverConfig (converterRelease converter) (hledgerConf opts)
            case config of
              Fallback r -> pure (Fallback r)
              Reusable conf -> do
                let common = [("source",pathText source),("effective CSV",pathText csv),("preprocess",maybe "" pathText pre)]
                      ++ map ("converter",) (converterKey converter)
                    configArgs = pinnedConfig ("<config>" <$ conf)
                    mk inputs files args extra construct =
                      JournalInputs converter inputs (common ++ map ("hledger argument",) args ++ extra ++ [(pathText p,digestHex d) | (p,d) <- files]) (Map.fromList files) construct
                case candidate of
                  Just script -> do
                    observation <- observeHook script csv
                    pure $ (\h -> mk (ConstructInput (fst <$> conf)) (maybeToList conf)
                                        (constructArguments configArgs "<journal>") (hookInputsKey dirs cwd "construct" h) (Just h)) <$> observation
                  Nothing | not (isSeparatedValues csv) -> pure $ atPath "unsupported input reader" (P.toFilePath csv) ""
                  Nothing -> do
                    rules <- selectRules csv dirs
                    case rules of
                      Fallback r -> pure (Fallback r)
                      Reusable Nothing -> pure $ Fallback (OrdinaryInputError "missing rules file" (pathText csv))
                      Reusable (Just rf) -> do
                        tree <- discoverRules (baseDir opts) (converterRelease converter) rf
                        directives <- discoverDirectives (directivesFile opts)
                        digest <- digestFile csv
                        pure $ do
                          rulesTree <- tree
                          ds <- directives
                          let files = (csv,digest) : rulesTree ++ maybeToList ds ++ maybeToList conf
                          pure $ mk (RulesInput csv rf (fst <$> ds) (fst <$> conf)) files
                                   (rulesArguments configArgs "<input>" "<rules>" ("<directives>" <$ ds)) [("rules",pathText rf)] Nothing
      ordinary = do
        script <- hook "construct"
        case script of
          Nothing -> hledgerImport opts ch dirs csv output
          Just s -> withStagedOutput output (customConstruct opts ch s dirs csv output)
      prior = old >>= journalRecord
      fallback reason = do
        report reason
        case reason of
          CyclicRules _ file -> ioError $ userError ("Cyclic CSV rules include at " ++ T.unpack file)
          _ -> ordinary >> pure (Nothing,True)
  rawDecision <- discover
  let agrees inputs = case preRecord of
        Nothing -> True
        Just entry -> Just (stageOutput entry) == case liveConstruct inputs of
          Nothing -> Map.lookup csv (copiedDigests inputs)
          Just (HookInputs _ _ _ observed) -> Just (observationDigest observed)
      decision = rawDecision >>= \inputs -> if agrees inputs then pure inputs else Fallback changed
  (journal,generated) <- case decision of
    Fallback r -> fallback r
    Reusable inputs -> do
      hit <- if skipScripts || liveConstruct inputs == Nothing
               then reuseStage "Journal" (recordedInputs inputs) output prior
               else logVerbose opts ch "Running construct script in run-scripts mode." >> pure False
      if hit then do
        when (liveConstruct inputs /= Nothing) $ logVerbose opts ch "Skipped unchanged construct script."
        logVerbose opts ch "Reused the verified journal."
        pure (prior,False)
      else do
        result <- withSnapshot snapshots (converterRelease (engine inputs)) (copiedInputs inputs) $ \snapshot -> do
          let pin conf = pinnedConfig (pathText <$> conf)
              translate = translateDiagnostic snapshot
              executable = converterExecutable (engine inputs)
          outDigest <- case (copiedInputs inputs,snapshotInputs snapshot,liveConstruct inputs) of
            (RulesInput src rf ds _,RulesInput src' rf' ds' conf,_) -> do
              contents <- convertRules opts ch executable (RulesPaths src rf ds) (RulesPaths src' rf' ds') (pin conf) translate
              target <- D.canonicalizePath (P.toFilePath output) >>= P.parseAbsFile
              installOutput False target (writeStagedText target contents)
            (ConstructInput _,ConstructInput conf,Just (HookInputs s _ _ _)) ->
              installOutput False output (constructWith opts ch executable (pin conf) translate s dirs csv output)
            _ -> ioError (userError "Inconsistent incremental conversion inputs")
          again <- discover
          preNow <- hook "preprocess"
          effective <- case preNow of
            Nothing -> P.doesFileExist intermediate >>= \exists -> pure (if exists then intermediate else source)
            Just _ -> pure intermediate
          let stable = again == Reusable inputs && snapshotManifest snapshot == copiedDigests inputs && preNow == pre && effective == csv
          if stable then pure $ Just (StageRecord (recordedInputs inputs) outDigest)
                    else when (skipScripts || liveConstruct inputs == Nothing) (report changed) >> pure Nothing
        case result of
          Fallback r -> fallback r
          Reusable entry -> pure (entry,True)
  -- A concurrent writer can replace the intermediate after the hook returned.
  -- A later run must verify its bytes before trusting the preprocessing entry.
  currentPre <- case preRecord of
    Just entry -> do
      actual <- digestIfPresent intermediate
      pure $ if actual == Just (stageOutput entry) then Just entry else Nothing
    Nothing -> pure Nothing
  let next = StatementRecord currentPre journal
  when (Just next /= old && (old /= Nothing || next /= StatementRecord Nothing Nothing)) $ do
    saved <- tryIOError $ saveRecord record next
    either (channelErrLn ch . ("Could not save incremental import record: " <>) . T.pack . show) pure saved
  pure (output,generated,preRan || generated)