packages feed

hledger-flow 0.20.0 → 0.20.1

raw patch · 38 files changed

+3115/−496 lines, 38 filesdep +Win32dep +cryptohash-sha256dep +unixPVP: major bump suggested

API removals or changes: PVP suggests a major version bump

Dependencies added: Win32, cryptohash-sha256, unix

API changes (from Hackage documentation)

- Hledger.Flow.Common: descriptiveOutput :: Text -> Text -> Text
- Hledger.Flow.Common: logTimedAction :: HasVerbosity o => o -> TChan LogMessage -> Text -> [Text] -> Text -> [Text] -> (TChan LogMessage -> Text -> IO ()) -> (TChan LogMessage -> Text -> IO ()) -> IO FullOutput -> IO FullTimedOutput
- Hledger.Flow.Common: timeAndExitOnErr :: HasVerbosity o => o -> TChan LogMessage -> Text -> (TChan LogMessage -> Text -> IO ()) -> (TChan LogMessage -> Text -> IO ()) -> ProcInput -> IO FullTimedOutput
- Hledger.Flow.Common: timeAndExitOnErr' :: HasVerbosity o => o -> TChan LogMessage -> Text -> [Text] -> [Text] -> (TChan LogMessage -> Text -> IO ()) -> (TChan LogMessage -> Text -> IO ()) -> ProcInput -> IO FullTimedOutput
- Hledger.Flow.Logging: dummyLogger :: TChan LogMessage -> Text -> IO ()
- Hledger.Flow.Logging: timestampPrefix :: Text -> IO Text
- Hledger.Flow.ReleaseNotice: checkRelease :: (String -> IO (Maybe ByteString)) -> IO FilePath -> IO ReleaseResult
- Hledger.Flow.ReleaseNotice: startCheck :: IO ReleaseResult -> IO (IO ())
- Hledger.Flow.RuntimeOptions: [onlyNewFiles] :: RuntimeOptions -> Bool
- Hledger.Flow.Types: type FullTimedOutput = (FullOutput, NominalDiffTime)
+ Hledger.Flow.Cache: cacheDir :: AbsDir -> AbsDir
+ Hledger.Flow.Cache: initializeCache :: AbsDir -> IO ()
+ Hledger.Flow.Common: External :: Text -> Text -> [[Text]] -> [Text] -> External
+ Hledger.Flow.Common: [extCommand] :: External -> [[Text]]
+ Hledger.Flow.Common: [extHints] :: External -> [Text]
+ Hledger.Flow.Common: [extName] :: External -> Text
+ Hledger.Flow.Common: [extPurpose] :: External -> Text
+ Hledger.Flow.Common: announceExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> IO ()
+ Hledger.Flow.Common: checkExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> Bool -> NominalDiffTime -> FullOutput -> IO ()
+ Hledger.Flow.Common: data External
+ Hledger.Flow.Common: externalCommand :: HasBaseDir o => o -> External -> IO Text
+ Hledger.Flow.Common: findFirstFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> Text -> [AbsFile] -> IO (Maybe AbsFile)
+ Hledger.Flow.Common: firstConfirmedFile :: [FilePath] -> IO (Maybe FilePath)
+ Hledger.Flow.Common: hledgerConfCandidates :: IO [FilePath]
+ Hledger.Flow.Common: lookupHledgerConf :: Maybe FilePath -> IO (Maybe HledgerConf)
+ Hledger.Flow.Common: outputSection :: Bool -> Bool -> Text -> Text -> [Text]
+ Hledger.Flow.Common: pasteCommand :: Shell -> FilePath -> [FilePath] -> [[Text]] -> Text
+ Hledger.Flow.Common: runExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> Bool -> ProcInput -> IO Text
+ Hledger.Flow.Common: showDuration :: NominalDiffTime -> Text
+ Hledger.Flow.Common: writeStagedText :: AbsFile -> Text -> AbsFile -> IO ()
+ Hledger.Flow.DocHelpers: IncrementalImportFallback :: DocLink
+ Hledger.Flow.DocHelpers: Reports :: DocLink
+ Hledger.Flow.DocHelpers: StatusCommand :: DocLink
+ Hledger.Flow.Import.Cache: StageRecord :: StageInputs -> Digest -> StageRecord
+ Hledger.Flow.Import.Cache: StatementRecord :: Maybe StageRecord -> Maybe StageRecord -> StatementRecord
+ Hledger.Flow.Import.Cache: [journalRecord] :: StatementRecord -> Maybe StageRecord
+ Hledger.Flow.Import.Cache: [preprocessRecord] :: StatementRecord -> Maybe StageRecord
+ Hledger.Flow.Import.Cache: [stageInputs] :: StageRecord -> StageInputs
+ Hledger.Flow.Import.Cache: [stageOutput] :: StageRecord -> Digest
+ Hledger.Flow.Import.Cache: data StageRecord
+ Hledger.Flow.Import.Cache: data StatementRecord
+ Hledger.Flow.Import.Cache: digestIfPresent :: AbsFile -> IO (Maybe Digest)
+ Hledger.Flow.Import.Cache: installOutput :: Bool -> AbsFile -> (AbsFile -> IO ()) -> IO Digest
+ Hledger.Flow.Import.Cache: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: instance GHC.Classes.Eq Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance GHC.Classes.Eq Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: instance GHC.Internal.Generics.Generic Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance GHC.Internal.Generics.Generic Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: instance GHC.Internal.Show.Show Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance GHC.Internal.Show.Show Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: readRecord :: AbsFile -> IO (Maybe StatementRecord)
+ Hledger.Flow.Import.Cache: recordPath :: AbsDir -> AbsFile -> IO AbsFile
+ Hledger.Flow.Import.Cache: saveRecord :: AbsFile -> StatementRecord -> IO ()
+ Hledger.Flow.Import.Cache: stageMiss :: StageInputs -> AbsFile -> Maybe StageRecord -> IO (Maybe Text)
+ Hledger.Flow.Import.Cache: type StageInputs = [(Text, Text)]
+ Hledger.Flow.Import.Conversion: RulesPaths :: AbsFile -> AbsFile -> Maybe AbsFile -> RulesPaths
+ Hledger.Flow.Import.Conversion: constructArguments :: [Text] -> Text -> [Text]
+ Hledger.Flow.Import.Conversion: constructWith :: RuntimeOptions -> TChan LogMessage -> AbsFile -> [Text] -> (Text -> IO Text) -> AbsFile -> ImportDirs Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: convertRules :: RuntimeOptions -> TChan LogMessage -> AbsFile -> RulesPaths -> RulesPaths -> [Text] -> (Text -> IO Text) -> IO Text
+ Hledger.Flow.Import.Conversion: customConstruct :: RuntimeOptions -> TChan LogMessage -> AbsFile -> ImportDirs Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: data RulesPaths
+ Hledger.Flow.Import.Conversion: findHook :: RuntimeOptions -> TChan LogMessage -> Text -> AbsFile -> IO (Maybe AbsFile)
+ Hledger.Flow.Import.Conversion: hledgerImport :: RuntimeOptions -> TChan LogMessage -> ImportDirs Abs -> AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: hookDirArgs :: ImportDirs b -> [Text]
+ Hledger.Flow.Import.Conversion: logNewFileSkip :: RuntimeOptions -> TChan LogMessage -> Text -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: pinnedConfig :: Maybe Text -> [Text]
+ Hledger.Flow.Import.Conversion: preprocess :: RuntimeOptions -> TChan LogMessage -> AbsFile -> ImportDirs Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: rulesArguments :: [Text] -> Text -> Text -> Maybe Text -> [Text]
+ Hledger.Flow.Import.Conversion: savedInput :: RuntimeOptions -> TChan LogMessage -> AbsFile -> AbsFile -> IO AbsFile
+ Hledger.Flow.Import.Dependencies: Converter :: Release -> AbsFile -> FileStamp -> [Text] -> Converter
+ Hledger.Flow.Import.Dependencies: CyclicRules :: Text -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: Fallback :: FallbackReason -> ReuseDecision a
+ Hledger.Flow.Import.Dependencies: FallbackReason :: Text -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: FileReason :: Text -> FilePath -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: OrdinaryInputError :: Text -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: OtherSupported :: Release
+ Hledger.Flow.Import.Dependencies: Reusable :: a -> ReuseDecision a
+ Hledger.Flow.Import.Dependencies: V1524 :: Release
+ Hledger.Flow.Import.Dependencies: V1996 :: Release
+ Hledger.Flow.Import.Dependencies: [converterExecutable] :: Converter -> AbsFile
+ Hledger.Flow.Import.Dependencies: [converterKey] :: Converter -> [Text]
+ Hledger.Flow.Import.Dependencies: [converterRelease] :: Converter -> Release
+ Hledger.Flow.Import.Dependencies: [converterStamp] :: Converter -> FileStamp
+ Hledger.Flow.Import.Dependencies: [reasonDetail] :: FallbackReason -> Text
+ Hledger.Flow.Import.Dependencies: [reasonFile] :: FallbackReason -> FilePath
+ Hledger.Flow.Import.Dependencies: [reasonScope] :: FallbackReason -> Text
+ Hledger.Flow.Import.Dependencies: atPath :: Text -> FilePath -> Text -> ReuseDecision a
+ Hledger.Flow.Import.Dependencies: checkArguments :: [Text] -> ReuseDecision ()
+ Hledger.Flow.Import.Dependencies: checkConfig :: Release -> AbsFile -> ByteString -> ReuseDecision ()
+ Hledger.Flow.Import.Dependencies: checkDirectives :: Release -> AbsFile -> ByteString -> ReuseDecision ()
+ Hledger.Flow.Import.Dependencies: checkedIO :: Text -> FilePath -> IO (ReuseDecision a) -> IO (ReuseDecision a)
+ Hledger.Flow.Import.Dependencies: converterProbeWith :: AbsFile -> (AbsFile -> IO (ExitCode, Text)) -> IO (IO (ReuseDecision Converter))
+ Hledger.Flow.Import.Dependencies: data Converter
+ Hledger.Flow.Import.Dependencies: data FallbackReason
+ Hledger.Flow.Import.Dependencies: data Release
+ Hledger.Flow.Import.Dependencies: data ReuseDecision a
+ Hledger.Flow.Import.Dependencies: discoverConfig :: Release -> Maybe HledgerConf -> IO (ReuseDecision (Maybe (AbsFile, Digest)))
+ Hledger.Flow.Import.Dependencies: discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile, Digest)))
+ Hledger.Flow.Import.Dependencies: discoverRules :: AbsDir -> Release -> AbsFile -> IO (ReuseDecision [(AbsFile, Digest)])
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq Hledger.Flow.Import.Dependencies.Converter
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq Hledger.Flow.Import.Dependencies.FallbackReason
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq Hledger.Flow.Import.Dependencies.Release
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq a => GHC.Classes.Eq (Hledger.Flow.Import.Dependencies.ReuseDecision a)
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Ord Hledger.Flow.Import.Dependencies.FallbackReason
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Base.Applicative Hledger.Flow.Import.Dependencies.ReuseDecision
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Base.Functor Hledger.Flow.Import.Dependencies.ReuseDecision
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Base.Monad Hledger.Flow.Import.Dependencies.ReuseDecision
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show Hledger.Flow.Import.Dependencies.Converter
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show Hledger.Flow.Import.Dependencies.FallbackReason
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show Hledger.Flow.Import.Dependencies.Release
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show a => GHC.Internal.Show.Show (Hledger.Flow.Import.Dependencies.ReuseDecision a)
+ Hledger.Flow.Import.Dependencies: isSeparatedValues :: AbsFile -> Bool
+ Hledger.Flow.Import.Dependencies: nestedRuleIncludes :: Release -> AbsFile -> ByteString -> ReuseDecision [FilePath]
+ Hledger.Flow.Import.Dependencies: newConverterProbe :: RuntimeOptions -> IO (IO (ReuseDecision Converter))
+ Hledger.Flow.Import.Dependencies: resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)
+ Hledger.Flow.Import.Dependencies: ruleIncludes :: Release -> AbsFile -> ByteString -> ReuseDecision [FilePath]
+ Hledger.Flow.Import.Dependencies: selectRules :: AbsFile -> ImportDirs Abs -> IO (ReuseDecision (Maybe AbsFile))
+ Hledger.Flow.Import.Fingerprint: Digest :: Text -> Digest
+ Hledger.Flow.Import.Fingerprint: FileObservation :: !FileStamp -> !Digest -> FileObservation
+ Hledger.Flow.Import.Fingerprint: FileStamp :: !(Integer, Integer) -> !Integer -> !Rational -> !Maybe Rational -> FileStamp
+ Hledger.Flow.Import.Fingerprint: [digestHex] :: Digest -> Text
+ Hledger.Flow.Import.Fingerprint: [fileStamp] :: FileObservation -> !FileStamp
+ Hledger.Flow.Import.Fingerprint: [observationDigest] :: FileObservation -> !Digest
+ Hledger.Flow.Import.Fingerprint: [stampChanged] :: FileStamp -> !Maybe Rational
+ Hledger.Flow.Import.Fingerprint: [stampIdentity] :: FileStamp -> !(Integer, Integer)
+ Hledger.Flow.Import.Fingerprint: [stampModified] :: FileStamp -> !Rational
+ Hledger.Flow.Import.Fingerprint: [stampSize] :: FileStamp -> !Integer
+ Hledger.Flow.Import.Fingerprint: copyDigest :: AbsFile -> AbsFile -> IO Digest
+ Hledger.Flow.Import.Fingerprint: data FileObservation
+ Hledger.Flow.Import.Fingerprint: data FileStamp
+ Hledger.Flow.Import.Fingerprint: digestBytes :: ByteString -> Digest
+ Hledger.Flow.Import.Fingerprint: digestFile :: AbsFile -> IO Digest
+ Hledger.Flow.Import.Fingerprint: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance GHC.Classes.Eq Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance GHC.Classes.Eq Hledger.Flow.Import.Fingerprint.FileObservation
+ Hledger.Flow.Import.Fingerprint: instance GHC.Classes.Eq Hledger.Flow.Import.Fingerprint.FileStamp
+ Hledger.Flow.Import.Fingerprint: instance GHC.Classes.Ord Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance GHC.Internal.Show.Show Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance GHC.Internal.Show.Show Hledger.Flow.Import.Fingerprint.FileObservation
+ Hledger.Flow.Import.Fingerprint: instance GHC.Internal.Show.Show Hledger.Flow.Import.Fingerprint.FileStamp
+ Hledger.Flow.Import.Fingerprint: newtype Digest
+ Hledger.Flow.Import.Fingerprint: observeFile :: AbsFile -> IO FileObservation
+ Hledger.Flow.Import.Fingerprint: statFile :: AbsFile -> IO FileStamp
+ Hledger.Flow.Import.Incremental: data IncrementalRun
+ Hledger.Flow.Import.Incremental: importStatement :: IncrementalRun -> RuntimeOptions -> TChan LogMessage -> ImportDirs Abs -> AbsFile -> IO (AbsFile, Bool, Bool)
+ Hledger.Flow.Import.Incremental: instance GHC.Classes.Eq Hledger.Flow.Import.Incremental.HookInputs
+ Hledger.Flow.Import.Incremental: instance GHC.Classes.Eq Hledger.Flow.Import.Incremental.JournalInputs
+ Hledger.Flow.Import.Incremental: instance GHC.Internal.Show.Show Hledger.Flow.Import.Incremental.HookInputs
+ Hledger.Flow.Import.Incremental: instance GHC.Internal.Show.Show Hledger.Flow.Import.Incremental.JournalInputs
+ Hledger.Flow.Import.Incremental: withIncrementalRun :: RuntimeOptions -> TChan LogMessage -> Int -> (IncrementalRun -> IO a) -> IO a
+ Hledger.Flow.Import.PrivateDirectory: createPrivateDirectory :: AbsDir -> IO ()
+ Hledger.Flow.Import.Snapshot: ConstructInput :: Maybe AbsFile -> SnapshotInput
+ Hledger.Flow.Import.Snapshot: RulesInput :: AbsFile -> AbsFile -> Maybe AbsFile -> Maybe AbsFile -> SnapshotInput
+ Hledger.Flow.Import.Snapshot: data PreparedSnapshot
+ Hledger.Flow.Import.Snapshot: data SnapshotInput
+ Hledger.Flow.Import.Snapshot: data SnapshotRun
+ Hledger.Flow.Import.Snapshot: instance GHC.Classes.Eq Hledger.Flow.Import.Snapshot.SnapshotInput
+ Hledger.Flow.Import.Snapshot: instance GHC.Internal.Exception.Type.Exception Hledger.Flow.Import.Snapshot.PreparationFailure
+ Hledger.Flow.Import.Snapshot: instance GHC.Internal.Show.Show Hledger.Flow.Import.Snapshot.PreparationFailure
+ Hledger.Flow.Import.Snapshot: instance GHC.Internal.Show.Show Hledger.Flow.Import.Snapshot.SnapshotInput
+ Hledger.Flow.Import.Snapshot: snapshotInputs :: PreparedSnapshot -> SnapshotInput
+ Hledger.Flow.Import.Snapshot: snapshotManifest :: PreparedSnapshot -> Map AbsFile Digest
+ Hledger.Flow.Import.Snapshot: translateDiagnostic :: PreparedSnapshot -> Text -> IO Text
+ Hledger.Flow.Import.Snapshot: windowsSnapshotPathFits :: FilePath -> Bool
+ Hledger.Flow.Import.Snapshot: withSnapshot :: SnapshotRun -> Release -> SnapshotInput -> (PreparedSnapshot -> IO a) -> IO (ReuseDecision a)
+ Hledger.Flow.Import.Snapshot: withSnapshotRun :: AbsDir -> (SnapshotRun -> IO a) -> IO a
+ Hledger.Flow.Import.Snapshot: withSnapshotRunUsing :: (AbsDir -> IO ()) -> AbsDir -> (SnapshotRun -> IO a) -> IO a
+ Hledger.Flow.Logging: bold :: Bool -> Text -> Text
+ Hledger.Flow.Logging: cyan :: Bool -> Text -> Text
+ Hledger.Flow.Logging: dim :: Bool -> Text -> Text
+ Hledger.Flow.Logging: green :: Bool -> Text -> Text
+ Hledger.Flow.Logging: paint :: Bool -> [SGR] -> Text -> Text
+ Hledger.Flow.Logging: red :: Bool -> Text -> Text
+ Hledger.Flow.Logging: resetCode :: Text
+ Hledger.Flow.Logging: synchronous :: SomeException -> Maybe SomeException
+ Hledger.Flow.Logging: useColor :: Handle -> IO Bool
+ Hledger.Flow.Logging: verboseLine :: Text -> IO Text
+ Hledger.Flow.Logging: withLogBlock :: HasVerbosity o => o -> Bool -> TChan LogMessage -> Text -> (TChan LogMessage -> IO a) -> IO a
+ Hledger.Flow.Logging: yellow :: Bool -> Text -> Text
+ Hledger.Flow.ReleaseNotice: cachedRelease :: AbsDir -> IO FilePath -> IO ReleaseResult
+ Hledger.Flow.ReleaseNotice: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReleaseNotice.Observation
+ Hledger.Flow.ReleaseNotice: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReleaseNotice.ReleaseCache
+ Hledger.Flow.ReleaseNotice: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReleaseNotice.Observation
+ Hledger.Flow.ReleaseNotice: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReleaseNotice.ReleaseCache
+ Hledger.Flow.ReleaseNotice: instance GHC.Internal.Generics.Generic Hledger.Flow.ReleaseNotice.Observation
+ Hledger.Flow.ReleaseNotice: instance GHC.Internal.Generics.Generic Hledger.Flow.ReleaseNotice.ReleaseCache
+ Hledger.Flow.ReleaseNotice: prepareCheck :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> IO (IO ReleaseResult)
+ Hledger.Flow.ReleaseNotice: refreshIfDue :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> IO ()
+ Hledger.Flow.ReleaseNotice: withCachedReleaseNotice :: AbsDir -> IO () -> IO ()
+ Hledger.Flow.RuntimeOptions: Full :: ImportPolicy
+ Hledger.Flow.RuntimeOptions: RunScripts :: ImportPolicy
+ Hledger.Flow.RuntimeOptions: SkipUnchangedScripts :: ImportPolicy
+ Hledger.Flow.RuntimeOptions: TimestampOnly :: ImportPolicy
+ Hledger.Flow.RuntimeOptions: [importPolicy] :: RuntimeOptions -> ImportPolicy
+ Hledger.Flow.RuntimeOptions: data ImportPolicy
+ Hledger.Flow.RuntimeOptions: instance GHC.Classes.Eq Hledger.Flow.RuntimeOptions.ImportPolicy
+ Hledger.Flow.RuntimeOptions: instance GHC.Internal.Show.Show Hledger.Flow.RuntimeOptions.ImportPolicy
+ Hledger.Flow.RuntimeOptions: onlyNewFiles :: RuntimeOptions -> Bool
- Hledger.Flow.Common: verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> AbsFile -> IO Bool
+ Hledger.Flow.Common: verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> Text -> AbsFile -> IO Bool
- Hledger.Flow.ReleaseNotice: withReleaseNotice :: IO () -> IO ()
+ Hledger.Flow.ReleaseNotice: withReleaseNotice :: AbsDir -> IO () -> IO ()
- Hledger.Flow.RuntimeOptions: RuntimeOptions :: BaseDir -> RunDir -> Maybe Integer -> Bool -> Text -> AbsFile -> IO Text -> Maybe HledgerConf -> SystemInfo -> Bool -> Bool -> Bool -> Int -> Bool -> RuntimeOptions
+ Hledger.Flow.RuntimeOptions: RuntimeOptions :: BaseDir -> RunDir -> Maybe Integer -> ImportPolicy -> Text -> AbsFile -> IO Text -> Maybe HledgerConf -> SystemInfo -> Bool -> Bool -> Bool -> Int -> Bool -> RuntimeOptions

Files

ChangeLog.md view
@@ -31,13 +31,33 @@   existing destination belongs in release-procedure.md. --> +## 0.20.1++- Use `hledger-flow import --incremental` with hledger 1.52.4 or later to reuse journals when their inputs, including nested rules files, still match. Choose `--incremental=skip-unchanged-scripts` to also reuse the outputs of unchanged scripts. See the [incremental import reference](https://hledger-flow.github.io/reference/incremental-import/).+- Interactive `import` and `report` share a [release check](https://hledger-flow.github.io/reference/release-notices/) for 24 hours. `status` can show a previously discovered upgrade while offline.+- On macOS, incremental import checks the hledger executable faster and still detects a replacement build that reports the same version.+- `hledger-flow import --verbose` and `hledger-flow report --verbose` are+  easier to read. Each statement's lines appear together, files are labelled+  by what import looked for, and every hledger or hook command is shown ready+  to paste into a shell at the base directory, with paths there starting+  with `./`. On a terminal, the output uses colour unless+  `NO_COLOR` is set.+- `--help` is easier to scan. Each command's help lists the global options,+  describes its examples and links to its docs. `import --help` shows each+  `--incremental` mode on its own line. On a terminal, examples and modes use+  colour unless `NO_COLOR` is set.+- When hledger or a preprocess or construct script fails, the message names+  that program, keeps its exit code and marks its output as its own. With+  `--verbose`, the full error output is shown instead of the first 20 lines.+  A construct script's error output now appears once the script finishes.+ ## 0.20.0  - If you have statements with brackets in their filenames, such as   `April [1].csv`, run `hledger-flow import` to regenerate their journals.   Each now imports exactly once alongside `April 1.csv`.-- If a statement is newer than its cached CSV and its preprocess hook is-  missing, import stops and names both files. Restore the hook or move the+- If a statement is newer than its cached CSV and its preprocess script is+  missing, import stops and names both files. Restore the script or move the   cached CSV aside, then run `hledger-flow import`. - The documentation has moved to the [project site](https://hledger-flow.github.io/).   Links in hledger-flow's messages and `--help` open the relevant section
README.md view
@@ -2,31 +2,59 @@  ## What is it? -hledger-flow organises bank statements by owner, bank, account and year.-It uses [hledger](https://hledger.org/) and its CSV rules to generate one-journal per statement, then includes those journals and your manual entries-in `all-years.journal`. You keep the original statements so you can rebuild-the imported entries when your rules change.+Keep every statement. Rebuild your journals from them. +hledger-flow is a command-line workflow for [hledger](https://hledger.org/),+the plain-text accounting tool. hledger keeps your accounts in journals:+human-readable text files you can open in any editor, in a format that+hledger reads and that+[Ledger](https://hledger-flow.github.io/guides/ledger/) and other plain-text+accounting tools largely share.++hledger-flow is opinionated about where files live. You save each statement,+exactly as your bank issued it, in a folder for its owner, institution,+account and year. `hledger-flow import` converts each one with your hledger+rules, or a script for formats rules can't read, and links the journals+through `all-years.journal`. The statements stay as permanent input, so+every journal can be rebuilt from its source and every entry traced to the+statement it came from.+ It grew out of scripts I wrote while using [adept's Full-fledged Hledger](https://github.com/adept/full-fledged-hledger/wiki).  ## Who should use this? -**Choose hledger-flow** if you keep non-overlapping statements and want to-apply better rules to several years of transactions. It is especially useful-when you manage several accounts or owners and want to review the changes-to each statement's journal before committing them.+There are many ways to keep your finances with hledger, and+[hledger's workflows page](https://hledger.org/workflows.html) describes them.+Three of them handle bank statements in quite different ways: -**Choose plain `hledger import`** if you want to append new transactions-and then review, split or annotate them directly in your journal. Your edits-survive the next import. This also suits rolling downloads such as “the last-90 days” from the same account. hledger remembers how far it got for each-source, which works well when dates and the order of same-day transactions-are stable. hledger-flow processes each statement independently, so overlapping-files can count the same purchase twice. You can keep the original CSVs with-either workflow. See [hledger's import guide](https://hledger.org/dev/hledger.html#import).+- **`hledger import`.** Your journal is the primary record. You classify,+  split and annotate entries by editing the journal, and a better rule changes+  future imports only. Overlapping downloads are fine: hledger skips entries+  it has already imported, as long as their dates and same-day order stay+  stable. [How hledger import works](https://hledger.org/hledger.html#import)+- **Read the CSV directly.** The CSV files are the record and nothing is+  generated. You classify by editing the rules, and a better rule applies the+  next time hledger reads the file. Suits a few files you name yourself, as in+  `hledger -f checking.csv bal`. The hledger 1.99 previews can also `include`+  a CSV in a journal. [Reading CSV with hledger](https://hledger.org/hledger.html#csv)+- **hledger-flow.** The original statements are the primary record, and+  journals are rebuilt from them. You classify by editing rules files, and a+  better rule rebuilds every journal that uses it, in every year. A fixed+  folder for each owner, institution, account and year keeps years of+  statements maintainable. An account's statements must not overlap, or a+  purchase appears twice. +Choose `hledger import` if your journal is your primary record and you want+to edit it by hand. Choose hledger-flow if you want to generate your journals+from statements exactly as they came from the source, classify by editing+rules, and keep years of files in a structure that stays maintainable and+auditable.++Full-fledged hledger teaches this last approach with build scripts you can adapt.+hledger-flow packages the folder conventions, import and standard reports as CLI commands.+[Full-fledged hledger’s principles and practices](https://github.com/adept/full-fledged-hledger/wiki/Key-principles-and-practices)+ ## Overview of the Basic Workflow  1. Save each statement under its [account and year](https://hledger-flow.github.io/reference/input-files/).@@ -68,13 +96,13 @@  ## How do I install it? -Install with Homebrew on macOS or Linux:+Install with Homebrew on Linux or macOS:  ```sh brew install ferntrail/tap/hledger-flow ``` -Or download a release for Windows, macOS or Linux, or build from source.+Or download a release for Linux, macOS, or Windows, or build from source. The [quick start](https://hledger-flow.github.io/quickstart/) covers each option, how to verify a download, and your first import. 
app/Main.hs view
@@ -11,32 +11,43 @@ import Prelude hiding (putStrLn)  import qualified Options.Applicative as Opts (command)-import Options.Applicative.Help.Pretty (Doc, pretty, vsep)+import Options.Applicative.Help.Pretty (Color (Cyan), Doc, annotate, bold, color, fillSep, indent, pretty, vsep, (<+>)) import Options.Applicative     ( auto,+      eitherReader,       argument,-      customExecParser,+      execParserPure,+      handleParseResult,       headerDoc,       helper,+      helpShowGlobals,       hsubparser,       info,       prefs,       progDesc,+      renderFailure,+      ParserResult (Failure),       footerDoc,       hidden,+      internal,       optional,       Alternative(many, (<|>)),       Parser,       flag',       help,+      helpDoc,       long,       metavar,       option,       short,       str,+      style,       switch )+-- Exposed by optparse-applicative, though not re-exported by Options.Applicative.+import Options.Applicative.Builder.Internal (noGlobal)+import System.Environment (getArgs) -import Hledger.Flow.Types (HledgerInfo (..))+import Hledger.Flow.Types (BaseDir, RunDir, HledgerInfo (..)) import Hledger.Flow.Common ( deferredHledgerVersion, findHledgerConf, hledgerPathFromOption, hledgerInfoFromPath ) import Hledger.Flow.Status ( Situation (..) ) import Hledger.Flow.Status.Gather ( classifyDir )@@ -44,13 +55,14 @@ import qualified Data.Text as Text import Path.IO ( getCurrentDir, resolveDir' ) import Hledger.Flow.Init ( runInit )-import Hledger.Flow.DocHelpers ( DocLink (Docs), docURL, siteURL )+import Hledger.Flow.DocHelpers ( DocLink (..), docURL, siteURL )+import Hledger.Flow.Logging ( useColor ) import Hledger.Flow.RulesTemplate ( rulesTemplateFor ) import Data.Maybe (fromMaybe) import Hledger.Flow.Internals (availableCores, versionInfo, systemInfo) import Hledger.Flow.BaseDir ( determineBaseDir, findBaseDir ) import qualified Hledger.Flow.RuntimeOptions as RT-import Hledger.Flow.ReleaseNotice ( withReleaseNotice )+import Hledger.Flow.ReleaseNotice ( withReleaseNotice, withCachedReleaseNotice ) import Hledger.Flow.Reports ( generateReports ) import Hledger.Flow.Import.CSVImport ( importCSVs ) @@ -66,7 +78,7 @@  data ImportParams = ImportParams { maybeImportBaseDir :: Maybe FilePath                                  , importStartYear :: Maybe StartYear-                                 , onlyNewFiles :: Bool+                                 , filePolicy :: RT.ImportPolicy                                  } deriving (Show)  data ReportParams = ReportParams { maybeReportBaseDir :: Maybe FilePath@@ -123,17 +135,26 @@  runCommand :: IO () runCommand = do+  colour <- useColor stdout   let description = pretty ("An hledger workflow focusing on automated statement import and classification:\n" <> Text.unpack siteURL)-  cmd <- customExecParser (prefs mempty)-    (info (helper <*> baseCommandParser) (headerDoc (Just description) <> footerDoc (Just topExamples)))+      cli = info (helper <*> baseCommandParser colour) (headerDoc (Just description) <> footerDoc (Just (topExamples colour)))+  args <- getArgs+  -- Subcommand help lists the global options. Keep them out of usage errors,+  -- where they would bury the message.+  let parse p = execParserPure (prefs p) cli args+  cmd <- handleParseResult $ case parse helpShowGlobals of+    Failure failure | (_, ExitFailure _) <- renderFailure failure "" -> parse mempty+    result -> result   case cmd of     Version -> do       sysInfo <- systemInfo       T.putStrLn $ versionInfo sysInfo -    Command mainParams' (Just (Import subParams)) ->-      withReleaseNotice $ (toRuntimeOptionsImport mainParams' subParams >>= importCSVs) `catch` reportFatal-    Command mainParams' (Just (Report subParams)) ->-      withReleaseNotice $ (toRuntimeOptionsReport mainParams' subParams >>= generateReports) `catch` reportFatal+    Command mainParams' (Just (Import subParams)) -> do+      (base, runDir) <- determineBaseDir (maybeImportBaseDir subParams)+      withReleaseNotice base $ (toRuntimeOptionsImport base runDir mainParams' subParams >>= importCSVs) `catch` reportFatal+    Command mainParams' (Just (Report subParams)) -> do+      (base, _) <- determineBaseDir (maybeReportBaseDir subParams)+      withReleaseNotice base $ (toRuntimeOptionsReport base mainParams' subParams >>= generateReports) `catch` reportFatal     Command mainParams' (Just (Status subParams)) -> runStatus mainParams' subParams     Command mainParams' (Just (Init subParams)) -> do       cwd <- getCurrentDir@@ -168,15 +189,19 @@   base <- findBaseDir dir >>= maybe (throwIO (MissingBaseDir dir)) return   case statusRulesTemplate params of     Just account -> rulesTemplateFor base (Text.pack account) >>= either (ioError . userError . Text.unpack) T.putStr-    Nothing -> printStatus (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') (statusJson params) base+    Nothing | statusJson params -> printStatus (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') True base+            | otherwise -> printInteractiveStatus mainParams' base +printInteractiveStatus :: MainParams -> BaseDir -> IO ()+printInteractiveStatus params base = withCachedReleaseNotice base $+  printStatus (hledgerPathOpt params) (hledgerConfOpt params) False base+ -- | Running hledger-flow with no command: say where we are and what to do. runBare :: MainParams -> IO () runBare mainParams' = do   situation <- getCurrentDir >>= classifyDir   case situation of-    InLedger base ->-      printStatus (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') False base+    InLedger base -> printInteractiveStatus mainParams' base     EmptyDir -> T.putStrLn "This folder is empty. Run `hledger-flow init` to start your ledger here."     Elsewhere ->       T.putStr $@@ -184,11 +209,9 @@           <> "\nNo hledger-flow ledger here or in any parent folder.\n"           <> "Run `hledger-flow init` in an empty folder to start one.\n" -toRuntimeOptionsImport :: MainParams -> ImportParams -> IO RT.RuntimeOptions-toRuntimeOptionsImport mainParams' subParams' = do+toRuntimeOptionsImport :: BaseDir -> RunDir -> MainParams -> ImportParams -> IO RT.RuntimeOptions+toRuntimeOptionsImport bd runDir mainParams' subParams' = do   startYear <- traverse resolveStartYear (importStartYear subParams')-  let maybeBD = maybeImportBaseDir subParams' :: Maybe FilePath-  (bd, runDir) <- determineBaseDir maybeBD   hlp <- hledgerPathFromOption $ hledgerPathOpt mainParams'   version <- deferredHledgerVersion hlp   conf <- findHledgerConf $ hledgerConfOpt mainParams'@@ -196,7 +219,7 @@   return RT.RuntimeOptions { RT.baseDir = bd                            , RT.importRunDir = runDir                            , RT.importStartYear = startYear-                           , RT.onlyNewFiles = onlyNewFiles subParams'+                           , RT.importPolicy = filePolicy subParams'                            , RT.hfVersion = versionInfo sysInfo                            , RT.hledgerExecutable = hlp                            , RT.hledgerVersion = version@@ -209,17 +232,15 @@                            , RT.prettyReports = True                            } -toRuntimeOptionsReport :: MainParams -> ReportParams -> IO RT.RuntimeOptions-toRuntimeOptionsReport mainParams' subParams' = do-  let maybeBD = maybeReportBaseDir subParams' :: Maybe FilePath-  (bd, _) <- determineBaseDir maybeBD+toRuntimeOptionsReport :: BaseDir -> MainParams -> ReportParams -> IO RT.RuntimeOptions+toRuntimeOptionsReport bd mainParams' subParams' = do   HledgerInfo hlp version <- hledgerInfoFromPath $ hledgerPathOpt mainParams'   conf <- findHledgerConf $ hledgerConfOpt mainParams'   sysInfo <- systemInfo   return RT.RuntimeOptions { RT.baseDir = bd                            , RT.importRunDir = [reldir|.|]                            , RT.importStartYear = Nothing-                           , RT.onlyNewFiles = False+                           , RT.importPolicy = RT.Full                            , RT.hfVersion = versionInfo sysInfo                            , RT.hledgerExecutable = hlp                            , RT.hledgerVersion = return version@@ -232,65 +253,101 @@                            , RT.prettyReports = not(asciiReports subParams')                            } -baseCommandParser :: Parser BaseCommand-baseCommandParser = (Command <$> verboseParser <*> commandParser)-  <|> flag' Version (long "version" <> short 'V' <> help "Display version information")+baseCommandParser :: Bool -> Parser BaseCommand+baseCommandParser colour = (Command <$> verboseParser <*> commandParser colour)+  <|> flag' Version (long "version" <> short 'V' <> noGlobal <> help "Display version information.") -commandParser :: Parser (Maybe Command)-commandParser = optional $ hsubparser $-  subcommand "status" (Status <$> subcommandParserStatus)-    <> subcommand "init" (Init . InitParams <$> optional (argument str (metavar "DIR" <> help "The folder for the new ledger, created if missing. Defaults to the current folder.")))-    <> subcommand "import" (Import <$> subcommandParserImport)-    <> subcommand "report" (Report <$> subcommandParserReport)+commandParser :: Bool -> Parser (Maybe Command)+commandParser colour = optional $ hsubparser $+  subcommand "status" StatusCommand (Status <$> subcommandParserStatus)+    [ ("hledger-flow status --json", "The same status, for scripts and agents"),+      ("hledger-flow status --rules-template me/mybank/checking", "Print a starter rules file for one account") ]+    <> subcommand "init" GettingStarted (Init . InitParams <$> optional (argument str (metavar "DIR" <> help "The folder for the new ledger, created if missing. Default: the current folder.")))+    [ ("hledger-flow init ~/finances", "Start a ledger in ~/finances") ]+    <> subcommand "import" InputFiles (Import <$> subcommandParserImport colour)+    [ ("hledger-flow import", "Import the current folder"),+      ("hledger-flow import import/me/mybank", "Import one bank's statements"),+      ("hledger-flow import --start-year current", "Import this year only"),+      ("hledger-flow import --incremental", "Rebuild only what changed") ]+    <> subcommand "report" Reports (Report <$> subcommandParserReport)+    [ ("hledger-flow report", "Write reports to the reports folder") ]     <> metavar "COMMAND"   where-    subcommand name parser = Opts.command name (info parser (progDesc (summaryOf name) <> footerDoc (lines' <$> lookup name subcommandExamples)))+    subcommand name link parser examples =+      Opts.command name (info parser (progDesc (summaryOf name) <> footerDoc (Just (examplesDoc colour link examples)))) -lines' :: [String] -> Doc-lines' = vsep . map pretty+-- optparse-applicative writes ANSI styles into help even when stdout is a+-- pipe, so style only after useColor has checked the terminal and NO_COLOR.+literal, heading :: Bool -> Doc -> Doc+literal colour = if colour then annotate (color Cyan) else id+heading colour = if colour then annotate bold else id -topExamples :: Doc-topExamples = lines'-  [ "Examples:",-    "  hledger-flow init            Start a ledger in the current folder",-    "  hledger-flow status          See what's there and what to do next",-    "  hledger-flow import          Turn statements into journals",-    "  hledger-flow report          Generate reports",-    "  hledger-flow status --json   The same status, for scripts and agents",-    "",-    "Docs: " <> Text.unpack (docURL Docs)-  ]+-- | Commands with descriptions in a column, or below each command when a+-- column would not fit in 80 characters, then the command's docs link.+examplesDoc :: Bool -> DocLink -> [(String, String)] -> Doc+examplesDoc colour link examples = vsep $+  heading colour (pretty (if length examples == 1 then "Example:" else "Examples:" :: String))+    : concatMap row examples+    ++ [mempty, heading colour (pretty ("Docs:" :: String)) <+> pretty (docURL link)]+  where+    width = maximum (map (length . fst) examples)+    fits = 2 + width + 3 + maximum (map (length . snd) examples) <= 80+    row (cmd, desc)+      | fits = [indent 2 (literal colour (pretty cmd) <> pretty (replicate (width - length cmd + 3) ' ' ++ desc))]+      | otherwise = [indent 2 (literal colour (pretty cmd)), indent 4 (pretty desc)] -subcommandExamples :: [(String, [String])]-subcommandExamples =-  [ ("status", ["Examples:", "  hledger-flow status --json", "  hledger-flow status --rules-template me/mybank/checking"]),-    ("init", ["Example:", "  hledger-flow init ~/finances"]),-    ("import", ["Examples:", "  hledger-flow import import/me/mybank", "  hledger-flow import --start-year current"]),-    ("report", ["Example:", "  hledger-flow report"])+topExamples :: Bool -> Doc+topExamples colour = examplesDoc colour Docs+  [ ("hledger-flow init", "Start a ledger in the current folder"),+    ("hledger-flow status", "See what's there and what to do next"),+    ("hledger-flow import", "Turn statements into journals"),+    ("hledger-flow report", "Generate reports"),+    ("hledger-flow status --json", "The same status, for scripts and agents")   ]  subcommandParserStatus :: Parser StatusParams subcommandParserStatus = StatusParams-  <$> optional (argument str (metavar "DIR" <> help "A folder inside the ledger. Defaults to the current folder."))-  <*> switch (long "json" <> help "Print the status as JSON, for scripts and agents")-  <*> optional (option str (long "rules-template" <> metavar "ACCOUNT" <> help "Print a starter rules file for owner/bank/account, built from its newest statement's header row"))+  <$> optional (argument str (metavar "DIR" <> help "A folder inside the ledger. Default: the current folder."))+  <*> switch (long "json" <> help "Print the status as JSON, for scripts and agents.")+  <*> optional (option str (long "rules-template" <> metavar "ACCOUNT" <> help "Print a starter rules file for owner/bank/account, built from its newest statement's header row."))  verboseParser :: Parser MainParams verboseParser = MainParams-  <$> (length <$> many (flag' () (long "verbose" <> short 'v' <> help "Print more verbose output")))-  <*> optional (option str (long "hledger-path" <> short 'H' <> metavar "HLEDGER-PATH" <> help "The full path to an hledger executable"))-  <*> optional (option str (long "hledger-conf" <> metavar "PATH" <> hidden <> help "The hledger config file to pass to hledger --conf"))-  <*> switch (long "show-options" <> hidden <> help "Print the hledger-flow version, hledger path and settings import or report runs with. status always shows them.")+  <$> (length <$> many (flag' () (long "verbose" <> short 'v' <> help "Show each step and the commands hledger-flow runs.")))+  <*> optional (option str (long "hledger-path" <> short 'H' <> metavar "FILE" <> help "Use this hledger executable instead of the one on PATH."))+  <*> optional (option str (long "hledger-conf" <> metavar "FILE" <> hidden <> help "Pass this config file to hledger. Default: the one hledger finds itself."))+  <*> switch (long "show-options" <> hidden <> help "Print the version, hledger path and settings before import or report runs. status always shows them.")   <*> optional (option auto (long "batch-size" <> metavar "SIZE" <> hidden <> help "Number of files to process in parallel. Must be positive. Default: the number of cores in use. Ignored during sequential processing."))-  <*> switch (long "sequential" <> hidden <> help "Disable parallel processing")+  <*> switch (long "sequential" <> hidden <> help "Disable parallel processing.") -subcommandParserImport :: Parser ImportParams-subcommandParserImport = ImportParams-  <$> optional (argument str (metavar "DIR" <> help "The directory to import. Use the base directory for a full import or a sub-directory for a partial import. Defaults to the current directory."))-  <*> optional (option startYearReader (long "start-year" <> metavar "YEAR" <> help "Import only from the specified year and onwards, ignoring previous years. By default all available years are imported. Use a 4-digit year or 'current' for the current year"))-  <*> switch (long "new-files-only" <> help "Skip regenerating output files that are at least as new as their source. This applies to preprocessed files and hledger journal files.")+subcommandParserImport :: Bool -> Parser ImportParams+subcommandParserImport colour = ImportParams <$> optional (argument str (metavar "DIR" <> help "The folder to import: the ledger root for everything, or a folder inside it, such as an owner, bank or account, for part of it. Default: the current folder."))+  <*> optional (option startYearReader (long "start-year" <> metavar "YEAR" <> help "Import statements from YEAR onwards only. YEAR is a four-digit year or current. Default: all years."))+  <*> filePolicyParser colour +incrementalModes :: [(String, RT.ImportPolicy)]+incrementalModes = [("run-scripts", RT.RunScripts), ("skip-unchanged-scripts", RT.SkipUnchangedScripts)]++filePolicyParser :: Bool -> Parser RT.ImportPolicy+filePolicyParser colour =+  flag' RT.TimestampOnly (long "new-files-only" <> help "Rebuild a file only when its statement or cached CSV is newer. Fast, but misses other changes, such as an edited rules file.")+  <|> flag' RT.RunScripts (long "incremental" <> style (const (pretty ("--incremental[=MODE]" :: String))) <> helpDoc (Just incrementalHelp))+  <|> option (eitherReader readPolicy) (long "incremental" <> internal)+  <|> pure RT.Full+  where+    readPolicy value = maybe (Left ("unknown mode '" ++ value ++ "'. Use run-scripts or skip-unchanged-scripts.")) Right (lookup value incrementalModes)+    para = fillSep . map pretty . words+    mode name = literal colour (pretty (name :: String))+    incrementalHelp = vsep+      [ para "Rebuild only the journals whose inputs changed, comparing file contents. Accounts without scripts behave the same in both modes.",+        mempty,+        mode "run-scripts" <+> pretty ("(default)" :: String),+        indent 2 (para "Run every preprocess and construct script. Reuse a journal when its inputs, including a preprocess script's output, still match."),+        mode "skip-unchanged-scripts",+        indent 2 (para "Also skip a script when the script and its statement are unchanged. Misses edits to other files the script reads, such as a lookup table.")+      ]+ subcommandParserReport :: Parser ReportParams subcommandParserReport = ReportParams-  <$> optional (argument str (metavar "BASEDIR" <> help "The hledger-flow base directory"))-  <*> switch (long "ascii-reports" <> help "If to avoid using hledger --pretty-tables flag when generating reports.")+  <$> optional (argument str (metavar "DIR" <> help "The ledger folder. Default: the ledger containing the current folder."))+  <*> switch (long "ascii-reports" <> help "Draw report tables with plain ASCII instead of box-drawing characters.")
+ cbits/private_directory_darwin.c view
@@ -0,0 +1,24 @@+#include <stddef.h>+#include <errno.h>+#include <fcntl.h>+#include <sys/stat.h>++/* Darwin ACL grants can authorize access independently of the Unix mode. */+int hf_verify_private_acl(int fd)+{+    filesec_t security = filesec_init();+    if (security == NULL)+        return -1;+    struct stat status;+    int has_acl = 0;+    int result = fstatx_np(fd, &status, security);+    if (result == 0)+        result = filesec_query_property(security, FILESEC_ACL, &has_acl);+    int error = errno;+    filesec_free(security);+    if (result != 0 || has_acl) {+        errno = result != 0 ? error : EACCES;+        return -1;+    }+    return 0;+}
+ cbits/private_directory_windows.c view
@@ -0,0 +1,88 @@+#ifndef _WIN32_WINNT+#define _WIN32_WINNT 0x0600+#endif+#include <windows.h>+#include <sddl.h>+#include <stdlib.h>+#include <wchar.h>++/* Check the volume of the newly created directory before any copies are made. */+static DWORD check_persistent_acls(LPCWSTR path)+{+    DWORD flags = 0;+    HANDLE dir = CreateFileW(path, FILE_READ_ATTRIBUTES,+                             FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,+                             NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, NULL);+    if (dir == INVALID_HANDLE_VALUE)+        return GetLastError();++    BOOL ok = GetVolumeInformationByHandleW(dir, NULL, 0, NULL, NULL, &flags, NULL, 0);+    DWORD error = ok ? ERROR_SUCCESS : GetLastError();+    CloseHandle(dir);+    if (error != ERROR_SUCCESS)+        return error;+    return (flags & FILE_PERSISTENT_ACLS) ? ERROR_SUCCESS : ERROR_NOT_SUPPORTED;+}++DWORD hf_create_private_directory(LPCWSTR path)+{+    DWORD error;++    HANDLE token = NULL;+    TOKEN_USER *user = NULL;+    LPWSTR sid = NULL;+    LPWSTR sddl = NULL;+    PSECURITY_DESCRIPTOR descriptor = NULL;+    SECURITY_ATTRIBUTES attributes = {sizeof(SECURITY_ATTRIBUTES), NULL, FALSE};+    DWORD size = 0;+    const WCHAR prefix[] = L"D:P(A;OICI;FA;;;";++    if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token))+        return GetLastError();++    GetTokenInformation(token, TokenUser, NULL, 0, &size);+    error = GetLastError();+    if (error != ERROR_INSUFFICIENT_BUFFER)+        goto done;+    user = malloc(size);+    if (user == NULL) {+        error = ERROR_OUTOFMEMORY;+        goto done;+    }+    if (!GetTokenInformation(token, TokenUser, user, size, &size) ||+        !ConvertSidToStringSidW(user->User.Sid, &sid)) {+        error = GetLastError();+        goto done;+    }+    sddl = malloc(sizeof(WCHAR) * (wcslen(prefix) + wcslen(sid) + 2));+    if (sddl == NULL) {+        error = ERROR_OUTOFMEMORY;+        goto done;+    }+    wcscpy(sddl, prefix);+    wcscat(sddl, sid);+    wcscat(sddl, L")");+    if (!ConvertStringSecurityDescriptorToSecurityDescriptorW(+            sddl, SDDL_REVISION_1, &descriptor, NULL)) {+        error = GetLastError();+        goto done;+    }+    attributes.lpSecurityDescriptor = descriptor;+    if (CreateDirectoryW(path, &attributes)) {+        error = check_persistent_acls(path);+        if (error != ERROR_SUCCESS)+            RemoveDirectoryW(path);+    } else {+        error = GetLastError();+    }++done:+    if (descriptor != NULL)+        LocalFree(descriptor);+    free(sddl);+    if (sid != NULL)+        LocalFree(sid);+    free(user);+    CloseHandle(token);+    return error;+}
+ cbits/sha256_darwin.c view
@@ -0,0 +1,23 @@+#include <CommonCrypto/CommonDigest.h>+#include <errno.h>+#include <unistd.h>++/* Read the already-open input with a bounded buffer. CommonCrypto selects+   the SHA-256 implementation for the running Mac. */+int hf_sha256_fd(int fd, unsigned char digest[CC_SHA256_DIGEST_LENGTH])+{+    CC_SHA256_CTX ctx;+    unsigned char buffer[64 * 1024];+    CC_SHA256_Init(&ctx);+    for (;;) {+        ssize_t count = read(fd, buffer, sizeof(buffer));+        if (count > 0) {+            CC_SHA256_Update(&ctx, buffer, (CC_LONG)count);+        } else if (count == 0) {+            CC_SHA256_Final(digest, &ctx);+            return 0;+        } else if (errno != EINTR) {+            return -1;+        }+    }+}
hledger-flow.cabal view
@@ -5,9 +5,13 @@ -- see: https://github.com/sol/hpack  name:           hledger-flow-version:        0.20.0+version:        0.20.1 synopsis:       An hledger workflow focusing on automated statement import and classification.-description:    Documentation and a quick start are at <https://hledger-flow.github.io/>+description:    hledger-flow uses hledger's CSV rules to turn each bank statement into a+                separate journal. It combines those journals with your manual entries for+                reporting. When your rules change, you can rebuild the journals from the+                original statements and review which transactions changed.+                See <https://hledger-flow.github.io/> for the quick start and tutorial. category:       Finance, Console homepage:       https://github.com/apauley/hledger-flow#readme bug-reports:    https://github.com/apauley/hledger-flow/issues@@ -29,14 +33,22 @@ library   exposed-modules:       Hledger.Flow.BaseDir+      Hledger.Flow.Cache       Hledger.Flow.Common       Hledger.Flow.DateTime       Hledger.Flow.DocHelpers       Hledger.Flow.Environment+      Hledger.Flow.Import.Cache+      Hledger.Flow.Import.Conversion       Hledger.Flow.Import.CSVImport+      Hledger.Flow.Import.Dependencies+      Hledger.Flow.Import.Fingerprint       Hledger.Flow.Import.ImportHelpers       Hledger.Flow.Import.IncludeFiles+      Hledger.Flow.Import.Incremental       Hledger.Flow.Import.Inventory+      Hledger.Flow.Import.PrivateDirectory+      Hledger.Flow.Import.Snapshot       Hledger.Flow.Import.Types       Hledger.Flow.Init       Hledger.Flow.Internals@@ -62,6 +74,7 @@     , base >=4.20 && <4.22     , bytestring >=0.12.2.0 && <0.13     , containers ==0.7.*+    , cryptohash-sha256 ==0.11.*     , directory >=1.3.8.5 && <1.4     , exceptions >=0.10.9 && <0.11     , filepath >=1.5.4.0 && <1.6@@ -75,6 +88,20 @@     , text >=2.1.3 && <2.2     , time >=1.12.2 && <1.13 || >=1.14 && <1.15   default-language: Haskell2010+  if os(darwin)+    c-sources:+        cbits/private_directory_darwin.c+        cbits/sha256_darwin.c+  if os(windows)+    c-sources:+        cbits/private_directory_windows.c+    extra-libraries:+        advapi32+    build-depends:+        Win32 ==2.14.*+  else+    build-depends:+        unix ==2.8.*  executable hledger-flow   main-is: Main.hs@@ -110,6 +137,10 @@       CSVImport.Unit       Environment.Unit       ImportHelpers.Integration+      Incremental.Cache+      Incremental.Dependencies+      Incremental.Platform+      Incremental.Snapshot       Init.Integration       PathHelpers.Unit       ReleaseNotice.Unit@@ -126,13 +157,18 @@       HUnit >=1.6.2.0 && <1.7     , aeson >=2.2.5.1 && <2.3     , base >=4.20 && <4.22+    , bytestring >=0.12.2.0 && <0.13     , containers ==0.7.*     , directory >=1.3.8.5 && <1.4     , filepath >=1.5.4.0 && <1.6     , hledger-flow     , path >=0.9.6 && <0.10     , path-io >=1.8.2 && <1.9+    , process >=1.6.26.1 && <1.7     , stm >=2.5.3.1 && <2.6     , text >=2.1.3 && <2.2     , time >=1.12.2 && <1.13 || >=1.14 && <1.15   default-language: Haskell2010+  if !os(windows)+    build-depends:+        unix ==2.8.*
site/go-links.json view
@@ -2,7 +2,10 @@   "docs": "/quickstart/",   "getting-started": "/quickstart/#import-your-first-statement",   "input-files": "/reference/input-files/",+  "incremental-import-fallback": "/reference/incremental-import/#when-reuse-is-unavailable",   "rules-files": "/reference/rules-files/",   "preprocess-script": "/reference/preprocess/",-  "set-ledger-file": "/quickstart/#import-your-first-statement"+  "set-ledger-file": "/quickstart/#import-your-first-statement",+  "status": "/reference/status/",+  "reports": "/tutorial/part-2/#reports" }
+ src/Hledger/Flow/Cache.hs view
@@ -0,0 +1,30 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Hledger.Flow.Cache (cacheDir, initializeCache) where++import Control.Monad (unless)+import qualified Data.ByteString as BS+import GHC.IO.Handle.Lock (LockMode (ExclusiveLock), hLock)+import Hledger.Flow.PathHelpers (AbsDir, withStagedOutput)+import qualified Path as P+import qualified Path.IO as P+import System.IO (IOMode (AppendMode), withBinaryFile)++cacheDir :: AbsDir -> AbsDir+cacheDir base = base P.</> [P.reldir|.hledger-flow/cache|]++initializeCache :: AbsDir -> IO ()+initializeCache base = do+  let cache = cacheDir base+      ignore = cache P.</> [P.relfile|.gitignore|]+      lock = cache P.</> [P.relfile|initialize.lock|]+  exists <- P.doesFileExist ignore+  unless exists $ do+    P.createDirIfMissing True cache+    -- Serialize first use across processes. Closing the handle releases the lock.+    withBinaryFile (P.toFilePath lock) AppendMode $ \handle -> do+      hLock handle ExclusiveLock+      initialized <- P.doesFileExist ignore+      unless initialized $ withStagedOutput ignore $ \stage ->+        BS.writeFile (P.toFilePath stage) "*\n"
src/Hledger/Flow/Common.hs view
@@ -11,7 +11,10 @@ import Control.Concurrent.STM import Data.Char (isAlphaNum, isAscii, isDigit) import Data.Either-import qualified Data.List as List (isPrefixOf, isSuffixOf, nub, null)+import Data.Maybe (fromMaybe)+import Data.Time.Clock (NominalDiffTime)+import Text.Printf (printf)+import qualified Data.List as List (take, isPrefixOf, isSuffixOf, nub) import qualified Data.Map.Strict as Map import qualified Data.Text as T import qualified Data.Text.IO as T@@ -33,7 +36,7 @@ import System.IO (hClose) import qualified System.Info as Sys import qualified System.Process as Process-import System.IO.Error (tryIOError)+import System.IO.Error (isDoesNotExistError, tryIOError) import System.FilePath ((<.>)) import Prelude hiding (putStrLn, readFile, writeFile) @@ -139,20 +142,43 @@ -- by itself, like possibleConfFiles in hledger's Hledger.Cli.Conf: hledger.conf in the current -- directory or a parent, ~/.hledger.conf, or hledger.conf in the XDG config directory. findHledgerConf :: Maybe FilePath -> IO (Maybe HledgerConf)-findHledgerConf (Just given) = return (Just (GivenConf given))+findHledgerConf (Just given) = pure (Just (GivenConf given)) findHledgerConf Nothing = do-  found <- tryIOError $ do-    here <- Directory.getCurrentDirectory-    home <- Directory.getHomeDirectory-    xdg <- Directory.getXdgDirectory Directory.XdgConfig "hledger"-    let file d = d FilePath.</> if d == home then ".hledger.conf" else "hledger.conf"-    filterM Directory.doesFileExist (map file (List.nub (upToRoot here ++ [home, xdg])))-  return $ case found of+  result <- tryIOError (hledgerConfCandidates >>= filterM Directory.doesFileExist)+  pure $ case result of     Right (f : _) -> Just (FoundConf f)     _ -> Nothing++-- | Fresh lookup for callers that need to distinguish absence from failed discovery.+lookupHledgerConf :: Maybe FilePath -> IO (Maybe HledgerConf)+lookupHledgerConf (Just given) = return (Just (GivenConf given))+lookupHledgerConf Nothing = fmap FoundConf <$> (hledgerConfCandidates >>= firstConfirmedFile)++hledgerConfCandidates :: IO [FilePath]+hledgerConfCandidates = do+  here <- Directory.getCurrentDirectory+  home <- Directory.getHomeDirectory+  xdg <- Directory.getXdgDirectory Directory.XdgConfig "hledger"+  let file d = d FilePath.</> if d == home then ".hledger.conf" else "hledger.conf"+  pure (map file (List.nub (List.take 100 (upToRoot here) ++ [home, xdg])))   where     upToRoot d = let up = FilePath.takeDirectory d in d : if up == d then [] else upToRoot up +-- doesFileExist also returns False on permission errors. Verify a negative result+-- before advancing to the next candidate so an uncertain selection cannot be cached.+firstConfirmedFile :: [FilePath] -> IO (Maybe FilePath)+firstConfirmedFile [] = pure Nothing+firstConfirmedFile (file : rest) = do+  exists <- Directory.doesFileExist file+  if exists then pure (Just file) else do+    status <- tryIOError (Directory.getPermissions file)+    case status of+      Left e | isDoesNotExistError e -> firstConfirmedFile rest+             | otherwise -> ioError e+      Right _ -> do+        dir <- Directory.doesDirectoryExist file+        if dir then firstConfirmedFile rest else ioError (userError ("Could not determine the file type of " ++ file))+ showCmdArgs :: [T.Text] -> T.Text showCmdArgs args = T.intercalate " " (map escapeArg args) @@ -161,8 +187,14 @@ homeDirs :: IO [FilePath] homeDirs = either (const []) id <$> tryIOError (do   home <- Directory.getHomeDirectory-  real <- Directory.canonicalizePath home-  return (List.nub [home, real]))+  -- PowerShell expands $HOME from USERPROFILE. The Windows profile API can+  -- still return the original profile after USERPROFILE has been overridden.+  shellHome <- if nativeShell == PowerShell then lookupEnv "USERPROFILE" else pure (Just home)+  case shellHome of+    Just value | FilePath.equalFilePath value home -> do+      real <- Directory.canonicalizePath home+      pure (List.nub [home, real])+    _ -> pure [])  -- | The path's components below the home folder; @Just []@ for the home folder itself. belowHome :: [FilePath] -> FilePath -> Maybe [FilePath]@@ -237,35 +269,121 @@   logFun errorMessage   exitWith (ExitFailure exitStatus) -descriptiveOutput :: T.Text -> T.Text -> T.Text-descriptiveOutput outputLabel outTxt = do-  if not (T.null outTxt)-    then outputLabel <> ":\n" <> outTxt <> "\n"-    else ""+-- | An external program hledger-flow runs, described for the people reading its output.+data External = External+  { -- | Who printed the output and who failed: @hledger@, or a hook script's path.+    extName :: T.Text,+    -- | What it runs for, after "to": @convert import/…/a.csv with …@.+    extPurpose :: T.Text,+    -- | The command line to show, as one or more commands piped together.+    extCommand :: [[T.Text]],+    -- | What to check when it fails.+    extHints :: [T.Text]+  } -logTimedAction ::-  (HasVerbosity o) =>-  o ->-  TChan LogMessage ->-  T.Text ->-  [T.Text] ->-  T.Text ->-  [T.Text] ->-  (TChan LogMessage -> T.Text -> IO ()) ->-  (TChan LogMessage -> T.Text -> IO ()) ->-  IO FullOutput ->-  IO FullTimedOutput-logTimedAction opts ch cmdLabel extraCmdLabels cmd args stdoutLogger stderrLogger action = do-  logVerbose opts ch $ "Begin: " <> cmdLabel+-- | A command line to paste into this platform's shell from the given directory: paths below it+-- start with @./@, and other paths under the home folder with @~@ or @$HOME@.+pasteCommand :: Shell -> FilePath -> [FilePath] -> [[T.Text]] -> T.Text+pasteCommand shell base homes = T.intercalate " | " . map command+  where+    command argv = case shell of+      Posix -> T.unwords (map posixArg argv)+      PowerShell -> case map powerShellArg argv of+        -- PowerShell reads a quoted first word as a string, not a command to run.+        first : rest | "'" `T.isPrefixOf` first || "\"" `T.isPrefixOf` first -> T.unwords ("&" : first : rest)+        ws -> T.unwords ws+    belowBase a = case belowHome [base] (T.unpack a) of+      Just [] -> Just "."+      Just rest -> Just (T.pack ('.' : FilePath.pathSeparator : FilePath.joinPath rest))+      Nothing -> Nothing+    posixArg a = maybe (showCmdArgsAt homes [a]) escapeArg (belowBase a)+    powerShellArg a = case belowBase a of+      Just rel -> shellArgument PowerShell rel+      Nothing -> fromMaybe (shellArgument PowerShell a) (homeShellPath PowerShell homes (T.unpack a))++-- hledger discovers config files from its working directory, and scripts can read+-- relative paths. Preserve that directory when replaying a command at the ledger root.+externalCommand :: HasBaseDir o => o -> External -> IO T.Text+externalCommand opts ext = do   homes <- homeDirs-  logVerbose opts ch $ showCmdArgsAt homes (cmd : args)-  if (List.null extraCmdLabels) then return () else logVerbose opts ch $ T.intercalate "\n" extraCmdLabels-  timed@((ec, stdOut, stdErr), diff) <- timeAction action-  stdoutLogger ch stdOut-  stderrLogger ch stdErr-  logVerbose opts ch $ "End:   " <> cmdLabel <> " " <> T.pack (show ec) <> " (" <> T.pack (show diff) <> ")"-  return timed+  cwd <- Directory.getCurrentDirectory+  let command = pasteCommand nativeShell cwd homes (extCommand ext)+      location = fromMaybe (shellArgument nativeShell (T.pack cwd)) (homeShellPath nativeShell homes cwd)+  pure $ if FilePath.equalFilePath cwd (toFilePath (baseDir opts))+    then command+    else case nativeShell of+      Posix -> "(cd " <> location <> " && " <> command <> ")"+      PowerShell -> "& { Push-Location -LiteralPath " <> location <> " -ErrorAction Stop; try { "+                    <> command <> " } finally { Pop-Location } }" +-- | In verbose mode, say which program is about to run and show its command line.+announceExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> IO ()+announceExternal opts ch ext = when (verbose opts) $ do+  command <- externalCommand opts ext+  color <- useColor H.stderr+  logToChannel ch $+    "Running " <> extName ext <> " to " <> extPurpose ext <> ":\n"+      <> "  " <> dim color "$" <> " " <> cyan color command++-- | Run an external program, show what it printed, and stop with its exit code when it fails.+-- Returns its standard output, which goes to the console only when 'showStdout' is set.+runExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> Bool -> ProcInput -> IO T.Text+runExternal opts ch ext showStdout input = do+  announceExternal opts ch ext+  (result@(_, out, _), duration) <- timeAction (runProcessCaptured input)+  checkExternal opts ch ext showStdout duration result+  return out++-- | Show what an external program printed. When it failed, stop with its exit code and+-- say plainly that the error came from that program.+checkExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> Bool -> NominalDiffTime -> FullOutput -> IO ()+checkExternal opts ch ext showStdout duration (status, out, err) = do+  outColor <- useColor H.stdout+  errColor <- useColor H.stderr+  case status of+    ExitSuccess -> do+      logVerbose opts ch (green errColor (extName ext <> " finished") <> " in " <> showDuration duration)+      when showStdout $ passOn (channelOut ch) outColor "Output" out+      passOn (channelErr ch) errColor "Messages" err+    ExitFailure code -> do+      when showStdout $ passOn (channelOut ch) outColor "Output" out+      commandLines <- if verbose opts then pure [] else do+        command <- externalCommand opts ext+        pure ["Command:", "  " <> command]+      let section label txt = outputSection errColor (verbose opts) (label <> " from " <> extName ext) txt+      errExit code ch $ T.intercalate "\n" $+        [ red errColor (bold errColor (extName ext <> " failed with exit code " <> T.pack (show code)))+            <> " while trying to " <> extPurpose ext <> "."+        ]+          -- Verbose output showed the command just before it ran.+          ++ commandLines+          ++ (if showStdout then [] else section "Output" out)+          ++ section "Error output" err+          ++ extHints ext+  where+    -- In verbose mode, mark the output as the program's; otherwise pass it on as it is.+    passOn write color label txt+      | T.null txt = return ()+      | verbose opts = case outputSection color True (label <> " from " <> extName ext) txt of+          heading : lns -> logToChannel ch heading >> write (T.unlines lns)+          [] -> return ()+      | otherwise = write txt++-- | Another program's output under a label, each line behind a bar. Without 'complete',+-- long output is trimmed.+outputSection :: Bool -> Bool -> T.Text -> T.Text -> [T.Text]+outputSection color complete label txt+  | T.null (T.strip txt) = []+  | otherwise = (label <> ":") : map bar (T.lines (if complete then txt else trimToolOutput txt))+  where+    -- Keep the program's own colours from spilling into the lines that follow.+    bar l = dim color "  │" <> " " <> l <> (if color && "\ESC" `T.isInfixOf` l then resetCode else "")++showDuration :: NominalDiffTime -> T.Text+showDuration d+  | d < 1 = T.pack (show (round (d * 1000) :: Integer)) <> " ms"+  | otherwise = T.pack (printf "%.1f s" (realToFrac d :: Double))+ -- Drain both output pipes concurrently so a full stderr pipe cannot stall stdout. runProcessCaptured :: ProcInput -> IO FullOutput runProcessCaptured (cmd, args, input) = do@@ -285,47 +403,6 @@     status <- Process.waitForProcess handle     return (status, stdoutText, stderrText) -timeAndExitOnErr ::-  (HasVerbosity o) =>-  o ->-  TChan LogMessage ->-  T.Text ->-  (TChan LogMessage -> T.Text -> IO ()) ->-  (TChan LogMessage -> T.Text -> IO ()) ->-  ProcInput ->-  IO FullTimedOutput-timeAndExitOnErr opts ch cmdLabel = timeAndExitOnErr' opts ch cmdLabel [] []--timeAndExitOnErr' ::-  (HasVerbosity o) =>-  o ->-  TChan LogMessage ->-  T.Text ->-  [T.Text] ->-  [T.Text] ->-  (TChan LogMessage -> T.Text -> IO ()) ->-  (TChan LogMessage -> T.Text -> IO ()) ->-  ProcInput ->-  IO FullTimedOutput-timeAndExitOnErr' opts ch cmdLabel extraCmdLabels hints stdoutLogger stderrLogger input@(cmd, args, _) = do-  let action = runProcessCaptured input-  -- On failure the error output goes into the (trimmed) message, not straight to the console.-  timed@((ec, stdOut, stdErr), _) <- logTimedAction opts ch cmdLabel extraCmdLabels cmd args stdoutLogger (\_ _ -> return ()) action-  case ec of-    ExitFailure i -> do-      homes <- homeDirs-      let cmdText = showCmdArgsAt homes (cmd : args)-      let msgOut = descriptiveOutput "Standard output" (trimToolOutput stdOut)-      let msgErr = descriptiveOutput "Error output" (trimToolOutput stdErr)-      let exitMsg =-            "Failed " <> cmdLabel <> " (exit code " <> T.pack (show i) <> ").\n"-              <> "Command: " <> cmdText <> "\n" <> msgOut <> msgErr-              <> T.unlines hints-      errExit i ch exitMsg-    ExitSuccess -> do-      stderrLogger ch stdErr-      return timed- -- | Keep a tool's output readable: at most 20 lines of at most 300 characters. -- hledger can echo a whole statement on one line when parsing fails. trimToolOutput :: T.Text -> T.Text@@ -338,7 +415,7 @@     maxWidth = 300     cut l = if T.length l > maxWidth then T.take maxWidth l <> "…" else l     dropped = case length ls - maxLines of-      n | n > 0 -> ["… (" <> T.pack (show n) <> " more lines)"]+      n | n > 0 -> ["… (" <> T.pack (show n) <> " more lines; run with --verbose to see them all)"]       _ -> []  parAwareActions :: (HasSequential o, HasBatchSize o) => o -> [IO a] -> IO [a]@@ -350,15 +427,25 @@   | null (drop batch todo) = (done ++) <$> mapConcurrently id todo   | otherwise = (done ++) <$> mapConcurrentlyBounded batch id todo -verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> AbsFile -> IO Bool-verboseTestFile opts ch p = do+-- | Whether the file exists, saying in verbose mode what was looked for and where.+verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> T.Text -> AbsFile -> IO Bool+verboseTestFile opts ch what p = do   fileExists <- Path.doesFileExist p+  color <- useColor H.stderr   let rel = T.pack (relativeToBase opts (toFilePath p))-  if fileExists-    then logVerbose opts ch ("Found '" <> rel <> "'")-    else logVerbose opts ch ("Looked for but did not find '" <> rel <> "'")+  logVerbose opts ch $+    if fileExists+      then green color ("Found " <> what) <> ": " <> rel+      else "No " <> what <> " at " <> rel   return fileExists +-- | The first of these files that exists, saying in verbose mode which ones were looked for.+findFirstFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> T.Text -> [AbsFile] -> IO (Maybe AbsFile)+findFirstFile _ _ _ [] = return Nothing+findFirstFile opts ch what (file : rest) = do+  exists <- verboseTestFile opts ch what file+  if exists then return (Just file) else findFirstFile opts ch what rest+ needsRegeneration :: AbsFile -> AbsFile -> IO Bool needsRegeneration src target = do   -- A missing or unreadable timestamp also requires regeneration.@@ -422,13 +509,16 @@ writeGeneratedText path contents = do   -- Replace the file behind an existing symlink and keep its permissions.   target <- Directory.canonicalizePath path >>= Path.parseAbsFile-  withStagedOutput target $ \staged -> do-    let tmp = toFilePath staged-    exists <- Path.doesFileExist target-    when exists $ do-      T.writeFile tmp ""-      Directory.copyPermissions (toFilePath target) tmp-    T.writeFile tmp contents+  withStagedOutput target (writeStagedText target contents)++writeStagedText :: AbsFile -> T.Text -> AbsFile -> IO ()+writeStagedText target contents staged = do+  let tmp = toFilePath staged+  exists <- Path.doesFileExist target+  when exists $ do+    T.writeFile tmp ""+    Directory.copyPermissions (toFilePath target) tmp+  T.writeFile tmp contents  textFileNeedsUpdate :: FilePath -> T.Text -> IO Bool textFileNeedsUpdate path contents = either (const True) (/= contents) <$> tryIOError (T.readFile path)
src/Hledger/Flow/DocHelpers.hs view
@@ -18,19 +18,25 @@ data DocLink   = Docs   | GettingStarted+  | IncrementalImportFallback   | InputFiles   | RulesFiles   | PreprocessScript   | SetLedgerFile+  | StatusCommand+  | Reports   deriving (Bounded, Enum, Eq, Show)  docSlug :: DocLink -> Text docSlug Docs = "docs" docSlug GettingStarted = "getting-started"+docSlug IncrementalImportFallback = "incremental-import-fallback" docSlug InputFiles = "input-files" docSlug RulesFiles = "rules-files" docSlug PreprocessScript = "preprocess-script" docSlug SetLedgerFile = "set-ledger-file"+docSlug StatusCommand = "status"+docSlug Reports = "reports"  siteURL :: Text siteURL = "https://hledger-flow.github.io/"
src/Hledger/Flow/Environment.hs view
@@ -68,10 +68,15 @@   optsEnvironment homes info o     ++ [ setting "Run from" runFrom "",          setting "Start year" (maybe "all years" (\y -> tshow y <> " onwards") (importStartYear o)) "--start-year",-         setting "Files" (if onlyNewFiles o then "only new or changed" else "regenerate all") "--new-files-only",+         files,          parallelism o        ]   where+    files = case importPolicy o of+      Full -> setting "Files" "regenerate all" "--new-files-only"+      TimestampOnly -> setting "Files" "only new or changed" "--new-files-only"+      RunScripts -> setting "Files" "incremental (run-scripts)" "--incremental"+      SkipUnchangedScripts -> setting "Files" "incremental (skip-unchanged-scripts)" "--incremental"     runFrom = case dropTrailingPathSeparator (toFilePath (importRunDir o)) of       "." -> ". (whole ledger)"       dir -> T.pack dir
src/Hledger/Flow/Import/CSVImport.hs view
@@ -14,7 +14,8 @@ import Hledger.Flow.Environment (importSettings, renderSettings) import Hledger.Flow.BaseDir (effectiveRunDir, relativeToBase) import Hledger.Flow.Common-import Hledger.Flow.DocHelpers (DocLink (..), docURL)+import Hledger.Flow.Import.Incremental+import Hledger.Flow.Import.Conversion import Hledger.Flow.Import.ImportHelpers import Hledger.Flow.Import.IncludeFiles (writeIncludesFromInventory, writeToplevelAllYearsInclude) import Hledger.Flow.Import.Inventory@@ -25,12 +26,7 @@ import qualified Hledger.Flow.Types as FlowTypes import qualified Path import qualified Path.IO as PathIO-import System.Environment (getEnv)-import System.Exit (ExitCode (..), exitWith)-import System.IO (hClose)-import qualified System.Info as Sys-import qualified System.Process as Process-import System.FilePath (dropTrailingPathSeparator, takeExtension, (</>))+import System.FilePath (dropTrailingPathSeparator) import qualified Data.List as List import Hledger.Flow.Status (plural) import Hledger.Flow.Status.Gather (accountName)@@ -40,18 +36,19 @@  importCSVs :: RuntimeOptions -> IO () importCSVs opts = withConsoleLogging $ \ch -> do+  homes <- homeDirs   when (showOptions opts) $ do-    homes <- homeDirs     info <- hledgerInfo opts     channelOutLn ch (T.unlines (renderSettings id (importSettings homes info opts)))-  logVerbose opts ch "Starting import"+  logVerbose opts ch $ "Base directory: " <> T.pack (tildePath homes (dropTrailingPathSeparator (Path.toFilePath (baseDir opts))))   (journals, diff) <- timeAction $ importCSVs' opts ch   let importRoot = baseDir opts Path.</> [Path.reldir|import|]       effectiveDir = effectiveRunDir (baseDir opts) (importRunDir opts)       scope = T.pack (dropTrailingPathSeparator (relativeToBase opts (Path.toFilePath effectiveDir)))       accounts = List.nub (map (accountName importRoot . fst) journals)+  when (verbose opts) $ channelErr ch "\n"   channelOutLn ch (importSummary scope (length journals) (length (filter snd journals)) accounts)-  logVerbose opts ch ("Import took " <> T.pack (show diff))+  logVerbose opts ch ("Import finished in " <> showDuration diff)  -- | One line saying what import did. importSummary :: T.Text -> Int -> Int -> [T.Text] -> T.Text@@ -71,42 +68,50 @@   let effectiveDir = effectiveRunDir (baseDir opts) (importRunDir opts)       importRoot = baseDir opts Path.</> [Path.reldir|import|]       selected = inImportScope effectiveDir (fromMaybe 0 (importStartYear opts))-  let startYearMsg = maybe " " (\y -> " (for the year " <> T.pack (show y) <> " and onwards) ") (importStartYear opts)-  let dirText = T.pack (Path.toFilePath effectiveDir)-  logVerbose opts ch $ "Collecting input files" <> startYearMsg <> "from " <> dirText+  let startYearMsg = maybe "" (\y -> " from " <> T.pack (show y) <> " onwards") (importStartYear opts)+  let relative = T.pack . relativeToBase opts . Path.toFilePath+  logVerbose opts ch $ "Looking for statements" <> startYearMsg <> " in " <> relative effectiveDir+    <> " (import/<owner>/<bank>/<account>/1-in/<year>/)"   (inventory, diff) <- timeAction $ findImportFiles importRoot   let inputFiles = filter selected (stageFiles importRoot [Path.reldir|1-in|] inventory)       fileCount = length inputFiles   collisions <- conflictingOutputs inputFiles-  let relative = T.pack . relativeToBase opts . Path.toFilePath   unless (null collisions) $     errExit 1 ch ("Multiple input files map to the same output:\n" <>       T.intercalate "\n" [relative output <> " <- " <> T.intercalate ", " (map relative sources)                          | (output, sources) <- collisions])-  logVerbose opts ch $ "Found " <> T.pack (show fileCount) <> " input files" <> startYearMsg <> "in " <> T.pack (show diff) <> ". Proceeding with import..."+  logVerbose opts ch $ "Found " <> plural fileCount "statement" <> " in " <> showDuration diff   -- Glob escaping depends on the hledger version. Check it before any worker   -- writes output; conversions reuse this result. directives.journal and   -- cached CSVs share these paths' base and statement names.   when (any (any isGlobMetacharacter . Path.toFilePath) inputFiles) $ void (hledgerVersion opts)-  importedJournals <- parAwareActions opts (map (extractAndImport opts ch importRoot) inputFiles)--  -- Running a preprocess hook also forces conversion. When every journal was-  -- skipped, neither a hook nor a conversion could have changed the inventory.-  let cached = if any snd importedJournals then Nothing else Just inventory+  let run tracking = parAwareActions opts (map (extractAndImport tracking opts ch importRoot) inputFiles)+  outcomes <- case importPolicy opts of+    RunScripts -> withIncrementalRun opts ch fileCount (run . Just)+    SkipUnchangedScripts -> withIncrementalRun opts ch fileCount (run . Just)+    _ -> run Nothing+  let importedJournals = [(journal, generated) | (journal, generated, _) <- outcomes]+      cached = if or [refresh | (_, _, refresh) <- outcomes] then Nothing else Just inventory   ((refreshed, orphans, journalsOnDisk), journalFindTime) <- timeAction $ reconciliationInventory importRoot cached   forM_ (filter selected orphans) $ \file -> do     target <- moveToTrash file     channelOutLn ch $ "Moved orphaned output " <> relative file <> " to " <> relative target-  (_, writeIncludeTime1) <- timeAction $ writeIncludesFromInventory opts ch importRoot refreshed journalsOnDisk-  (_, writeIncludeTime2) <- timeAction $ writeToplevelAllYearsInclude opts-  let includeGenTime = journalFindTime + writeIncludeTime1 + writeIncludeTime2-  logVerbose opts ch $ "Wrote include files for " <> T.pack (show (length journalsOnDisk)) <> " generated journals in " <> T.pack (show includeGenTime)+  withLogBlock opts True ch "Include files" $ \_ -> do+    (_, writeIncludeTime1) <- timeAction $ writeIncludesFromInventory opts ch importRoot refreshed journalsOnDisk+    (_, writeIncludeTime2) <- timeAction $ writeToplevelAllYearsInclude opts+    let includeGenTime = journalFindTime + writeIncludeTime1 + writeIncludeTime2+    logVerbose opts ch $ "Updated include files for " <> plural (length journalsOnDisk) "generated journal" <> " in " <> showDuration includeGenTime   return importedJournals -extractAndImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsDir -> AbsFile -> IO (AbsFile, FileWasGenerated)-extractAndImport opts ch root inputFile = do+extractAndImport :: Maybe IncrementalRun -> RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsDir -> AbsFile -> IO (AbsFile, Bool, Bool)+extractAndImport tracking opts parent root inputFile =+  withLogBlock opts (sequential opts) parent ("Statement " <> T.pack (relativeToBase opts (Path.toFilePath inputFile))) $ \ch ->   case extractImportDirs root inputFile of-    Right importDirs -> importCSV opts ch importDirs inputFile+    Right importDirs -> case tracking of+      Just run -> importStatement run opts ch importDirs inputFile+      Nothing -> do+        (output, generated) <- importCSV opts ch importDirs inputFile+        pure (output, generated, generated)     Left errorMessage -> do       errExit 1 ch errorMessage @@ -122,148 +127,26 @@   PathIO.createDirIfMissing True (Path.parent journalOut)   if shouldImport     then do-      hook <- findHook opts ch (accountDir importDirs Path.</> [Path.relfile|construct|])+      hook <- findHook opts ch "construct script" (accountDir importDirs Path.</> [Path.relfile|construct|])       case hook of-        Just script -> withStagedOutput journalOut (customConstruct opts ch script importDirs csvFile)+        Just script -> withStagedOutput journalOut (customConstruct opts ch script importDirs csvFile journalOut)         Nothing -> hledgerImport opts ch importDirs csvFile journalOut+      logVerbose opts ch ("Wrote " <> T.pack (relativeToBase opts (Path.toFilePath journalOut)))     else logNewFileSkip opts ch "import" journalOut   return (journalOut, shouldImport)  preprocessIfNeeded :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO (AbsFile, Bool) preprocessIfNeeded opts ch dirs src csvOut = do-  hook <- findHook opts ch (accountDir dirs Path.</> [Path.relfile|preprocess|])+  hook <- findHook opts ch "preprocess script" (accountDir dirs Path.</> [Path.relfile|preprocess|])   case hook of     Nothing -> do-      cached <- verboseTestFile opts ch csvOut-      if cached-        then do-          stale <- needsRegeneration src csvOut-          when stale $ ioError $ userError $-            relativeToBase opts (Path.toFilePath src) ++ " is newer than its cached CSV "-              ++ relativeToBase opts (Path.toFilePath csvOut) ++ ". Restore the preprocess hook to regenerate it,"-              ++ " or move the cached CSV aside to use the source directly. Then run hledger-flow import."-          return (csvOut, False)-        else return (src, False)+      effective <- savedInput opts ch src csvOut+      return (effective, False)     Just script -> do       regenerate <- if onlyNewFiles opts then needsRegeneration src csvOut else return True       if regenerate-        then withStagedOutput csvOut (preprocess opts ch script dirs src)+        then do+          withStagedOutput csvOut (preprocess opts ch script dirs src csvOut)+          logVerbose opts ch ("Wrote " <> T.pack (relativeToBase opts (Path.toFilePath csvOut)))         else logNewFileSkip opts ch "preprocess" csvOut       return (csvOut, regenerate)--findHook :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> IO (Maybe AbsFile)-findHook opts ch base = do-  alternatives <- if Sys.os == "mingw32"-    then mapM (`Path.addExtension` base) [".ps1", ".exe"]-    else return []-  first (base : alternatives)-  where-    first [] = return Nothing-    first (file : rest) = do-      exists <- verboseTestFile opts ch file-      if exists then return (Just file) else first rest--hookCommand :: AbsFile -> [T.Text] -> IO (T.Text, [T.Text])-hookCommand script args-  | Sys.os == "mingw32" && takeExtension path == ".ps1" = do-      root <- getEnv "SystemRoot"-      let shell = root </> "System32" </> "WindowsPowerShell" </> "v1.0" </> "powershell.exe"-      -- Windows PowerShell -File treats construct's literal "-" as an option.-      -- Invoke the script with literal strings and propagate its exit status.-      let command = "[Console]::OutputEncoding = New-Object Text.UTF8Encoding($false); & "-            <> T.unwords (map powerShellLiteral (T.pack path : args))-            <> "; if (!$?) { if ($LASTEXITCODE) { exit $LASTEXITCODE }; exit 1 }"-      return (T.pack shell, ["-NoProfile", "-NonInteractive", "-OutputFormat", "Text", "-Command", command])-  | otherwise = return (T.pack path, args)-  where-    path = Path.toFilePath script--logNewFileSkip :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> AbsFile -> IO ()-logNewFileSkip opts ch logIdentifier absTarget =-  Control.Monad.when (onlyNewFiles opts) $ do-    let relativeTarget = relativeToBase opts (Path.toFilePath absTarget)-    logVerbose opts ch $-      "Skipping " <> logIdentifier <> " - output file is at least as new as source: '" <> T.pack relativeTarget <> "'"--preprocess :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()-preprocess opts ch script dirs src csvOut = do-  let args = map (T.pack . Path.toFilePath) [src, csvOut] ++ hookDirArgs dirs-  let relScript = relativeToBase opts (Path.toFilePath script)-  let relSrc = relativeToBase opts (Path.toFilePath src)-  let cmdLabel = "executing '" <> T.pack relScript <> "' on '" <> T.pack relSrc <> "'"-  (cmd, hookArgs) <- hookCommand script args-  _ <- timeAndExitOnErr opts ch cmdLabel channelOut channelErr (cmd, hookArgs, Nothing)-  return ()--hledgerImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()-hledgerImport opts ch importDirs csvFile journalOut = do-  let csvSrc = Path.toFilePath csvFile-  candidates <- rulesFileCandidates csvFile importDirs-  maybeRulesFile <- firstExistingFile candidates-  let relCSV = relativeToBase opts csvSrc-  case maybeRulesFile of-    Just rf -> do-      let relRules = relativeToBase opts (Path.toFilePath rf)-      let hledger = T.pack (Path.toFilePath (hledgerExecutable opts))-      let directives = directivesFile opts-      directivesExist <- PathIO.doesFileExist directives-      let paths = csvSrc : [Path.toFilePath directives | directivesExist]-      fileArgument <- if any (any isGlobMetacharacter) paths-        then hledgerFileArgument <$> hledgerInfo opts-        else return T.pack-      let directivesArgs = if directivesExist then ["--file", fileArgument (Path.toFilePath directives)] else []-      let csvArgs = ["--file", fileArgument csvSrc, "--rules-file", T.pack (Path.toFilePath rf)]-      let args = hledgerConfArgs opts ++ ["print", "--explicit"] ++ directivesArgs ++ csvArgs--      let cmdLabel = "importing '" <> T.pack relCSV <> "' using rules file '" <> T.pack relRules <> "'"-      let hints = ["Check the skip, fields and separator lines in " <> T.pack relRules <> ".", "Docs: " <> docURL RulesFiles]-      ((_, stdOut, _), _) <- timeAndExitOnErr' opts ch cmdLabel [] hints dummyLogger channelErr (hledger, args, Nothing)-      writeGeneratedText (Path.toFilePath journalOut) stdOut-    Nothing ->-      do-        let candidatesTxt = T.intercalate "\n" $ map (T.pack . relativeToBase opts . Path.toFilePath) candidates-        let msg =-              "I couldn't find an hledger rules file while trying to import\n" <> T.pack relCSV-                <> "\n\nI will happily use the first rules file I can find from any one of these "-                <> T.pack (show (length candidates)) <> " files:\n" <> candidatesTxt-                <> "\n\nHere is a bit of documentation about rules files that you may find helpful:\n"-                <> docURL RulesFiles-        errExit 1 ch msg--hookDirArgs :: ImportDirs b -> [T.Text]-hookDirArgs dirs = map (T.concat . importDirLines) [bankDir dirs, accountDir dirs, ownerDir dirs]--customConstruct :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()-customConstruct opts ch constructScript dirs csvFile journalOut = do-  let csvSrc = Path.toFilePath csvFile-  let relScript = relativeToBase opts (Path.toFilePath constructScript)-  let constructArgs = [T.pack csvSrc, "-"] ++ hookDirArgs dirs-  homes <- homeDirs-  let constructCmdText = "Running: " <> T.pack relScript <> " " <> showCmdArgsAt homes constructArgs-  (cmd, hookArgs) <- hookCommand constructScript constructArgs-  let hook = (Process.proc (T.unpack cmd) (map T.unpack hookArgs))-        { Process.std_in = Process.CreatePipe, Process.std_out = Process.CreatePipe }-  let hledger = T.pack (Path.toFilePath (hledgerExecutable opts))-  let args =-        hledgerConfArgs opts-          ++ [ "print",-               "--ignore-assertions",-               "--file",-               "-",-               "--output-file",-               T.pack (Path.toFilePath journalOut)-             ]--  let relSrc = relativeToBase opts csvSrc-  let cmdLabel = "executing '" <> T.pack relScript <> "' on '" <> T.pack relSrc <> "'"-  Process.withCreateProcess hook $ \input output _ handle -> do-    Just stdinPipe <- return input-    Just stdoutPipe <- return output-    hClose stdinPipe-    _ <- timeAndExitOnErr' opts ch cmdLabel [constructCmdText] [] channelOut channelErr-      (hledger, args, Just stdoutPipe)-    hClose stdoutPipe-    status <- Process.waitForProcess handle-    case status of-      ExitSuccess -> return ()-      ExitFailure _ -> exitWith status
+ src/Hledger/Flow/Import/Cache.hs view
@@ -0,0 +1,100 @@+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Hledger.Flow.Import.Cache+  ( StageRecord (..), StatementRecord (..), StageInputs+  , recordPath, readRecord, saveRecord+  , stageMiss, installOutput, digestIfPresent+  ) where++import Control.Monad (unless)+import Data.Aeson (FromJSON, ToJSON, eitherDecodeStrict', encode)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BL+import Data.Text (Text)+import qualified Data.Text as T+import GHC.Generics (Generic)+import Hledger.Flow.Cache (cacheDir)+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.PathHelpers+import qualified Path as P+import qualified Path.IO as P+import System.IO.Error (isDoesNotExistError, tryIOError)++type StageInputs = [(Text, Text)]++data StageRecord = StageRecord+  { stageInputs :: StageInputs+  , stageOutput :: Digest+  } deriving (Eq, Show, Generic, FromJSON, ToJSON)++data StatementRecord = StatementRecord+  { preprocessRecord :: Maybe StageRecord+  , journalRecord :: Maybe StageRecord+  } deriving (Eq, Show, Generic, FromJSON, ToJSON)++-- Bump when a conversion's semantics, cache key or serialization changes.+recordVersion :: Int+recordVersion = 2  -- Earlier snapshots could overwrite a differently spelled input.++recordPath :: AbsDir -> AbsFile -> IO AbsFile+recordPath base source = do+  let year = P.parent source+      account = P.parent (P.parent year)+  group <- P.stripProperPrefix (base P.</> [P.reldir|import|]) account+  pure $ cacheDir base P.</> [P.reldir|import|] P.</> group+              P.</> P.dirname year P.</> P.filename source++readRecord :: AbsFile -> IO (Maybe StatementRecord)+readRecord path = do+  result <- tryIOError (BS.readFile (P.toFilePath path))+  pure $ case result of+    Right bytes | Right (version, record) <- eitherDecodeStrict' bytes+                , version == recordVersion -> Just record+    _ -> Nothing++saveRecord :: AbsFile -> StatementRecord -> IO ()+saveRecord path record = withStagedOutput path $ \stage ->+  BL.writeFile (P.toFilePath stage) (encode (recordVersion, record))++digestIfPresent :: AbsFile -> IO (Maybe Digest)+digestIfPresent path = either (const Nothing) Just <$> tryIOError (digestFile path)++-- Return the first difference in the same ordered key used for cache lookup.+-- File dependencies use their resolved path as the key, including nested rules.+stageMiss :: StageInputs -> AbsFile -> Maybe StageRecord -> IO (Maybe Text)+stageMiss _ _ Nothing = pure (Just "no valid record for this stage")+stageMiss inputs output (Just record) =+  case changedInput (stageInputs record) inputs of+    Just key -> pure (Just ("changed input: " <> key))+    Nothing -> do+      result <- tryIOError (digestFile output)+      pure $ case result of+        Left err | isDoesNotExistError err -> Just ("missing output: " <> path)+                 | otherwise -> Just ("could not read output: " <> path)+        Right digest | digest /= stageOutput record -> Just ("changed output: " <> path)+                     | otherwise -> Nothing+  where+    path = T.pack (P.toFilePath output)+    changedInput [] [] = Nothing+    changedInput ((key,_):_) [] = Just key+    changedInput [] ((key,_):_) = Just key+    changedInput (old:xs) (new:ys)+      | old == new = changedInput xs ys+      | otherwise = Just (fst new)++-- The writer owns its staged file until it returns. Hash it before publishing:+-- another process can install a different output as soon as the rename finishes.+installOutput :: Bool -> AbsFile -> (AbsFile -> IO ()) -> IO Digest+installOutput keepIdentical target writeOutput = do+  let dir = P.parent target+  P.createDirIfMissing True dir+  P.withTempDir dir ".hledger-flow-" $ \tmp -> do+    let staged = tmp P.</> P.filename target+    writeOutput staged+    digest <- digestFile staged+    existing <- if keepIdentical then digestIfPresent target else pure Nothing+    unless (existing == Just digest) $ P.renameFile staged target+    pure digest
+ src/Hledger/Flow/Import/Conversion.hs view
@@ -0,0 +1,193 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+module Hledger.Flow.Import.Conversion+  ( findHook, preprocess, savedInput, logNewFileSkip, hledgerImport, customConstruct+  , hookDirArgs, RulesPaths(..), convertRules, constructWith, rulesArguments, constructArguments, pinnedConfig+  ) where++import Control.Concurrent.STM+import Control.Monad+import qualified Data.Text as T+import Hledger.Flow.DateTime (timeAction)+import Hledger.Flow.BaseDir (relativeToBase)+import Hledger.Flow.Common+import Hledger.Flow.DocHelpers (DocLink (..), docURL)+import Hledger.Flow.Import.ImportHelpers+import Hledger.Flow.Import.Types+import Hledger.Flow.Logging+import Hledger.Flow.PathHelpers (AbsFile, isGlobMetacharacter)+import Hledger.Flow.RuntimeOptions+import qualified Hledger.Flow.Types as FlowTypes+import qualified Path+import qualified Path.IO as PathIO+import System.Environment (getEnv)+import System.Exit (ExitCode (..))+import Control.Concurrent.Async (concurrently)+import qualified Data.Text.IO as T+import System.IO (hClose, stderr)+import qualified System.Info as Sys+import qualified System.Process as Process+import System.FilePath (takeExtension, (</>))+import Prelude hiding (putStrLn, take, writeFile)++findHook :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> AbsFile -> IO (Maybe AbsFile)+findHook opts ch what base = do+  alternatives <- if Sys.os == "mingw32"+    then mapM (`Path.addExtension` base) [".ps1", ".exe"]+    else return []+  findFirstFile opts ch what (base : alternatives)++hookCommand :: AbsFile -> [T.Text] -> IO (T.Text, [T.Text])+hookCommand script args+  | Sys.os == "mingw32" && takeExtension path == ".ps1" = do+      root <- getEnv "SystemRoot"+      let shell = root </> "System32" </> "WindowsPowerShell" </> "v1.0" </> "powershell.exe"+      -- Windows PowerShell -File treats construct's literal "-" as an option.+      -- Invoke the script with literal strings and propagate its exit status.+      let command = "[Console]::OutputEncoding = New-Object Text.UTF8Encoding($false); & "+            <> T.unwords (map powerShellLiteral (T.pack path : args))+            <> "; if (!$?) { if ($LASTEXITCODE) { exit $LASTEXITCODE }; exit 1 }"+      return (T.pack shell, ["-NoProfile", "-NonInteractive", "-OutputFormat", "Text", "-Command", command])+  | otherwise = return (T.pack path, args)+  where+    path = Path.toFilePath script++logNewFileSkip :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> AbsFile -> IO ()+logNewFileSkip opts ch logIdentifier absTarget =+  Control.Monad.when (onlyNewFiles opts) $ do+    let relativeTarget = relativeToBase opts (Path.toFilePath absTarget)+    logVerbose opts ch $+      "Skipping " <> logIdentifier <> ": " <> T.pack relativeTarget <> " is at least as new as its source"++-- | Run the preprocess script, writing to the staged file. The command shown names the final+-- output instead, so a pasted command writes where hledger-flow does.+preprocess :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()+preprocess opts ch script dirs src csvOut staged = do+  let args out = map (T.pack . Path.toFilePath) [src, out] ++ hookDirArgs dirs+      relSrc = relativeToBase opts (Path.toFilePath src)+      external = External (relativeHook opts script) ("preprocess " <> T.pack relSrc)+        [T.pack (Path.toFilePath script) : args csvOut] []+  (cmd, hookArgs) <- hookCommand script (args staged)+  void $ runExternal opts ch external True (cmd, hookArgs, Nothing)++relativeHook :: RuntimeOptions -> AbsFile -> T.Text+relativeHook opts = T.pack . relativeToBase opts . Path.toFilePath++savedInput :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> AbsFile -> IO AbsFile+savedInput opts ch src csvOut = do+  cached <- verboseTestFile opts ch "preprocessed CSV from an earlier run" csvOut+  if cached+    then do+      stale <- needsRegeneration src csvOut+      when stale $ ioError $ userError $+        relativeToBase opts (Path.toFilePath src) ++ " is newer than its cached CSV "+          ++ relativeToBase opts (Path.toFilePath csvOut) ++ ". Restore the preprocess script to regenerate it,"+          ++ " or move the cached CSV aside to use the source directly. Then run hledger-flow import."+      return csvOut+    else return src++hledgerImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()+hledgerImport opts ch importDirs csvFile journalOut = do+  let csvSrc = Path.toFilePath csvFile+  candidates <- rulesFileCandidates csvFile importDirs+  maybeRulesFile <- findFirstFile opts ch "rules file" candidates+  let relCSV = relativeToBase opts csvSrc+  case maybeRulesFile of+    Just rf -> do+      let directives = directivesFile opts+      exists <- PathIO.doesFileExist directives+      let paths = RulesPaths csvFile rf (if exists then Just directives else Nothing)+      stdOut <- convertRules opts ch (hledgerExecutable opts) paths paths (hledgerConfArgs opts) pure+      writeGeneratedText (Path.toFilePath journalOut) stdOut+    Nothing ->+      do+        let candidatesTxt = T.intercalate "\n" $ map (T.pack . relativeToBase opts . Path.toFilePath) candidates+        let msg =+              "I couldn't find an hledger rules file while trying to import\n" <> T.pack relCSV+                <> "\n\nI will happily use the first rules file I can find from any one of these "+                <> T.pack (show (length candidates)) <> " files:\n" <> candidatesTxt+                <> "\n\nHere is a bit of documentation about rules files that you may find helpful:\n"+                <> docURL RulesFiles+        errExit 1 ch msg++data RulesPaths = RulesPaths AbsFile AbsFile (Maybe AbsFile)++-- The journal cache uses these same argument builders with stable path labels.+pinnedConfig :: Maybe T.Text -> [T.Text]+pinnedConfig = maybe ["--no-conf"] (\path -> ["--conf", path])++rulesArguments :: [T.Text] -> T.Text -> T.Text -> Maybe T.Text -> [T.Text]+rulesArguments conf input rules directives =+  conf ++ ["print", "--explicit"] ++ maybe [] (\path -> ["--file", path]) directives+       ++ ["--file", input, "--rules-file", rules]++constructArguments :: [T.Text] -> T.Text -> [T.Text]+constructArguments conf out = conf ++ ["print", "--ignore-assertions", "--file", "-", "--output-file", out]++convertRules :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> RulesPaths -> RulesPaths -> [T.Text]+             -> (T.Text -> IO T.Text) -> IO T.Text+convertRules opts ch executable original@(RulesPaths csv rules _) actual config translate = do+  let shown = T.pack . relativeToBase opts . Path.toFilePath+      hledger = T.pack (Path.toFilePath (hledgerExecutable opts))+      args conf (RulesPaths input rf directives) = do+        let paths = input : maybe [] pure directives+        fileArg <- if any (any isGlobMetacharacter . Path.toFilePath) paths+          then hledgerFileArgument <$> hledgerInfo opts else pure T.pack+        pure $ rulesArguments conf (fileArg (Path.toFilePath input)) (T.pack (Path.toFilePath rf))+                 (fileArg . Path.toFilePath <$> directives)+  command <- args (hledgerConfArgs opts) original+  invocation <- args config actual+  let ext = External "hledger" ("convert " <> shown csv <> " with the rules in " <> shown rules)+              [hledger : command] ["Check the skip, fields and separator lines in " <> shown rules <> ".", "Docs: " <> docURL RulesFiles]+  announceExternal opts ch ext+  ((status,out,err),duration) <- timeAction $ runProcessCaptured (T.pack (Path.toFilePath executable),invocation,Nothing)+  diagnostic <- translate err+  diagnosticOut <- if status == ExitSuccess then pure out else translate out+  checkExternal opts ch ext False duration (status,diagnosticOut,diagnostic)+  pure out++hookDirArgs :: ImportDirs b -> [T.Text]+hookDirArgs dirs = map (T.concat . importDirLines) [bankDir dirs, accountDir dirs, ownerDir dirs]++-- | Pipe the construct script's output through hledger into the staged journal. The command+-- shown names the final journal instead, so a pasted command writes where hledger-flow does.+customConstruct :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()+customConstruct opts ch = constructWith opts ch (hledgerExecutable opts) (hledgerConfArgs opts) pure++constructWith :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> [T.Text] -> (T.Text -> IO T.Text) -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()+constructWith opts ch executable config translate constructScript dirs csvFile journalOut staged = do+  let csvSrc = Path.toFilePath csvFile+      relScript = relativeHook opts constructScript+      relSrc = T.pack (relativeToBase opts csvSrc)+      constructArgs = [T.pack csvSrc, "-"] ++ hookDirArgs dirs+      hledger = T.pack (Path.toFilePath (hledgerExecutable opts))+      hledgerArgs conf = constructArguments conf . T.pack . Path.toFilePath+      shown = [T.pack (Path.toFilePath constructScript) : constructArgs, hledger : hledgerArgs (hledgerConfArgs opts) journalOut]+      constructExt = External relScript ("construct a journal from " <> relSrc) shown []+      hledgerExt = External "hledger" ("read the journal " <> relScript <> " constructed from " <> relSrc) shown []+  (cmd, hookArgs) <- hookCommand constructScript constructArgs+  let hook = (Process.proc (T.unpack cmd) (map T.unpack hookArgs))+        { Process.std_in = Process.CreatePipe, Process.std_out = Process.CreatePipe, Process.std_err = Process.CreatePipe }+  announceExternal opts ch constructExt+  Process.withCreateProcess hook $ \input output errors handle -> do+    Just stdinPipe <- return input+    Just stdoutPipe <- return output+    Just stderrPipe <- return errors+    hClose stdinPipe+    (((hledgerStatus, converterOut, converterErr), duration), hookErr) <- concurrently+      (timeAction (runProcessCaptured (T.pack (Path.toFilePath executable), hledgerArgs config staged, Just stdoutPipe)))+      (T.hGetContents stderrPipe)+    mappedErr <- translate converterErr+    let converted = (hledgerStatus, converterOut, mappedErr)+    status <- Process.waitForProcess handle+    color <- useColor stderr+    -- When hledger stops reading early, the script fails with a broken pipe, so hledger's error+    -- comes first, along with anything the script said.+    let scriptNote = case status of+          ExitFailure code -> [relScript <> " also failed, with exit code " <> T.pack (show code) <> "."]+          ExitSuccess -> []+        scriptOutput = outputSection color (verbose opts) ("Error output from " <> relScript) hookErr+    when (hledgerStatus /= ExitSuccess) $+      checkExternal opts ch hledgerExt {extHints = scriptNote ++ scriptOutput} True duration converted+    checkExternal opts ch constructExt True duration (status, "", hookErr)+    checkExternal opts ch hledgerExt True duration converted
+ src/Hledger/Flow/Import/Dependencies.hs view
@@ -0,0 +1,360 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE DeriveFunctor #-}+{-# LANGUAGE OverloadedStrings #-}++module Hledger.Flow.Import.Dependencies+  ( Release (..), FallbackReason (..), ReuseDecision (..), Converter (..),+    ruleIncludes, nestedRuleIncludes, resolveRuleInclude, discoverRules, selectRules, isSeparatedValues,+    checkedIO, atPath, checkArguments, discoverConfig, discoverDirectives, checkConfig, checkDirectives,+    newConverterProbe, converterProbeWith+  ) where++import Control.Concurrent.MVar (modifyMVar, newMVar)+import Control.Monad (ap, foldM)+import qualified Data.ByteString as B+import Data.Char (isAlpha, isAlphaNum, isAscii, isControl, isDigit, isSpace, toLower, GeneralCategory(CurrencySymbol), generalCategory)+import Data.List (isPrefixOf)+import Data.Maybe (mapMaybe)+import qualified Data.Set as Set+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import qualified Data.Text.Read as TR+import Data.Time (getCurrentTimeZone, timeZoneMinutes)+import GHC.IO.Encoding (getLocaleEncoding, getFileSystemEncoding, getForeignEncoding)+import Hledger.Flow.Common (firstConfirmedFile, lookupHledgerConf, probeHledgerVersion, hledgerVersionNumber)+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.ImportHelpers (rulesFileCandidates)+import Hledger.Flow.Import.Types (ImportDirs)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile)+import qualified Hledger.Flow.RuntimeOptions as RT+import Hledger.Flow.Types (HledgerConf(..), hledgerConfPath)+import Path (Abs, parseAbsFile, toFilePath)+import qualified System.Directory as D+import qualified System.FilePath as F+import System.Exit (ExitCode(..))+import System.IO (IOMode(ReadMode), withBinaryFile)+import System.IO.Error (tryIOError, ioeGetFileName, ioeGetErrorString)+import qualified System.Info as Sys+#ifdef mingw32_HOST_OS+import System.Win32.Console (getConsoleCP, getConsoleOutputCP)+import System.Win32.NLS (getACP)+#endif++-- Exact releases have known reader quirks. Other eligible versions use syntax+-- shared by both readers, leaving ambiguous inputs to the ordinary converter.+data Release = V1524 | V1996 | OtherSupported deriving (Eq, Show)++data FallbackReason+  = FallbackReason {reasonScope :: T.Text, reasonDetail :: T.Text}+  | FileReason {reasonScope :: T.Text, reasonFile :: FilePath, reasonDetail :: T.Text}+  | OrdinaryInputError {reasonScope :: T.Text, reasonDetail :: T.Text}+  | CyclicRules {reasonScope :: T.Text, reasonDetail :: T.Text}+  deriving (Eq, Ord, Show)+data ReuseDecision a = Reusable a | Fallback FallbackReason deriving (Functor, Eq, Show)++instance Applicative ReuseDecision where+  pure = Reusable+  (<*>) = ap++instance Monad ReuseDecision where+  Reusable a >>= f = f a+  Fallback r >>= _ = Fallback r++atPath :: T.Text -> FilePath -> T.Text -> ReuseDecision a+atPath reason path detail = Fallback (FileReason reason path detail)++checkedIO :: T.Text -> FilePath -> IO (ReuseDecision a) -> IO (ReuseDecision a)+checkedIO reason path action = do+  result <- tryIOError action+  pure $ case result of+    Right a -> a+    Left e -> atPath reason (maybe path id (ioeGetFileName e)) (T.pack (ioeGetErrorString e))++physicalLines :: AbsFile -> B.ByteString -> ReuseDecision [T.Text]+physicalLines path bytes = case TE.decodeUtf8' bytes of+  Left _ -> atPath "unrecognised text encoding" (toFilePath path) "expected UTF-8"+  Right s -> pure (T.lines (T.replace "\r\n" "\n" s))++ruleIncludes :: Release -> AbsFile -> B.ByteString -> ReuseDecision [FilePath]+ruleIncludes release path bytes = do+  ls <- physicalLines path (dropUtf8Bom bytes)+  sequence (mapMaybe include (zip [1 :: Int ..] ls))+  where+    include (n,line)+      | Just raw <- T.stripPrefix "include " line = Just (checkPath n raw)+      | release == OtherSupported && looksLikeInclude line = Just (uncertain n)+      | otherwise = Nothing+    looksLikeInclude line = case T.words line of+      word:_ -> T.toCaseFold word == "include"+      [] -> False+    checkPath n raw+      | T.null name || T.any isControl name = uncertain n+      | release == OtherSupported && name /= T.stripEnd name = uncertain n+      | otherwise = pure (T.unpack name)+      where name = if release == V1996 then T.strip raw else T.stripStart raw+    uncertain n = atPath "uncertain rules include" (toFilePath path) ("line " <> T.pack (show n))++-- Older readers open nested rules through the native text handle. On POSIX a+-- CR at the end of an include line may be part of the filename.+nestedRuleIncludes :: Release -> AbsFile -> B.ByteString -> ReuseDecision [FilePath]+nestedRuleIncludes release path bytes+  | release /= V1996, any ambiguous (B.split 10 bytes) =+      atPath "uncertain nested rules include" (toFilePath path) "native line ending or byte-order mark"+  | otherwise = ruleIncludes release path bytes+  where+    ambiguous line = "include " `B.isPrefixOf` dropUtf8Bom line &&+                     (B.elem 13 line || dropUtf8Bom line /= line)++dropUtf8Bom :: B.ByteString -> B.ByteString+dropUtf8Bom b = maybe b id (B.stripPrefix (B.pack [0xef,0xbb,0xbf]) b)++-- Walk leading .. components relative to the including file's spelled path.+-- Verify the result against the filesystem's resolution, which can differ when a+-- symlink precedes .. . An internal symlink is mirrored under its original name.+resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)+resolveRuleInclude base parent child = checkedIO "uncertain rules include" raw $+  if F.hasDrive child || not (F.isRelative child) || any F.isPathSeparator (take 1 child)+    then pure (atPath "absolute rules include" child (T.pack (toFilePath parent)))+    else do+      let (ups, rest) = span (== "..") (filter (/= ".") (F.splitDirectories child))+          lexical = foldr (const F.takeDirectory) (F.takeDirectory (toFilePath parent)) ups F.</> F.joinPath rest+      if ".." `elem` rest || null rest || not (inside (toFilePath base) lexical)+        then pure outside+        else do+          actual <- D.canonicalizePath raw+          mirror <- D.canonicalizePath lexical+          root <- D.canonicalizePath (toFilePath base)+          if F.equalFilePath actual mirror && inside root actual+            then pure (maybe outside Reusable (parseAbsFile lexical))+            else pure outside+  where+    raw = F.takeDirectory (toFilePath parent) F.</> child+    outside = atPath "rules include escapes the ledger or cannot be mirrored" raw (T.pack (toFilePath parent))+    inside root path = F.splitDirectories (F.dropTrailingPathSeparator root) `isPrefixOf` F.splitDirectories path++discoverRules :: AbsDir -> Release -> AbsFile -> IO (ReuseDecision [(AbsFile,Digest)])+discoverRules base release = walk Set.empty+  where+    walk ancestors path = checkedIO "could not read rules file" (toFilePath path) $ do+      canonical <- D.canonicalizePath (toFilePath path)+      directory <- D.canonicalizePath (F.takeDirectory (toFilePath path))+      -- The same rules bytes can resolve includes in different directories.+      -- In 1.52.4 the first read also decodes line endings differently.+      let node = (canonical, directory, release /= V1996 && Set.null ancestors)+      if Set.member node ancestors && release /= V1996+        then pure $ Fallback (CyclicRules "cyclic rules include" (T.pack (toFilePath path)))+        else if Set.member node ancestors || Set.size ancestors >= 256+          then pure (atPath "cyclic or excessively deep rules include" (toFilePath path) "")+          else do+            bytes <- B.readFile (toFilePath path)+            let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes+            case scan release path bytes of+              Fallback r -> pure (Fallback r)+              Reusable children -> do+                nested <- descend (Set.insert node ancestors) path children+                pure (((path,digestBytes bytes):) <$> nested)+    descend _ _ [] = pure (pure [])+    descend ancestors path (c:cs) = do+      selected <- resolveRuleInclude base path c+      tree <- case selected of+        Fallback r -> pure (Fallback r)+        Reusable file -> walk ancestors file+      case tree of+        Fallback r -> pure (Fallback r)+        Reusable files -> fmap (files ++) <$> descend ancestors path cs++selectRules :: AbsFile -> ImportDirs Abs -> IO (ReuseDecision (Maybe AbsFile))+selectRules csv dirs = checkedIO "could not select rules file" (toFilePath csv) $ do+  candidates <- rulesFileCandidates csv dirs+  selected <- firstConfirmedFile (map toFilePath candidates)+  Reusable <$> traverse parseAbsFile selected++isSeparatedValues :: AbsFile -> Bool+isSeparatedValues path = map toLower (F.takeExtension (toFilePath path)) `elem` [".csv",".tsv",".ssv"]++checkArguments :: [T.Text] -> ReuseDecision ()+checkArguments args = case filter ("@" `T.isPrefixOf`) args of+  a:_ -> Fallback (FallbackReason ("indirect hledger argument " <> a) "")+  [] -> pure ()++discoverConfig :: Release -> Maybe HledgerConf -> IO (ReuseDecision (Maybe (AbsFile,Digest)))+discoverConfig release configured = checkedIO "could not discover hledger config" "hledger.conf" $ do+  let given = case configured of+        Just (GivenConf p) -> Just p+        _ -> Nothing+  case checkArguments (maybe [] (pure . T.pack) given) of+    Fallback r -> pure (Fallback r)+    Reusable () -> do+      selected <- lookupHledgerConf given+      case hledgerConfPath <$> selected of+        Nothing -> pure (pure Nothing)+        Just path | "~" `isPrefixOf` path || ".." `elem` F.splitDirectories path ->+          pure (atPath "uncertain config path" path "")+        Just path -> do+          absolute <- D.makeAbsolute path+          case parseAbsFile absolute of+            Nothing -> pure (atPath "uncertain config path" path "")+            Just absPath -> fmap Just <$> readDependency (checkConfig release) absPath++discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile,Digest)))+discoverDirectives file = checkedIO "could not discover directives" (toFilePath file) $ do+  exists <- firstConfirmedFile [toFilePath file]+  case exists of+    Nothing -> pure (pure Nothing)+    Just _ -> fmap Just <$> readDependency (checkDirectives V1996) file++readDependency :: (AbsFile -> B.ByteString -> ReuseDecision ()) -> AbsFile -> IO (ReuseDecision (AbsFile,Digest))+readDependency check path = checkedIO "could not read dependency" (toFilePath path) $ do+  bytes <- B.readFile (toFilePath path)+  pure ((path,digestBytes bytes) <$ check path bytes)++checkConfig :: Release -> AbsFile -> B.ByteString -> ReuseDecision ()+checkConfig release path bytes = do+  ls <- physicalLines path bytes+  () <$ foldM line "general" (zip [1 :: Int ..] ls)+  where+    -- Only canonical built-ins other than print have an inert section. Addons,+    -- aliases, abbreviations and names introduced in another release can have+    -- different scope and must be passed through to the ordinary converter.+    inactive = T.words $ "accounts activity add aregister balance balancesheet balancesheetequity cashflow check close codes commodities descriptions diff files help import incomestatement notes payees prices register repl rewrite roi run setup stats tags " <>+      case release of+        V1524 -> "commands demo test"+        V1996 -> "get holdings transactions"+        OtherSupported -> ""+    line section (n,raw)+      | T.null body || "#" `T.isPrefixOf` body = pure section+      | release /= V1996, "[" `T.isPrefixOf` body, T.stripStart raw /= raw = unsupported n body+      | Just start <- T.stripPrefix "[" body = case T.breakOn "]" start of+          (s,end) | not (T.null end), not ("#" `T.isInfixOf` s), let name = T.strip s,+                    name `elem` ("general":"print":inactive) -> option n name (T.drop 1 end)+          _ -> unsupported n body+      | otherwise = option n section body+      where body = T.strip raw+    option n section arg+      | section `elem` inactive = pure section+      | T.null setting = pure section+      | section == "print" && release /= V1524 && setting == "--layout=hledger1" = pure section+      | otherwise = unsupported n setting+      where setting = T.strip (T.takeWhile (/= '#') arg)+    unsupported n value = atPath "unsupported hledger config" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)++checkDirectives :: Release -> AbsFile -> B.ByteString -> ReuseDecision ()+checkDirectives _ path bytes = do+  ls <- physicalLines path (dropUtf8Bom bytes)+  case [(n,T.strip l) | (n,l) <- zip [1 :: Int ..] ls, not (allowed l)] of+    [] -> pure ()+    (n,value):_ -> atPath "unsupported journal directive" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)+  where+    allowed raw+      | T.null stripped || T.isPrefixOf ";" stripped || T.isPrefixOf "#" stripped || T.isPrefixOf "*" stripped = True+      | Just (c,_) <- T.uncons raw, isSpace c = False+      | otherwise = case T.words (T.takeWhile (/= ';') raw) of+          "account":xs -> not (null xs) && all (T.all accountChar) xs+          "commodity":xs -> amount xs+          "P":date:symbol:xs -> fullDate date && commodity symbol && amount xs+          _ -> False+      where stripped = T.strip raw+    accountChar c = isAlphaNum c || c `elem` (":_-/.'’" :: String)+    commodity s = not (T.null s) && T.all (\c -> isAlpha c || generalCategory c == CurrencySymbol) s+    amount [a,b] = (number a && commodity b) || (commodity a && number b)+    amount [a] =+      let (symbol,rest) = T.span (\c -> isAlpha c || generalCategory c == CurrencySymbol) a+          (value,suffix) = T.span (\c -> isDigit c || c `elem` (".,+-" :: String)) a+      in (commodity symbol && number rest) || (number value && commodity suffix)+    amount _ = False+    number s = case T.uncons (T.dropWhile (\c -> c == '+' || c == '-') s) of+      Just (c,rest) -> digit c && T.all (\x -> digit x || x == ',' || x == '.') rest+      _ -> False+    digit c = isAscii c && isDigit c+    fullDate s = case T.split (\c -> c == '-' || c == '/') s of+      [y,m,d] -> map T.length [y,m,d] == [4,2,2] && all (T.all digit) [y,m,d]+      _ -> False++data Converter = Converter+  { converterRelease :: Release+  , converterExecutable :: AbsFile+  , converterStamp :: FileStamp+  , converterKey :: [T.Text]+  } deriving (Eq, Show)++newConverterProbe :: RT.RuntimeOptions -> IO (IO (ReuseDecision Converter))+newConverterProbe opts = converterProbeWith (RT.hledgerExecutable opts) probeHledgerVersion++-- The expensive executable hash and version probe are shared by workers only+-- while the resolved executable and its handle-derived status still agree.+converterProbeWith :: AbsFile -> (AbsFile -> IO (ExitCode,T.Text)) -> IO (IO (ReuseDecision Converter))+converterProbeWith executable getVersion = do+  context <- checkedIO "could not identify the conversion context" original conversionContext+  cache <- newMVar Nothing+  pure $ case context of+    Fallback r -> pure (Fallback r)+    Reusable environment -> checkedIO "could not identify hledger executable" original $ do+      resolved <- D.canonicalizePath original >>= parseAbsFile+      let target = toFilePath resolved+      wrapped <- launcher original target+      if wrapped+        then pure (atPath "hledger launcher" original "select the real executable with --hledger-path")+        else modifyMVar cache $ \old -> do+          stamp <- statFile resolved+          case old of+            Just (p,s,c) | p == resolved && s == stamp -> pure (old, c)+            _ -> do+              observed <- observeFile resolved+              (status,version) <- getVersion resolved+              after <- statFile resolved+              live <- D.canonicalizePath original+              let key = [T.pack original,T.pack target,digestHex (observationDigest observed),version] ++ environment+                  result | after /= stamp || fileStamp observed /= stamp || not (F.equalFilePath live target) =+                             atPath "hledger executable changed" original ""+                         | status /= ExitSuccess = atPath "could not identify hledger version" original ""+                         | otherwise = case supportedRelease version of+                             Just release -> Reusable (Converter release resolved stamp key)+                             Nothing -> Fallback (FallbackReason ("unsupported hledger version " <> version) (T.pack original))+              pure (Just (resolved,stamp,result),result)+  where original = toFilePath executable++conversionContext :: IO (ReuseDecision [T.Text])+conversionContext = do+  offset <- timeZoneMinutes <$> getCurrentTimeZone+  locale <- show <$> getLocaleEncoding+  fs <- show <$> getFileSystemEncoding+  foreignEnc <- show <$> getForeignEncoding+#ifdef mingw32_HOST_OS+  pages <- sequence [show <$> getConsoleCP, show <$> getConsoleOutputCP, show <$> getACP]+#else+  let pages = []+#endif+  -- Rules and directives are inspected from their exact UTF-8 snapshot bytes.+  -- A different reader encoding could interpret a byte sequence as another path.+  pure $ if map toLower locale `elem` ["utf-8","utf8","cp65001"]+    then Reusable (map T.pack ([Sys.os,Sys.arch,show offset,locale,fs,foreignEnc] ++ pages))+    else Fallback (FallbackReason ("unsupported conversion encoding " <> T.pack locale) "")++supportedRelease :: T.Text -> Maybe Release+supportedRelease text = do+  let number = T.takeWhile (/= '-') (hledgerVersionNumber text)+      component part = case TR.decimal part of+        Right (n, rest) | T.null rest -> Just (n :: Integer)+        _ -> Nothing+  version <- traverse component (T.splitOn "." number)+  case version of+    [1,52,4] -> Just V1524+    [1,99,6] -> Just V1996+    _ | version >= [1,52,4] -> Just OtherSupported+      | otherwise -> Nothing++launcher :: FilePath -> FilePath -> IO Bool+launcher original target+  | any (\p -> map toLower (F.takeExtension p) `elem` [".cmd",".bat"]) [original,target] = pure True+  | otherwise = do+      shim <- firstConfirmedFile [F.replaceExtension p "shim" | p <- [original,target], map toLower (F.takeExtension p) == ".exe"]+      case shim of+        Just _ -> pure True+        Nothing -> do+          header <- withBinaryFile target ReadMode (flip B.hGet 4)+          pure $ not (B.take 2 header == "MZ" || header `elem` nativeMagic)+  where+    -- ELF, Mach-O (both byte orders) and universal Mach-O (32- and 64-bit).+    nativeMagic = map B.pack [[127,69,76,70],[0xfe,0xed,0xfa,0xce],[0xce,0xfa,0xed,0xfe],+      [0xfe,0xed,0xfa,0xcf],[0xcf,0xfa,0xed,0xfe],[0xca,0xfe,0xba,0xbe],[0xbe,0xba,0xfe,0xca],+      [0xca,0xfe,0xba,0xbf],[0xbf,0xba,0xfe,0xca]]
+ src/Hledger/Flow/Import/Fingerprint.hs view
@@ -0,0 +1,136 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE GeneralizedNewtypeDeriving #-}++module Hledger.Flow.Import.Fingerprint+  ( Digest (..), FileObservation (..), FileStamp (..),+    copyDigest, digestBytes, digestFile, observeFile, statFile+  ) where++import qualified Crypto.Hash.SHA256 as SHA+import Data.Aeson (FromJSON, ToJSON)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Builder as Builder+import qualified Data.ByteString.Lazy as BL+import Data.Text (Text)+import Data.Text.Encoding (decodeLatin1)+import Hledger.Flow.PathHelpers (AbsFile)+import Path (toFilePath)+import System.IO (Handle, IOMode (ReadMode, WriteMode), withBinaryFile)+#ifdef mingw32_HOST_OS+import qualified System.Win32.File as Win+import System.Win32.Time (FILETIME (..))+import System.Win32.Types (withHandleToHANDLE)+#else+import Control.Exception (bracket, bracketOnError)+import System.IO (hClose)+import qualified System.Posix.Files as Posix+import qualified System.Posix.IO as Posix+#endif+#ifdef darwin_HOST_OS+import Foreign.C.Error (throwErrnoPathIfMinus1_)+import Foreign.C.Types (CInt(..))+import Foreign.Marshal.Alloc (allocaBytes)+import Foreign.Ptr (Ptr, castPtr)+import Data.Word (Word8)+import System.Posix.Types (Fd(..))++foreign import ccall safe "hf_sha256_fd"+  sha256Fd :: Fd -> Ptr Word8 -> IO CInt+#endif++-- | Only the content digest is persisted across invocations.+newtype Digest = Digest {digestHex :: Text}+  deriving (Eq, Ord, Show, FromJSON, ToJSON)++data FileStamp = FileStamp+  { stampIdentity :: !(Integer, Integer),+    stampSize :: !Integer,+    stampModified :: !Rational,+    stampChanged :: !(Maybe Rational)+  }+  deriving (Eq, Show)++data FileObservation = FileObservation+  { fileStamp :: !FileStamp,+    observationDigest :: !Digest+  }+  deriving (Eq, Show)++hexDigest :: BS.ByteString -> Digest+hexDigest = Digest . decodeLatin1 . BL.toStrict . Builder.toLazyByteString+          . BS.foldr (mappend . Builder.word8HexFixed) mempty++digestBytes :: BS.ByteString -> Digest+digestBytes = hexDigest . SHA.hash++digestFile :: AbsFile -> IO Digest+digestFile path = withBinaryFile (toFilePath path) ReadMode (digestHandle (const (pure ())))++copyDigest :: AbsFile -> AbsFile -> IO Digest+copyDigest source destination =+  withBinaryFile (toFilePath source) ReadMode $ \input ->+    withBinaryFile (toFilePath destination) WriteMode $ \output ->+      digestHandle (BS.hPut output) input++digestHandle :: (BS.ByteString -> IO ()) -> Handle -> IO Digest+digestHandle consume handle = loop SHA.init+  where+    loop !context = do+      chunk <- BS.hGetSome handle (64 * 1024)+      if BS.null chunk+        then pure $! hexDigest (SHA.finalize context)+        else consume chunk >> loop (SHA.update context chunk)++-- | Reject an observation if the open file changed while it was being hashed.+observeFile :: AbsFile -> IO FileObservation+observeFile path = withOpenFile path $ \readDigest readStamp -> do+  before <- readStamp+  digest <- readDigest+  after <- readStamp+  if before == after+    then pure (FileObservation after digest)+    else ioError (userError ("File changed while hashing " ++ toFilePath path))++statFile :: AbsFile -> IO FileStamp+statFile path = withOpenFile path $ \_ readStamp -> readStamp++-- Both actions operate on one open file even if its path is replaced.+withOpenFile :: AbsFile -> (IO Digest -> IO FileStamp -> IO a) -> IO a+#ifdef mingw32_HOST_OS+withOpenFile path action =+  withBinaryFile (toFilePath path) ReadMode $ \handle -> action (digestHandle (const (pure ())) handle) $+    withHandleToHANDLE handle $ \native -> do+      info <- Win.getFileInformationByHandle native+      let FILETIME ticks = Win.bhfiLastWriteTime info+      pure FileStamp+        { stampIdentity = (toInteger (Win.bhfiVolumeSerialNumber info), toInteger (Win.bhfiFileIndex info)),+          stampSize = toInteger (Win.bhfiSize info),+          stampModified = fromIntegral ticks / 10000000,+          stampChanged = Nothing+        }+#else+withOpenFile path action = bracket acquire (hClose . snd) $ \opened@(fd, _) ->+  action (readDigest opened) $ do+    info <- Posix.getFdStatus fd+    pure FileStamp+      { stampIdentity = (toInteger (Posix.deviceID info), toInteger (Posix.fileID info)),+        stampSize = toInteger (Posix.fileSize info),+        stampModified = toRational (Posix.modificationTimeHiRes info),+        stampChanged = Just (toRational (Posix.statusChangeTimeHiRes info))+      }+  where+#ifdef darwin_HOST_OS+    readDigest (fd, _) = allocaBytes 32 $ \out -> do+      throwErrnoPathIfMinus1_ "hash open file" (toFilePath path) (sha256Fd fd out)+      hexDigest <$> BS.packCStringLen (castPtr out, 32)+#else+    readDigest (_, handle) = digestHandle (const (pure ())) handle+#endif+    -- fdToHandle transfers ownership. Keep the borrowed fd for fstat until hClose.+    acquire = bracketOnError+      (Posix.openFd (toFilePath path) Posix.ReadOnly Posix.defaultFileFlags {Posix.cloexec = True})+      Posix.closeFd+      (\fd -> do handle <- Posix.fdToHandle fd+                 pure (fd, handle))+#endif
src/Hledger/Flow/Import/IncludeFiles.hs view
@@ -154,12 +154,18 @@   let priceFiles = map ((((FilePath.takeDirectory file </> "..") </> "prices") </> dirprefix) </>) prices   let extraFiles = suffixFiles ++ suffixDirFiles ++ priceFiles   filtered <- filterPaths doesFileExist extraFiles-  let relative = relativeToBase' (Path.toFilePath base)+  let relative = T.pack . withoutDotDot . relativeToBase' (Path.toFilePath base)+      list = T.intercalate ", " . map relative+      missing = filter (`notElem` filtered) extraFiles       logMsg =-        "Looking for possible extra include files for '" <> T.pack (relative file)-          <> "' among these " <> T.pack (show (length extraFiles)) <> " options: "-          <> T.pack (show (map (T.pack . relative) extraFiles))-          <> ". Found " <> T.pack (show (length filtered)) <> ": "-          <> T.pack (show (map (T.pack . relative) filtered))+        "Extra files for " <> relative file <> ": "+          <> T.intercalate "; " (["found " <> list filtered | not (null filtered)] ++ ["not found: " <> list missing | not (null missing)])   logMessage logMsg   return $ Map.fromList [(file, filtered)]++-- | @import/john/../prices@ → @import/prices@, for display.+withoutDotDot :: FilePath -> FilePath+withoutDotDot = FilePath.joinPath . reverse . foldl step [] . FilePath.splitDirectories+  where+    step (dir : dirs) ".." | dir /= ".." = dirs+    step dirs part = part : dirs
+ src/Hledger/Flow/Import/Incremental.hs view
@@ -0,0 +1,237 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE TupleSections #-}++module Hledger.Flow.Import.Incremental+  ( IncrementalRun, withIncrementalRun, importStatement ) where++import Control.Concurrent.STM (TChan)+import Control.Exception (finally)+import Control.Monad (unless, when)+import Data.IORef+import qualified Data.Map.Strict as Map+import Data.Maybe (maybeToList)+import qualified Data.Set as Set+import qualified Data.Text as T+import Hledger.Flow.BaseDir (relativeToBase)+import Hledger.Flow.Common (directivesFile, writeStagedText)+import Hledger.Flow.DocHelpers+import Hledger.Flow.Cache (initializeCache)+import Hledger.Flow.Import.Cache+import Hledger.Flow.Import.Conversion+import Hledger.Flow.Import.Dependencies+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.Inventory (generatedOutput)+import Hledger.Flow.Import.Snapshot+import Hledger.Flow.Import.Types+import Hledger.Flow.Logging+import Hledger.Flow.PathHelpers (AbsFile, withStagedOutput)+import Hledger.Flow.RuntimeOptions+import Hledger.Flow.Types (LogMessage)+import qualified Path as P+import qualified Path.IO as P+import qualified System.Directory as D+import System.IO.Error (tryIOError)+import qualified System.Info as Sys++-- Keep one path only when every occurrence of a reason names that file.+data Notices = Notices !(Set.Set AbsFile) !(Map.Map T.Text (Maybe T.Text))+data IncrementalRun = IncrementalRun SnapshotRun (IO (ReuseDecision Converter)) (IORef Notices)++withIncrementalRun :: RuntimeOptions -> TChan LogMessage -> Int -> (IncrementalRun -> IO a) -> IO a+withIncrementalRun opts ch count action = withSnapshotRun (baseDir opts) $ \snapshots -> do+  reasons <- newIORef (Notices Set.empty Map.empty)+  probe <- newConverterProbe opts+  when (count > 0) $ do+    initialized <- tryIOError $ initializeCache (baseDir opts)+    either (channelErrLn ch . ("Could not initialize incremental cache: " <>) . T.pack . show) pure initialized+  finally (action (IncrementalRun snapshots probe reasons)) $ do+    Notices affected warnings <- readIORef reasons+    unless (Map.null warnings) $ do+      let scope | Set.size affected == count = "all selected statements"+                | otherwise = T.pack (show (Set.size affected)) <> " of " <> T.pack (show count) <> " selected statements"+          describe (why,path) = why <> maybe "" (": " <>) path+      channelErrLn ch $ "Incremental reuse unavailable for " <> scope <> ": "+        <> T.intercalate ", " (map describe (Map.toAscList warnings)) <> ". See " <> docURL IncrementalImportFallback++data HookInputs = HookInputs AbsFile FileObservation AbsFile FileObservation+  deriving (Eq, Show)++observeHook :: AbsFile -> AbsFile -> IO (ReuseDecision HookInputs)+observeHook script source = checkedIO "could not check script inputs" (P.toFilePath script) $ do+  runnable <- if Sys.os == "mingw32" then pure True else D.executable <$> D.getPermissions (P.toFilePath script)+  if runnable+    then Reusable <$> (HookInputs script <$> observeFile script <*> pure source <*> observeFile source)+    else pure $ Fallback (OrdinaryInputError "script is not executable" (pathText script))++hookInputsKey :: ImportDirs P.Abs -> FilePath -> T.Text -> HookInputs -> StageInputs+hookInputsKey dirs cwd stage (HookInputs script s input i) =+  [("stage",stage),("working directory",T.pack cwd),("hook",pathText script),+   ("hook digest",digestHex (observationDigest s)),("input",pathText input),+   ("input digest",digestHex (observationDigest i))] ++ map ("argument",) (hookDirArgs dirs)++pathText :: AbsFile -> T.Text+pathText = T.pack . P.toFilePath++data JournalInputs = JournalInputs+  { engine :: Converter+  , copiedInputs :: SnapshotInput+  , recordedInputs :: StageInputs+  , copiedDigests :: Map.Map AbsFile Digest+  , liveConstruct :: Maybe HookInputs+  } deriving (Eq, Show)++importStatement :: IncrementalRun -> RuntimeOptions -> TChan LogMessage -> ImportDirs P.Abs -> AbsFile -> IO (AbsFile,Bool,Bool)+importStatement (IncrementalRun snapshots probe reasons) opts ch dirs source = do+  cwd <- D.getCurrentDirectory+  record <- recordPath (baseDir opts) source+  old <- readRecord record+  intermediate <- generatedOutput [P.reldir|2-preprocessed|] ".csv" source+  let hook name = findHook opts ch name (accountDir dirs P.</> (if name == "preprocess" then [P.relfile|preprocess|] else [P.relfile|construct|]))+      report OrdinaryInputError{} = pure ()+      report problem = do+        let reason = reasonScope problem+            path = case problem of+              FileReason _ file _ -> Just (T.pack (relativeToBase opts file))+              _ -> Nothing+            detail = reasonDetail problem+            common a b = if a == b then a else Nothing+        atomicModifyIORef' reasons $ \(Notices files summaries) ->+          (Notices (Set.insert source files) (Map.insertWith common reason path summaries), ())+        logVerbose opts ch $ T.intercalate " " (filter (not . T.null) ([reason] ++ maybeToList path ++ [detail]))+      reuseStage stage inputs target prior = do+        miss <- stageMiss inputs target prior+        case miss of+          Nothing -> pure True+          Just why -> logVerbose opts ch (stage <> ": " <> why) >> pure False+      changed = FileReason "dependency changed during conversion" (P.toFilePath source) ""+      skipScripts = importPolicy opts == SkipUnchangedScripts+  pre <- hook "preprocess"+  (csv,preRecord,preRan) <- case pre of+    Nothing -> do+      input <- savedInput opts ch source intermediate+      pure (input,Nothing,False)+    Just script -> do+      before <- observeHook script source+      let key = hookInputsKey dirs cwd "preprocess" <$> before+          prior = old >>= preprocessRecord+      hit <- case key of+        Reusable k | skipScripts -> reuseStage "Preprocess" k intermediate prior+        _ -> do+          unless skipScripts $ logVerbose opts ch "Running preprocess script in run-scripts mode."+          pure False+      if hit then do+        logVerbose opts ch "Skipped unchanged preprocess script."+        pure (intermediate,prior,False)+      else do+        previous <- digestIfPresent intermediate+        digest <- installOutput True intermediate (preprocess opts ch script dirs source intermediate)+        when (Just digest == previous) $ logVerbose opts ch "Preprocessing ran; the CSV is unchanged."+        after <- observeHook script source+        selected <- hook "preprocess"+        usable <- case key of+          Fallback reason -> when skipScripts (report reason) >> pure Nothing+          Reusable k | selected == pre && after == before -> pure (Just (StageRecord k digest))+                     | otherwise -> when skipScripts (report changed) >> pure Nothing+        pure (intermediate,usable,True)+  output <- generatedOutput [P.reldir|3-journal|] ".journal" csv+  let discover = do+        candidate <- hook "construct"+        context <- probe+        case context of+          Fallback r -> pure (Fallback r)+          Reusable converter -> checkedIO "could not verify journal inputs" (P.toFilePath csv) $ do+            config <- discoverConfig (converterRelease converter) (hledgerConf opts)+            case config of+              Fallback r -> pure (Fallback r)+              Reusable conf -> do+                let common = [("source",pathText source),("effective CSV",pathText csv),("preprocess",maybe "" pathText pre)]+                      ++ map ("converter",) (converterKey converter)+                    configArgs = pinnedConfig ("<config>" <$ conf)+                    mk inputs files args extra construct =+                      JournalInputs converter inputs (common ++ map ("hledger argument",) args ++ extra ++ [(pathText p,digestHex d) | (p,d) <- files]) (Map.fromList files) construct+                case candidate of+                  Just script -> do+                    observation <- observeHook script csv+                    pure $ (\h -> mk (ConstructInput (fst <$> conf)) (maybeToList conf)+                                        (constructArguments configArgs "<journal>") (hookInputsKey dirs cwd "construct" h) (Just h)) <$> observation+                  Nothing | not (isSeparatedValues csv) -> pure $ atPath "unsupported input reader" (P.toFilePath csv) ""+                  Nothing -> do+                    rules <- selectRules csv dirs+                    case rules of+                      Fallback r -> pure (Fallback r)+                      Reusable Nothing -> pure $ Fallback (OrdinaryInputError "missing rules file" (pathText csv))+                      Reusable (Just rf) -> do+                        tree <- discoverRules (baseDir opts) (converterRelease converter) rf+                        directives <- discoverDirectives (directivesFile opts)+                        digest <- digestFile csv+                        pure $ do+                          rulesTree <- tree+                          ds <- directives+                          let files = (csv,digest) : rulesTree ++ maybeToList ds ++ maybeToList conf+                          pure $ mk (RulesInput csv rf (fst <$> ds) (fst <$> conf)) files+                                   (rulesArguments configArgs "<input>" "<rules>" ("<directives>" <$ ds)) [("rules",pathText rf)] Nothing+      ordinary = do+        script <- hook "construct"+        case script of+          Nothing -> hledgerImport opts ch dirs csv output+          Just s -> withStagedOutput output (customConstruct opts ch s dirs csv output)+      prior = old >>= journalRecord+      fallback reason = do+        report reason+        case reason of+          CyclicRules _ file -> ioError $ userError ("Cyclic CSV rules include at " ++ T.unpack file)+          _ -> ordinary >> pure (Nothing,True)+  rawDecision <- discover+  let agrees inputs = case preRecord of+        Nothing -> True+        Just entry -> Just (stageOutput entry) == case liveConstruct inputs of+          Nothing -> Map.lookup csv (copiedDigests inputs)+          Just (HookInputs _ _ _ observed) -> Just (observationDigest observed)+      decision = rawDecision >>= \inputs -> if agrees inputs then pure inputs else Fallback changed+  (journal,generated) <- case decision of+    Fallback r -> fallback r+    Reusable inputs -> do+      hit <- if skipScripts || liveConstruct inputs == Nothing+               then reuseStage "Journal" (recordedInputs inputs) output prior+               else logVerbose opts ch "Running construct script in run-scripts mode." >> pure False+      if hit then do+        when (liveConstruct inputs /= Nothing) $ logVerbose opts ch "Skipped unchanged construct script."+        logVerbose opts ch "Reused the verified journal."+        pure (prior,False)+      else do+        result <- withSnapshot snapshots (converterRelease (engine inputs)) (copiedInputs inputs) $ \snapshot -> do+          let pin conf = pinnedConfig (pathText <$> conf)+              translate = translateDiagnostic snapshot+              executable = converterExecutable (engine inputs)+          outDigest <- case (copiedInputs inputs,snapshotInputs snapshot,liveConstruct inputs) of+            (RulesInput src rf ds _,RulesInput src' rf' ds' conf,_) -> do+              contents <- convertRules opts ch executable (RulesPaths src rf ds) (RulesPaths src' rf' ds') (pin conf) translate+              target <- D.canonicalizePath (P.toFilePath output) >>= P.parseAbsFile+              installOutput False target (writeStagedText target contents)+            (ConstructInput _,ConstructInput conf,Just (HookInputs s _ _ _)) ->+              installOutput False output (constructWith opts ch executable (pin conf) translate s dirs csv output)+            _ -> ioError (userError "Inconsistent incremental conversion inputs")+          again <- discover+          preNow <- hook "preprocess"+          effective <- case preNow of+            Nothing -> P.doesFileExist intermediate >>= \exists -> pure (if exists then intermediate else source)+            Just _ -> pure intermediate+          let stable = again == Reusable inputs && snapshotManifest snapshot == copiedDigests inputs && preNow == pre && effective == csv+          if stable then pure $ Just (StageRecord (recordedInputs inputs) outDigest)+                    else when (skipScripts || liveConstruct inputs == Nothing) (report changed) >> pure Nothing+        case result of+          Fallback r -> fallback r+          Reusable entry -> pure (entry,True)+  -- A concurrent writer can replace the intermediate after the hook returned.+  -- A later run must verify its bytes before trusting the preprocessing entry.+  currentPre <- case preRecord of+    Just entry -> do+      actual <- digestIfPresent intermediate+      pure $ if actual == Just (stageOutput entry) then Just entry else Nothing+    Nothing -> pure Nothing+  let next = StatementRecord currentPre journal+  when (Just next /= old && (old /= Nothing || next /= StatementRecord Nothing Nothing)) $ do+    saved <- tryIOError $ saveRecord record next+    either (channelErrLn ch . ("Could not save incremental import record: " <>) . T.pack . show) pure saved+  pure (output,generated,preRan || generated)
+ src/Hledger/Flow/Import/PrivateDirectory.hs view
@@ -0,0 +1,53 @@+{-# LANGUAGE CPP #-}++module Hledger.Flow.Import.PrivateDirectory (createPrivateDirectory) where++import Control.Monad (unless)+import Hledger.Flow.PathHelpers (AbsDir)+import Path (toFilePath)+import System.FilePath (dropTrailingPathSeparator)+#ifdef mingw32_HOST_OS+import Foreign.C.String (CWString, withCWString)+import System.Win32.Types (DWORD, failWith)++foreign import ccall safe "hf_create_private_directory"+  createDirectoryWithDACL :: CWString -> IO DWORD+#else+import Control.Exception (bracket, bracketOnError)+import Data.Bits ((.&.))+import qualified System.Posix.Directory as Posix+import qualified System.Posix.Files as Posix+import qualified System.Posix.IO as Posix+import qualified System.Posix.User as Posix+#ifdef darwin_HOST_OS+import Foreign.C.Error (throwErrnoPathIfMinus1_)+import Foreign.C.Types (CInt (..))+import System.Posix.Types (Fd (..))++foreign import ccall unsafe "hf_verify_private_acl"+  verifyPrivateACL :: Fd -> IO CInt+#endif+#endif++-- | Create a fresh directory and verify privacy before any copies are made.+createPrivateDirectory :: AbsDir -> IO ()+#ifdef mingw32_HOST_OS+createPrivateDirectory path =+  withCWString (dropTrailingPathSeparator (toFilePath path)) $ \name -> do+    err <- createDirectoryWithDACL name+    unless (err == 0) (failWith ("Create private directory " ++ toFilePath path) err)+#else+createPrivateDirectory path =+  bracketOnError (Posix.createDirectory name 0o700) (const (Posix.removeDirectory name)) $ \() ->+    bracket (Posix.openFd name Posix.ReadOnly flags) Posix.closeFd $ \fd -> do+      status <- Posix.getFdStatus fd+      owner <- Posix.getEffectiveUserID+      unless (Posix.fileOwner status == owner && Posix.fileMode status .&. 0o777 == 0o700) $+        ioError (userError ("Cannot verify owner-only permissions on " ++ name))+#ifdef darwin_HOST_OS+      throwErrnoPathIfMinus1_ "Verify private directory ACL" name (verifyPrivateACL fd)+#endif+  where+    name = dropTrailingPathSeparator (toFilePath path)+    flags = Posix.defaultFileFlags {Posix.nofollow = True, Posix.directory = True, Posix.cloexec = True}+#endif
+ src/Hledger/Flow/Import/Snapshot.hs view
@@ -0,0 +1,329 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}++module Hledger.Flow.Import.Snapshot+  ( SnapshotRun, SnapshotInput (..), PreparedSnapshot, snapshotInputs, snapshotManifest,+    withSnapshotRun, withSnapshotRunUsing, withSnapshot,+    translateDiagnostic, windowsSnapshotPathFits+  ) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.MVar+import Control.Exception hiding (handle)+import Control.Monad (filterM, forM_, unless, void, when)+import qualified Data.ByteString as BS+import Data.Char (isAlphaNum, isAsciiLower, isDigit)+import Data.IORef+import qualified Data.Map.Strict as Map+import qualified Data.Set as Set+import Data.Text (Text)+import qualified Data.Text as T+import Data.Word (Word32)+import GHC.IO.Handle.Lock (LockMode (ExclusiveLock), hLock, hTryLock)+import Hledger.Flow.Import.Dependencies+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.PrivateDirectory+import Hledger.Flow.PathHelpers (AbsDir, AbsFile)+import Numeric (showHex)+import Path+import qualified Path.IO as P+import qualified System.Directory as D+import qualified System.FilePath as FP+import System.IO+#ifndef mingw32_HOST_OS+import qualified System.Posix.IO as Posix+#endif+import Text.Read (readMaybe)+import System.IO.Error++data SnapshotInput+  = RulesInput AbsFile AbsFile (Maybe AbsFile) (Maybe AbsFile) -- data, rules, directives, config+  | ConstructInput (Maybe AbsFile)+  deriving (Eq, Show)++data PreparedSnapshot = PreparedSnapshot+  { snapshotInputs :: SnapshotInput,+    snapshotManifest :: Map.Map AbsFile Digest,+    originalPaths :: Map.Map AbsFile AbsFile+  }++data SnapshotRun = SnapshotRun+  { ledger :: AbsDir,+    registration :: MVar (Maybe (AbsDir, Handle)),+    nextTree :: IORef Word32,+    makePrivate :: AbsDir -> IO ()+  }++withSnapshotRun :: AbsDir -> (SnapshotRun -> IO a) -> IO a+withSnapshotRun = withSnapshotRunUsing createPrivateDirectory++withSnapshotRunUsing :: (AbsDir -> IO ()) -> AbsDir -> (SnapshotRun -> IO a) -> IO a+withSnapshotRunUsing create base use = do+  run <- SnapshotRun base <$> newMVar Nothing <*> newIORef 0 <*> pure create+  let area = snapshotArea run+  -- Failure to claim an abandoned tree leaves it for a later invocation.+  void $ tryIOError $ do+    exists <- P.doesDirExist area+    when exists $ withRegistration area $ reap area+  use run `finally` (readMVar (registration run) >>= mapM_ (releaseRun area))++snapshotArea :: SnapshotRun -> AbsDir+snapshotArea run = ledger run </> [reldir|.hledger-flow/cache/snapshots|]++-- GHC prevents two writable opens of the same file within one process. Wait+-- for that case too (concurrent imports in one process). The OS lock also+-- serializes independent processes, including while an initializer is paused.+withRegistration :: AbsDir -> IO a -> IO a+withRegistration area action = bracket open hClose $ \handle -> hLock handle ExclusiveLock >> action+  where+    file = toFilePath (area </> [relfile|registration.lock|])+    open = catchIOError (openLockFile file) $ \e ->+      if isAlreadyInUseError e then threadDelay 1000 >> open else ioError e++-- A surviving hook must not retain its killed parent's lifetime lock.+openLockFile :: FilePath -> IO Handle+#ifdef mingw32_HOST_OS+openLockFile file = openBinaryFile file ReadWriteMode+#else+openLockFile file = bracketOnError (Posix.openFd file Posix.ReadWrite flags) Posix.closeFd Posix.fdToHandle+  where flags = Posix.defaultFileFlags {Posix.creat = Just 0o666, Posix.cloexec = True}+#endif++shortName :: Char -> Word32 -> String+shortName prefix n = prefix : replicate (8 - length digits) '0' ++ digits+  where digits = showHex n ""++lockFile :: AbsDir -> FilePath+lockFile dir = toFilePath (dir </> [relfile|run.lock|])++acquireTree :: SnapshotRun -> IO AbsDir+acquireTree run = do+  dir <- modifyMVarMasked (registration run) $ \current -> case current of+    Just registered -> pure (current, fst registered)+    Nothing -> do+      registered <- catchIOError register $ \e -> reject "private staging unavailable" (T.pack (displayException e))+      pure (Just registered, fst registered)+  n <- atomicModifyIORef' (nextTree run) $ \i -> (i + 1, i)+  tree <- (dir </>) <$> parseRelDir (shortName 't' n)+  P.createDir tree+  pure tree+  where+    area = snapshotArea run+    register = do+      P.createDirIfMissing True area+      withRegistration area (createRun 0)+    createRun n = do+      dir <- (area </>) <$> parseRelDir (shortName 'r' n)+      result <- tryIOError (makePrivate run dir)+      case result of+        Left e | isAlreadyExistsError e && n < maxBound -> createRun (n + 1)+        Left e -> ioError e+        Right () -> flip onException (P.removeDirRecur dir) $+          bracketOnError (openLockFile (lockFile dir)) hClose $ \handle -> do+            hLock handle ExclusiveLock+            pure (dir, handle)++releaseRun :: AbsDir -> (AbsDir, Handle) -> IO ()+releaseRun area (dir, handle) = withRegistration area $+  (clearRun dir `finally` hClose handle) >> removeEmptyRun dir++-- Keep the ownership marker until every byte of statement data is gone. A+-- process killed during reclamation leaves a lockable tree for the next run.+clearRun :: AbsDir -> IO ()+clearRun dir = do+  entries <- D.listDirectory (toFilePath dir)+  forM_ (filter (/= "run.lock") entries) $ \entry ->+    D.removePathForcibly (toFilePath dir FP.</> entry)++removeEmptyRun :: AbsDir -> IO ()+removeEmptyRun dir = D.removeFile (lockFile dir) >> P.removeDir dir++reap :: AbsDir -> IO ()+reap area = do+  names <- D.listDirectory (toFilePath area)+  forM_ (filter isRunName names) $ \name -> void $ tryIOError $ do+    dir <- (area </>) <$> parseRelDir name+    let path = FP.dropTrailingPathSeparator (toFilePath dir)+    linked <- D.pathIsSymbolicLink path+    unless linked $ do+      let marker = lockFile dir+      exists <- D.doesFileExist marker+      when exists $ do+        link <- D.pathIsSymbolicLink marker+        unless link $ do+          claimed <- bracket (openLockFile marker) hClose $ \handle -> do+            mine <- hTryLock handle ExclusiveLock+            when mine (clearRun dir)+            pure mine+          when claimed (removeEmptyRun dir)+  where+    isRunName ('r':digits) = length digits == 8 && all hex digits+    isRunName _ = False+    hex c = c >= '0' && c <= '9' || isAsciiLower c && c <= 'f'++newtype PreparationFailure = PreparationFailure FallbackReason deriving Show+instance Exception PreparationFailure++reject :: Text -> Text -> IO a+reject reason detail = throwIO (PreparationFailure (FallbackReason reason detail))++fromDecision :: ReuseDecision a -> IO a+fromDecision (Reusable x) = pure x+fromDecision (Fallback reason) = throwIO (PreparationFailure reason)++prepare :: IO a -> IO (ReuseDecision a)+prepare io = (Reusable <$> io) `catches`+  [ Handler $ \(PreparationFailure reason) -> pure (Fallback reason),+    Handler $ \(e :: IOException) -> pure (Fallback (FallbackReason "snapshot creation failed" (T.pack (displayException e))))+  ]++withSnapshot :: SnapshotRun -> Release -> SnapshotInput -> (PreparedSnapshot -> IO a) -> IO (ReuseDecision a)+withSnapshot run version input convert = mask $ \restore -> do+  allocation <- prepare (acquireTree run)+  case allocation of+    Fallback reason -> pure (Fallback reason)+    Reusable tree -> flip finally (P.removeDirRecur tree) $ restore $ do+      result <- prepare (copyInputs (ledger run) tree version input)+      case result of+        Fallback reason -> pure (Fallback reason)+        Reusable snapshot -> Reusable <$> convert snapshot++windowsSnapshotPathFits :: FilePath -> Bool+windowsSnapshotPathFits = (< 260) . sum . map units+  where units c = if fromEnum c > 0xffff then 2 :: Int else 1++copyInputs :: AbsDir -> AbsDir -> Release -> SnapshotInput -> IO PreparedSnapshot+copyInputs base tree version input = do+  manifest <- newIORef Map.empty+  paths <- newIORef Map.empty+  completed <- newIORef Set.empty+  let mirror file = case stripProperPrefix base file of+        Nothing -> reject "snapshot input outside ledger" (T.pack (toFilePath file))+        Just rel -> pure (tree </> [reldir|l|] </> rel)+      destination source target = do+#ifdef mingw32_HOST_OS+        unless (windowsSnapshotPathFits (toFilePath target)) $+          reject "overlong Windows snapshot path" (T.pack (toFilePath target))+#endif+        P.createDirIfMissing True (parent target)+        -- Source and cache directories can have different case or Unicode+        -- equivalence rules. Never overwrite a copy made for another spelling.+        previous <- Map.lookup target <$> readIORef paths+        collision <- case previous of+          Just original -> pure (toFilePath original /= toFilePath source)+          Nothing -> D.doesPathExist (toFilePath target)+        when collision $ reject "snapshot filenames collide" (T.pack (toFilePath source))+      record source target digest = do+        previous <- Map.lookup source <$> readIORef manifest+        when (maybe False (/= digest) previous) $+          reject "dependency changed during snapshot" (T.pack (toFilePath source))+        modifyIORef' manifest (Map.insert source digest)+        modifyIORef' paths (Map.insert target source)+      copyBytes source target = do+        destination source target+        bytes <- BS.readFile (toFilePath source)+        BS.writeFile (toFilePath target) bytes+        record source target (digestBytes bytes)+        pure bytes+      validated check source target = do+        bytes <- copyBytes source target+        () <- fromDecision (check version source bytes)+        pure target+      copyConfig source = validated checkConfig source (tree </> [reldir|c|] </> filename source)+      copyRule ancestors source = do+        target <- mirror source+        identity <- D.canonicalizePath (toFilePath source)+        when (Set.member identity ancestors || Set.size ancestors >= 256) $ reject "cyclic or excessively deep rules include" (T.pack (toFilePath source))+        done <- Set.member (toFilePath source) <$> readIORef completed+        unless done $ do+          bytes <- copyBytes source target+          let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes+          includes <- fromDecision (scan version source bytes)+          forM_ includes $ \spelling -> do+            child <- resolveRuleInclude base source spelling >>= fromDecision+            void $ copyRule (Set.insert identity ancestors) child+          modifyIORef' completed (Set.insert (toFilePath source))+        pure target+  copied <- case input of+    ConstructInput conf -> ConstructInput <$> traverse copyConfig conf+    RulesInput csv rules directives conf -> do+      dataCopy <- mirror csv+      destination csv dataCopy+      copyDigest csv dataCopy >>= record csv dataCopy+      rulesCopy <- copyRule Set.empty rules+      directivesCopy <- traverse (\source -> mirror source >>= validated checkDirectives source) directives+      RulesInput dataCopy rulesCopy directivesCopy <$> traverse copyConfig conf+  PreparedSnapshot copied <$> readIORef manifest <*> readIORef paths++-- hledger emits locations as "FILE:line:column:" (also "FILE:line:" and+-- "FILE: explanation"). Restrict replacement to its location fields.+-- Source excerpts following a Megaparsec gutter are user data.+translateDiagnostic :: PreparedSnapshot -> Text -> IO Text+translateDiagnostic snap diagnostic = T.intercalate "\n" <$> mapM translate (T.splitOn "\n" diagnostic)+  where+    translate line = do+      let (prefix, body) = splitHeader line+          candidates value = [ (T.take i value, T.drop i value)+                       | (i, ':') <- zip [0..] (T.unpack value)+                       , let next = T.take 1 (T.drop (i+1) value)+                       , T.null next || next == " " || T.all isDigit next ]+      if FP.isAbsolute (T.unpack body)+        then firstKnown prefix line (candidates body)+        else case T.stripPrefix "\"" body of+          Just quoted -> quotedPath prefix line (candidates quoted)+          Nothing -> translateRulePosition line+    -- Some config errors quote the entire location with Haskell string escaping.+    quotedPath _ line [] = pure line+    quotedPath prefix line ((token,suffix):rest) =+      case readMaybe (T.unpack ("\"" <> token <> "\"")) of+        Just name | FP.isAbsolute name -> do+          mapped <- lookupPath (T.pack name)+          case mapped of+            Just p -> pure (prefix <> T.dropEnd 1 (T.pack (show (toFilePath p))) <> suffix)+            Nothing -> quotedPath prefix line rest+        _ -> quotedPath prefix line rest+    -- 1.99.6 appends a second location to a field-assignment diagnostic.+    -- Keep arbitrary template/record text ahead of the final location intact.+    translateRulePosition line+      | any (`T.isPrefixOf` line) ["the ", "rule ", "    (overrides: "] || fieldLabel =+          let (before, body) = T.breakOnEnd "  (" line+              (fileColon, lineEnd) = T.breakOnEnd ":" body+              (digits, close) = T.span isDigit lineEnd+          in if T.null before || T.null fileColon || T.null digits || close `notElem` [")", "))"]+             then pure line+             else firstKnown before line [(T.dropEnd 1 fileColon, ":" <> lineEnd)]+      | otherwise = pure line+      where+        fieldLabel = case T.stripPrefix "  " line of+          Just field -> let (name, rest) = T.span isAlphaNum field+                        in not (T.null name) && ":" `T.isPrefixOf` rest+          Nothing -> False+    splitHeader line =+      let (spaces, rest) = T.span (== ' ') line+          (program, message) = splitPrefix ["hledger: ", "hledger.exe: "] rest+          (label, body) = splitPrefix ["Warning: ", "Error: ", "error: ", "in "] message+      in (spaces <> program <> label, body)+    splitPrefix prefixes value = case [(p,s) | p <- prefixes, Just s <- [T.stripPrefix p value]] of+      (parts:_) -> parts+      [] -> ("",value)+    firstKnown _ original [] = pure original+    firstKnown prefix original ((token, suffix):more) = do+      mapped <- lookupPath token+      case mapped of+        Just path -> pure (prefix <> T.pack (toFilePath path) <> suffix)+        Nothing -> firstKnown prefix original more+    lookupPath token = case parseAbsFile (T.unpack token) >>= (\p -> Map.lookup p (originalPaths snap)) of+      Just path -> pure (Just path)+      Nothing -> catchIOError (byIdentity (T.unpack token)) (const (pure Nothing))+    byIdentity name = do+      actual <- D.canonicalizePath name+      -- canonicalizePath resolves symlinked ancestors and native spellings.+      -- File IDs also identify a Windows short path when its spelling differs.+      source <- parseAbsFile actual+      targetId <- stampIdentity <$> statFile source+      matches <- filterM (\p -> (== targetId) . stampIdentity <$> statFile p) (Map.keys (originalPaths snap))+      pure $ case matches of+        [p] -> Map.lookup p (originalPaths snap)+        _ -> Nothing
src/Hledger/Flow/Logging.hs view
@@ -8,14 +8,34 @@ import Control.Monad (when) import qualified Data.Text as T import qualified Data.Text.IO as T-import Data.Time.LocalTime (getZonedTime) import qualified GHC.IO.Handle.FD as H-import System.IO (hFlush)+import System.Console.ANSI (Color (..), ColorIntensity (..), ConsoleIntensity (..), ConsoleLayer (..), SGR (..), hSupportsANSIColor, setSGRCode)+import System.Environment (lookupEnv)+import System.IO (Handle, hFlush) import Hledger.Flow.Types -dummyLogger :: TChan LogMessage -> T.Text -> IO ()-dummyLogger _ _ = return ()+-- | Colour only on a terminal, and only when NO_COLOR is unset or empty.+useColor :: Handle -> IO Bool+useColor h = do+  noColor <- lookupEnv "NO_COLOR"+  supported <- hSupportsANSIColor h+  return (supported && maybe True null noColor) +paint :: Bool -> [SGR] -> T.Text -> T.Text+paint False _ txt = txt+paint True sgr txt = T.pack (setSGRCode sgr) <> txt <> resetCode++resetCode :: T.Text+resetCode = T.pack (setSGRCode [Reset])++green, red, yellow, cyan, bold, dim :: Bool -> T.Text -> T.Text+green c = paint c [SetColor Foreground Vivid Green]+red c = paint c [SetColor Foreground Vivid Red]+yellow c = paint c [SetColor Foreground Vivid Yellow]+cyan c = paint c [SetColor Foreground Vivid Cyan]+bold c = paint c [SetConsoleIntensity BoldIntensity]+dim c = paint c [SetConsoleIntensity FaintIntensity]+ channelOut :: TChan LogMessage -> T.Text -> IO () channelOut ch txt = atomically $ writeTChan ch $ StdOut txt @@ -28,30 +48,50 @@ channelErrLn :: TChan LogMessage -> T.Text -> IO () channelErrLn ch txt = channelErr ch (txt <> "\n") +-- | Mark the first line as hledger-flow's own, so it stands apart from what other programs print.+-- Callers indent any further lines.+verboseLine :: T.Text -> IO T.Text+verboseLine msg = do+  color <- useColor H.stderr+  return (dim color "hledger-flow:" <> " " <> msg <> "\n")+ logToChannel :: TChan LogMessage -> T.Text -> IO ()-logToChannel ch msg = do-  ts <- timestampPrefix msg-  channelErrLn ch ts+logToChannel ch msg = verboseLine msg >>= channelErr ch -timestampPrefix :: T.Text -> IO T.Text-timestampPrefix txt = do-  t <- getZonedTime-  return $ T.pack (show t) <> "\thledger-flow " <> txt+-- | In verbose mode, show one item's lines together under a heading. With 'streaming' they+-- appear as they're logged; otherwise they're held back until the item is done, so items+-- that run in parallel don't interleave.+withLogBlock :: (HasVerbosity o) => o -> Bool -> TChan LogMessage -> T.Text -> (TChan LogMessage -> IO a) -> IO a+withLogBlock opts streaming ch heading action+  | not (verbose opts) = action ch+  | otherwise = do+      color <- useColor H.stderr+      line <- verboseLine (bold color heading)+      if streaming+        then channelErr ch ("\n" <> line) >> action ch+        else do+          buffer <- newTChanIO+          result <- tryJust synchronous (action buffer)+          atomically $ writeTChan ch (StdErr ("\n" <> line)) >> drain buffer+          either throwIO return result+  where+    drain buffer = tryReadTChan buffer >>= maybe (return ()) (\msg -> writeTChan ch msg >> drain buffer) +-- | Errors from the action itself, not cancellation from outside.+synchronous :: SomeException -> Maybe SomeException+synchronous err = case fromException err :: Maybe SomeAsyncException of+  Just _ -> Nothing+  Nothing -> Just err+ withConsoleLogging :: (TChan LogMessage -> IO a) -> IO a withConsoleLogging action = do   ch <- newTChanIO   withAsync (consoleChannelLoop ch) $ \logger -> do+    -- Drain queued diagnostics on failure, but allow Ctrl-C to cancel a blocked writer.     result <- tryJust synchronous (action ch)     terminateChannelLoop ch     wait logger     either throwIO return result-  where-    -- Drain queued diagnostics on failure, but allow Ctrl-C to cancel a blocked writer.-    synchronous :: SomeException -> Maybe SomeException-    synchronous err = case fromException err :: Maybe SomeAsyncException of-      Just _ -> Nothing-      Nothing -> Just err  consoleChannelLoop :: TChan LogMessage -> IO () consoleChannelLoop ch = do
src/Hledger/Flow/ReleaseNotice.hs view
@@ -1,14 +1,18 @@+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}  module Hledger.Flow.ReleaseNotice-  ( ReleaseResult (..), withReleaseNotice, startCheck,-    finishCommand, checkRelease, renderUpdate+  ( ReleaseResult (..), withReleaseNotice, withCachedReleaseNotice,+    prepareCheck, refreshIfDue, cachedRelease,+    finishCommand, renderUpdate   ) where -import Control.Concurrent.Async (async, poll)-import Control.Exception (catch, evaluate, throwIO, try)-import Control.Monad (guard, join)-import Data.Aeson (decode, decodeFileStrict, withObject, (.:), (.:?), (.!=))+import Control.Concurrent.Async (async, waitCatch)+import Control.Exception (catch, throwIO, try)+import Control.Monad (guard, join, void, when)+import Data.Aeson (FromJSON, ToJSON, decode, decodeFileStrict, encode, withObject, (.:), (.:?), (.!=)) import Data.Aeson.Types (parseMaybe) import qualified Data.ByteString.Lazy as BL import Data.Char (isDigit)@@ -16,7 +20,13 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T+import Data.Time (NominalDiffTime, UTCTime, diffUTCTime, getCurrentTime) import Data.Version (Version, parseVersion, showVersion)+import GHC.Generics (Generic)+import GHC.IO.Handle.Lock (LockMode (ExclusiveLock), hTryLock)+import Hledger.Flow.Cache (cacheDir, initializeCache)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, withStagedOutput)+import qualified Path as P import Hledger.Flow.Status.Render (useColor, yellow) import Network.HTTP.Client (HttpException, checkResponse, httpLbs, parseRequest, requestHeaders, responseBody, throwErrorStatusCodes) import Network.HTTP.Client.TLS (getGlobalManager)@@ -25,7 +35,7 @@ import System.Environment (getExecutablePath, lookupEnv) import System.Exit (ExitCode) import System.FilePath ((</>), takeDirectory, takeFileName)-import System.IO (hIsTerminalDevice, stderr, stdout)+import System.IO (IOMode (AppendMode), hIsTerminalDevice, stderr, stdout, withBinaryFile) import System.IO.Error (catchIOError) import System.Timeout (timeout) import Text.ParserCombinators.ReadP (eof, readP_to_S)@@ -33,8 +43,63 @@ data ReleaseResult = UpToDate | UpdateAvailable !Version !T.Text | CheckUnavailable   deriving (Eq, Show) -withReleaseNotice :: IO () -> IO ()-withReleaseNotice command = do+-- The cached version is compared with the binary reading it on every run.+data Observation = Observation+  { latestVersion :: !Version+  , checkedAt :: !UTCTime+  , formulaConfirmed :: !Bool+  } deriving (Generic, FromJSON, ToJSON)++data ReleaseCache = ReleaseCache+  { lastSuccess :: !(Maybe Observation)+  , lastFailure :: !(Maybe UTCTime)+  } deriving (Generic, FromJSON, ToJSON)++type Fetch = String -> IO (Maybe BL.ByteString)++cacheFile :: AbsDir -> AbsFile+cacheFile base = cacheDir base P.</> [P.relfile|release-check.json|]++readCache :: AbsDir -> IO ReleaseCache+readCache base = do+  record <- decodeFileStrict (P.toFilePath (cacheFile base)) `catchIOError` const (pure Nothing)+  pure (fromMaybe (ReleaseCache Nothing Nothing) record)++saveCache :: AbsDir -> ReleaseCache -> IO ()+saveCache base record = withStagedOutput (cacheFile base) $ \stage ->+  BL.writeFile (P.toFilePath stage) (encode record)++refreshDue :: UTCTime -> ReleaseCache -> Bool+refreshDue now record =+  not (maybe False (recent (24 * 60 * 60) . checkedAt) (lastSuccess record)+    || maybe False (recent (15 * 60)) (lastFailure record))+  where+    recent :: NominalDiffTime -> UTCTime -> Bool+    recent limit stamp = let age = diffUTCTime now stamp in age >= 0 && age < limit++cachedRelease :: AbsDir -> IO FilePath -> IO ReleaseResult+cachedRelease base executable = readCache base >>= releaseResult executable++releaseResult :: IO FilePath -> ReleaseCache -> IO ReleaseResult+releaseResult executable record = case lastSuccess record of+  Nothing -> pure CheckUnavailable+  Just observation+    | latestVersion observation <= version -> pure UpToDate+    | otherwise -> do+        owned <- if formulaConfirmed observation then brewOwned executable else pure False+        let advice = if owned+              then "Upgrade with: brew update && brew upgrade ferntrail/tap/hledger-flow"+              else "https://github.com/apauley/hledger-flow/releases/latest"+        pure (UpdateAvailable (latestVersion observation) advice)++withReleaseNotice :: AbsDir -> IO () -> IO ()+withReleaseNotice base = withNotice (prepareCheck base getCurrentTime fetch getExecutablePath)++withCachedReleaseNotice :: AbsDir -> IO () -> IO ()+withCachedReleaseNotice base = withNotice (pure (cachedRelease base getExecutablePath))++withNotice :: IO (IO ReleaseResult) -> IO () -> IO ()+withNotice prepare command = do   finish <- setup `catchIOError` const (pure (pure ()))   finishCommand finish command   where@@ -43,21 +108,42 @@       outTTY <- hIsTerminalDevice stdout       errTTY <- hIsTerminalDevice stderr       if disabled /= Just "1" && outTTY && errTTY-        then startCheck (checkRelease fetch getExecutablePath)+        then do+          result <- prepare+          pure (result >>= display)         else pure (pure ())+    display (UpdateAvailable latest advice) = do+      color <- useColor stderr+      T.hPutStrLn stderr ("\n" <> renderUpdate color latest advice)+    display _ = pure () --- The CLI owns the worker. Let process exit handle any pending network cleanup.-startCheck :: IO ReleaseResult -> IO (IO ())-startCheck check = do-  worker <- async (check >>= evaluate)-  pure $ do-    result <- poll worker-    case result of-      Just (Right (UpdateAvailable latest advice)) -> do-        color <- useColor stderr-        T.hPutStrLn stderr ("\n" <> renderUpdate color latest advice)-      _ -> pure ()+prepareCheck :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> IO (IO ReleaseResult)+prepareCheck base clock get executable = do+  record <- readCache base+  now <- clock+  if refreshDue now record then do+    initializeCache base+    worker <- async (refreshIfDue base clock get executable)+    pure $ do+      case lastSuccess record of+        Nothing -> void (timeout 250000 (waitCatch worker))+        Just _ -> pure ()+      cachedRelease base executable+  else pure (releaseResult executable record) +-- Called after initializeCache. Keep the lock file in place so all processes+-- contend on the same file even as the JSON record is atomically replaced.+refreshIfDue :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> IO ()+refreshIfDue base clock get executable = do+  let lock = cacheDir base P.</> [P.relfile|release-check.lock|]+  withBinaryFile (P.toFilePath lock) AppendMode $ \handle -> do+    acquired <- hTryLock handle ExclusiveLock+    when acquired $ do+      -- A different process may have published since our first read.+      current <- readCache base+      started <- clock+      when (refreshDue started current) (refreshCache base clock get executable current)+ renderUpdate :: Bool -> Version -> T.Text -> T.Text renderUpdate color latest advice =   "  " <> yellow color ("↑ " <> availableMessage latest) <> "\n    " <> advice@@ -90,30 +176,30 @@     ignoreHttpError :: HttpException -> IO (Maybe BL.ByteString)     ignoreHttpError _ = pure Nothing -checkRelease :: (String -> IO (Maybe BL.ByteString)) -> IO FilePath -> IO ReleaseResult-checkRelease get executable = do+refreshCache :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> ReleaseCache -> IO ()+refreshCache base clock get executable previous = do   result <- timeout 3000000 (get "https://api.github.com/repos/apauley/hledger-flow/releases/latest")+  now <- clock   case join result >>= releaseVersion of-    Just latest | latest > version -> do-      confirmed <- timeout 1000000 (homebrewOffers latest `catchIOError` const (pure False))-      let advice = if confirmed == Just True-            then "Upgrade with: brew update && brew upgrade ferntrail/tap/hledger-flow"-            else "https://github.com/apauley/hledger-flow/releases/latest"-      pure (UpdateAvailable latest advice)-    Just _ -> pure UpToDate-    Nothing -> pure CheckUnavailable-  where-    homebrewOffers latest = do-      path <- executable >>= canonicalizePath-      owned <- ownsHomebrewBinary path-      if owned then do-        formula <- get "https://raw.githubusercontent.com/FernTrail/homebrew-tap/main/Formula/hledger-flow.rb"-        let tagLine = "tag = \"v" <> renderVersion latest <> "\""-        pure $ fromMaybe False $ do-          body <- formula-          contents <- either (const Nothing) Just (T.decodeUtf8' (BL.toStrict body))-          pure $ tagLine `elem` map T.strip (T.lines contents)-      else pure False+    Nothing -> saveCache base previous { lastFailure = Just now }+    Just latest -> do+      let save confirmed = saveCache base (ReleaseCache (Just (Observation latest now confirmed)) Nothing)+      -- A short command can exit during the optional formula lookup.+      save False+      when (latest > version) $ do+        owned <- brewOwned executable+        when owned $ do+          response <- timeout 1000000 (get "https://raw.githubusercontent.com/FernTrail/homebrew-tap/main/Formula/hledger-flow.rb")+          let published = do+                body <- join response+                contents <- either (const Nothing) Just (T.decodeUtf8' (BL.toStrict body))+                let tagLine = "tag = \"v" <> renderVersion latest <> "\""+                pure (tagLine `elem` map T.strip (T.lines contents))+          when (published == Just True) (save True)++brewOwned :: IO FilePath -> IO Bool+brewOwned executable =+  (executable >>= canonicalizePath >>= ownsHomebrewBinary) `catchIOError` const (pure False)  releaseVersion :: BL.ByteString -> Maybe Version releaseVersion bytes = do
src/Hledger/Flow/Reports.hs view
@@ -47,7 +47,7 @@       skippedText = if skipped == 0 then "" else " Skipped " <> plural skipped "empty report" <> "."   channelOutLn ch $ "Generated " <> plural (length (rights reports)) "report" <> " in reports/." <> skippedText   channelOutLn ch "Each report starts with the hledger command that made it. Copy that line to tailor the report."-  logVerbose opts ch ("Reports took " <> T.pack (show diff))+  logVerbose opts ch ("Reports finished in " <> showDuration diff)  generateReports' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> IO [Either AbsFile AbsFile] generateReports' opts ch = do@@ -115,31 +115,33 @@   generateReport opts ch journals year baseOutDir [relfile|transfer-balance.txt|] reportArgs (\txt -> (length $ T.lines txt) > 4)  generateReport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [FilePath] -> Integer -> AbsDir -> RelFile -> [T.Text] -> (T.Text -> Bool) -> IO (Either AbsFile AbsFile)-generateReport opts ch journals year reportsDir fileName args successCheck = do+generateReport opts logCh journals year reportsDir fileName args successCheck = do   yearDir <- parseRelDir (show year)   let outputFile = reportsDir </> yearDir </> fileName-  PathIO.createDirIfMissing True (parent outputFile)-  let relativeJournals = map (relativeToBase opts) journals-  let relativeOutputFile = relativeToBase opts (toFilePath outputFile)-  let confArgs = hledgerConfArgs opts-  info <- hledgerInfo opts-  let fileArgs = concatMap (\file -> ["--file", hledgerFileArgument info file])-  let periodArgs = ["--period", T.pack (show year)]-  let reportArgs = confArgs ++ fileArgs journals ++ periodArgs ++ args-  let reportDisplayArgs = confArgs ++ fileArgs relativeJournals ++ periodArgs ++ args-  let hledger = T.pack (toFilePath (FlowTypes.hlPath info))-  let cmdLabel = "hledger " <> showCmdArgs reportDisplayArgs-  ((_, stdOut, _), _) <- timeAndExitOnErr opts ch cmdLabel dummyLogger channelErr (hledger, reportArgs, Nothing)-  if (successCheck stdOut)-    then do-      T.writeFile (toFilePath outputFile) (cmdLabel <> "\n\n" <> stdOut)-      logVerbose opts ch $ "Wrote " <> T.pack relativeOutputFile-      return $ Right outputFile-    else do-      logVerbose opts ch $ "Skipped empty report " <> T.pack relativeOutputFile-      exists <- PathIO.doesFileExist outputFile-      when exists (PathIO.removeFile outputFile)-      return $ Left outputFile+  withLogBlock opts (sequential opts) logCh ("Report " <> T.pack (relativeToBase opts (toFilePath outputFile))) $ \ch -> do+    PathIO.createDirIfMissing True (parent outputFile)+    let relativeJournals = map (relativeToBase opts) journals+    let relativeOutputFile = relativeToBase opts (toFilePath outputFile)+    let confArgs = hledgerConfArgs opts+    info <- hledgerInfo opts+    let fileArgs = concatMap (\file -> ["--file", hledgerFileArgument info file])+    let periodArgs = ["--period", T.pack (show year)]+    let reportArgs = confArgs ++ fileArgs journals ++ periodArgs ++ args+    let reportDisplayArgs = confArgs ++ fileArgs relativeJournals ++ periodArgs ++ args+    let hledger = T.pack (toFilePath (FlowTypes.hlPath info))+    let cmdLabel = "hledger " <> showCmdArgs reportDisplayArgs+    let external = External "hledger" ("write " <> T.pack relativeOutputFile) [hledger : reportArgs] []+    stdOut <- runExternal opts ch external False (hledger, reportArgs, Nothing)+    if (successCheck stdOut)+      then do+        T.writeFile (toFilePath outputFile) (cmdLabel <> "\n\n" <> stdOut)+        logVerbose opts ch $ "Wrote " <> T.pack relativeOutputFile+        return $ Right outputFile+      else do+        logVerbose opts ch $ "Skipped empty report " <> T.pack relativeOutputFile+        exists <- PathIO.doesFileExist outputFile+        when exists (PathIO.removeFile outputFile)+        return $ Left outputFile  ownerParameters :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [RelDir] -> IO [ReportParams] ownerParameters opts ch owners = do
src/Hledger/Flow/RuntimeOptions.hs view
@@ -6,11 +6,17 @@ import Hledger.Flow.Types (BaseDir, HledgerConf, HledgerInfo (..), RunDir) import qualified Hledger.Flow.Types as Types +data ImportPolicy = Full | TimestampOnly | RunScripts | SkipUnchangedScripts+  deriving (Eq, Show)++onlyNewFiles :: RuntimeOptions -> Bool+onlyNewFiles = (== TimestampOnly) . importPolicy+ data RuntimeOptions = RuntimeOptions   { baseDir :: BaseDir,     importRunDir :: RunDir,     importStartYear :: Maybe Integer,-    onlyNewFiles :: Bool,+    importPolicy :: ImportPolicy,     hfVersion :: T.Text,     hledgerExecutable :: AbsFile,     hledgerVersion :: IO T.Text,
src/Hledger/Flow/Status/Render.hs view
@@ -29,27 +29,10 @@ import Hledger.Flow.Status import Hledger.Flow.Types (HledgerConf, HledgerInfo (..)) import Path (toFilePath)-import System.Console.ANSI (Color (..), ColorIntensity (..), ConsoleIntensity (..), ConsoleLayer (..), SGR (..), hSupportsANSIColor, setSGRCode)-import System.Environment (lookupEnv)+import Hledger.Flow.Logging (bold, green, red, useColor, yellow) import qualified System.FilePath as FilePath-import System.IO (Handle, stdout)---- | Colour only on a terminal, and only when NO_COLOR is unset or empty.-useColor :: Handle -> IO Bool-useColor h = do-  noColor <- lookupEnv "NO_COLOR"-  supported <- hSupportsANSIColor h-  return (supported && maybe True null noColor)--paint :: Bool -> [SGR] -> T.Text -> T.Text-paint False _ txt = txt-paint True sgr txt = T.pack (setSGRCode sgr) <> txt <> T.pack (setSGRCode [Reset])+import System.IO (stdout) -green, red, yellow, bold :: Bool -> T.Text -> T.Text-green c = paint c [SetColor Foreground Vivid Green]-red c = paint c [SetColor Foreground Vivid Red]-yellow c = paint c [SetColor Foreground Vivid Yellow]-bold c = paint c [SetConsoleIntensity BoldIntensity]  baseText :: LedgerStatus -> FilePath baseText = FilePath.dropTrailingPathSeparator . toFilePath . lsBaseDir
src/Hledger/Flow/Types.hs view
@@ -4,7 +4,6 @@  import qualified Data.Text as T import Hledger.Flow.PathHelpers-import Data.Time.Clock (NominalDiffTime) import System.Exit (ExitCode) import System.IO (Handle) @@ -15,8 +14,6 @@ data LogMessage = StdOut T.Text | StdErr T.Text | Terminate deriving (Show)  type FullOutput = (ExitCode, T.Text, T.Text)--type FullTimedOutput = (FullOutput, NominalDiffTime)  type ProcInput = (T.Text, [T.Text], Maybe Handle) 
test/Common/Integration.hs view
@@ -9,7 +9,6 @@ import Control.Exception (bracket, bracket_, finally) import Control.Monad (forM_, unless) import qualified Data.List as List (intercalate, isInfixOf, isPrefixOf)-import qualified Data.Text as T import qualified Data.Text.IO as T import Data.Time.Clock (UTCTime(..), addUTCTime, secondsToDiffTime) import Data.Time.Calendar (fromGregorian)@@ -102,19 +101,16 @@       file = base Path.</> [Path.relfile|statement.csv|]   ch <- newTChanIO   let check expected message = do-        exists <- verboseTestFile opts ch file+        exists <- verboseTestFile opts ch "statement" file         assertEqual "file existence" expected exists         entry <- atomically (tryReadTChan ch)         case entry of-          Just (StdErr txt) -> do-            let (timestamp, body) = T.breakOn "\t" txt-            assertBool "verbose log starts with a timestamp" (not (T.null timestamp))-            assertEqual "verbose log uses the relative path" ("\thledger-flow " <> message) body+          Just (StdErr txt) -> assertEqual "verbose log names hledger-flow and uses the relative path" ("hledger-flow: " <> message) txt           _ -> assertFailure "expected a verbose message on stderr"-  check False "Looked for but did not find 'statement.csv'\n"+  check False "No statement at statement.csv\n"   T.writeFile (Path.toFilePath file) ""-  check True "Found 'statement.csv'\n"-  _ <- verboseTestFile (defaultOpts base) ch file+  check True "Found statement: statement.csv\n"+  _ <- verboseTestFile (defaultOpts base) ch "statement" file   quietEntry <- atomically (tryReadTChan ch)   case quietEntry of     Nothing -> return ()@@ -215,15 +211,18 @@  -- Paths come out resolved (macOS /private/var, symlinked homes), so match both spellings of HOME. testHomeDirs :: Test-testHomeDirs = TestLabel "home folder spellings" $ TestCase $ unless (Sys.os == "mingw32") $ withTestDir $ \base -> do+testHomeDirs = TestLabel "home folder spellings" $ TestCase $ withTestDir $ \base -> do   let root = Path.toFilePath base-  Directory.createDirectory (root </> "real")-  Directory.createDirectoryLink (root </> "real") (root </> "link")-  real <- Directory.canonicalizePath (root </> "real")-  forM_ ([root </> "link", root </> "link/", real ++ "/"] :: [FilePath]) $ \home -> withEnv "HOME" home $ do-    homes <- homeDirs-    assertEqual ("resolved path with HOME=" ++ home) ("~" </> "fin") (tildePath homes (real </> "fin"))-    assertEqual ("HOME's spelling with HOME=" ++ home) ("~" </> "fin") (tildePath homes (home </> "fin"))+  if Sys.os == "mingw32"+    then withEnv "USERPROFILE" root $ homeDirs >>= assertEqual "keep native profile paths absolute" []+    else do+      Directory.createDirectory (root </> "real")+      Directory.createDirectoryLink (root </> "real") (root </> "link")+      real <- Directory.canonicalizePath (root </> "real")+      forM_ ([root </> "link", root </> "link/", real ++ "/"] :: [FilePath]) $ \home -> withEnv "HOME" home $ do+        homes <- homeDirs+        assertEqual ("resolved path with HOME=" ++ home) ("~" </> "fin") (tildePath homes (real </> "fin"))+        assertEqual ("HOME's spelling with HOME=" ++ home) ("~" </> "fin") (tildePath homes (home </> "fin"))  testWriteGeneratedText :: Test testWriteGeneratedText = TestLabel "write generated text" $ TestCase $ withTestDir $ \base -> do
test/Common/Unit.hs view
@@ -94,7 +94,16 @@           "trailing backslash is special unless it separates paths" ~: homeShellPath Posix homes "/home/me/a\\"             ~?= if FilePath.isPathSeparator '\\' then Just (quoted ["a"]) else Nothing,           "powershell specials fall back" ~: map (\c -> homeShellPath PowerShell homes ("/home/me/a" ++ [c])) "$`\"\x201c\x201d\x201e" ~?= replicate 6 Nothing,-          "posix-only specials are fine in powershell" ~: homeShellPath PowerShell homes "/home/me/a!" ~?= Just (quoted ["a!"])+          "posix-only specials are fine in powershell" ~: homeShellPath PowerShell homes "/home/me/a!" ~?= Just (quoted ["a!"]),+          "posix pipeline" ~: pasteCommand Posix "/home/me/fin/" homes [["/srv/construct", "a b", "-"], ["/srv/hledger", "print"]]+            ~?= "/srv/construct 'a b' - | /srv/hledger print",+          "powershell runs a quoted program with &" ~: pasteCommand PowerShell "/home/me/fin/" homes [["/home/me/bin/pre.ps1", "Lee's"]]+            ~?= "& " <> quoted ["bin", "pre.ps1"] <> " 'Lee''s'",+          "powershell leaves a bare program alone" ~: pasteCommand PowerShell "/home/me/fin/" homes [["hledger", "print"]] ~?= "hledger print",+          "paths below the base start with ./" ~: pasteCommand Posix "/home/me/fin/" homes [["/home/me/bin/hledger", "--file", "/home/me/fin/import/my a.csv", "/home/me/fin"]]+            ~?= displayedProgram <> " --file '" <> native [".", "import", "my a.csv"] <> "' .",+          "powershell paths below the base" ~: pasteCommand PowerShell "/home/me/fin/" homes [["/home/me/fin/my hooks/pre.ps1", "/home/me/fin/a b.csv"]]+            ~?= "& '" <> native [".", "my hooks", "pre.ps1"] <> "' '" <> native [".", "a b.csv"] <> "'"         ]  testRelativeToBase :: Test@@ -175,6 +184,6 @@   TestList     [ "long line" ~: trimToolOutput (T.replicate 400 "x") ~?= T.replicate 300 "x" <> "…",       "many lines" ~: T.lines (trimToolOutput (T.unlines (map (T.pack . show) [1 .. 25 :: Int])))-        ~?= map (T.pack . show) [1 .. 20 :: Int] ++ ["… (5 more lines)"],+        ~?= map (T.pack . show) [1 .. 20 :: Int] ++ ["… (5 more lines; run with --verbose to see them all)"],       "short output unchanged" ~: trimToolOutput "a\nb\n" ~?= "a\nb\n"     ]
test/Environment/Unit.hs view
@@ -19,6 +19,9 @@ testImportLines = TestCase $ do   let out = renderSettings id (importSettings [] defaultHlInfo opts {importStartYear = Just 2024, batchSize = 20})       line l = filter ((l <> " ") `T.isPrefixOf`) out+  let files policy = [(sValue row,sFlags row) | row <- importSettings [] defaultHlInfo opts {importPolicy = policy}, sLabel row == "Files"]+  files RunScripts @?= [("incremental (run-scripts)", "--incremental")]+  files SkipUnchangedScripts @?= [("incremental (skip-unchanged-scripts)", "--incremental")]   assertEqual "one line per setting" 8 (length out)   assertBool "full hledger path and version" (any ((T.pack (toFilePath [relfile|path/to/hledger|]) <> "  (1.2.3)") `T.isInfixOf`) (line "hledger"))   assertBool "flag beside its value" (any ("2024 onwards" `T.isInfixOf`) (line "Start year") && any ("--start-year" `T.isSuffixOf`) (line "Start year"))
+ test/Incremental/Cache.hs view
@@ -0,0 +1,56 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Incremental.Cache (tests) where++import Control.Exception (IOException, try)+import Data.Maybe (isJust)+import qualified Data.Text.IO as T+import Hledger.Flow.Cache (initializeCache)+import Hledger.Flow.Import.Cache+import Hledger.Flow.Import.Fingerprint+import qualified Path as P+import qualified Path.IO as P+import Test.HUnit++tests :: Test+tests = TestLabel "incremental records" $ TestCase $ P.withSystemTempDir "records" $ \base -> do+  let source = base P.</> [P.relfile|import/alice/bank/card/1-in/2026/march.csv|]+      other = base P.</> [P.relfile|import/alice/bank/card/1-in/2026/march.tsv|]+      out = base P.</> [P.relfile|march.journal|]+      ignored = base P.</> [P.relfile|.hledger-flow/cache/.gitignore|]+      bytes = "2026-03-01 coffee\n"+  record <- recordPath base source+  another <- recordPath base other+  assertBool "the source extension distinguishes records" (another /= record)+  assertEqual "missing record" Nothing =<< readRecord record+  initializeCache base+  assertEqual "self-ignoring cache" "*\n" =<< T.readFile (P.toFilePath ignored)+  T.writeFile (P.toFilePath ignored) "mine"+  initializeCache base+  assertEqual "preserve custom ignore" "mine" =<< T.readFile (P.toFilePath ignored)+  digest <- installOutput False out $ \staged -> T.writeFile (P.toFilePath staged) bytes+  assertEqual "published digest is the staged bytes" digest =<< digestFile out+  let stage = StageRecord [("source", "march.csv")] digest+      entry = StatementRecord Nothing (Just stage)+  saveRecord record entry+  assertEqual "read published record" (Just entry) =<< readRecord record+  assertEqual "verified output" Nothing =<< stageMiss (stageInputs stage) out (Just stage)+  before <- P.getModificationTime out+  _ <- installOutput True out $ \staged -> T.writeFile (P.toFilePath staged) bytes+  assertEqual "preserve identical intermediate" before =<< P.getModificationTime out+  _ <- try (installOutput False out $ \_ -> ioError (userError "conversion failed")) :: IO (Either IOException Digest)+  assertEqual "failed conversion leaves installed output" digest =<< digestFile out+  T.writeFile (P.toFilePath out) "another importer"+  assertBool "intervening writer invalidates record" . isJust =<< stageMiss (stageInputs stage) out (Just stage)+  digestB <- installOutput False out $ \staged -> T.writeFile (P.toFilePath staged) "output from writer B"+  let stageB = StageRecord [("source", "B")] digestB+  saveRecord record (StatementRecord Nothing (Just stageB))+  -- Writer A, paused after installing A, saves its record after B installs B.+  saveRecord record entry+  late <- readRecord record+  assertBool "late publication must miss on B's output" . isJust =<< stageMiss (stageInputs stage) out (late >>= journalRecord)+  T.writeFile (P.toFilePath record) "[1,{\"preprocessRecord\":null,\"journalRecord\":null}]"+  assertEqual "discard records predating the snapshot collision check" Nothing =<< readRecord record+  T.writeFile (P.toFilePath record) "{}"+  assertEqual "invalid record" Nothing =<< readRecord record
+ test/Incremental/Dependencies.hs view
@@ -0,0 +1,185 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Incremental.Dependencies (tests) where++import Control.Monad (forM_)+import Control.Exception (bracket)+import GHC.IO.Encoding (getLocaleEncoding, setLocaleEncoding, utf8, latin1)+import qualified Data.ByteString as B+import Data.IORef+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import Hledger.Flow.Import.Dependencies+import Hledger.Flow.Import.Fingerprint (digestBytes)+import Hledger.Flow.Import.ImportHelpers (extractImportDirs)+import Hledger.Flow.Types (HledgerConf(..))+import Path+import qualified Path.IO as P+import qualified System.Directory as D+import System.Exit (ExitCode(..))+import Data.Time (addUTCTime)+import Test.HUnit hiding (path)+import TestHelpers (withTestDir)++fallback :: Show a => ReuseDecision a -> Assertion+fallback (Fallback _) = pure ()+fallback r = assertFailure ("expected fallback, got " ++ show r)++tests :: Test+tests = TestList+  [ "include expansion and selection" ~: withTestDir (\base -> do+      let root = base </> [relfile|main.rules|]+          mid = base </> [relfile|middle.rules|]+          leaf = base </> [relfile|empty.rules|]+          mainBytes = "include  middle.rules\r\ninclude empty.rules\n"+          midBytes = "include empty.rules"+          write p = B.writeFile (toFilePath p)+      write root mainBytes+      write mid midBytes+      write leaf ""+      forM_ [V1524,V1996,OtherSupported] $ \release -> do+        tree <- discoverRules base release root+        tree @?= Reusable [(root,digestBytes mainBytes),(mid,digestBytes midBytes),(leaf,digestBytes ""),(leaf,digestBytes "")]+      ruleIncludes V1524 root "include   a  \r\n include b\ninclude\tc\n" @?= Reusable ["a  "]+      ruleIncludes V1996 root "include   a  \r\n include b\ninclude\tc\n" @?= Reusable ["a"]+      fallback (nestedRuleIncludes V1524 mid "include final.rules\r\n")+      nestedRuleIncludes V1996 mid "include final.rules\r\n" @?= Reusable ["final.rules"]+      forM_ ["include filename.rules  ", " include file.rules", "include\tfile.rules", "Include file.rules"] $+        fallback . ruleIncludes OtherSupported root+      fallback (nestedRuleIncludes OtherSupported mid "include final.rules\r\n")+      fallback (nestedRuleIncludes OtherSupported mid (B.pack [0xef,0xbb,0xbf] <> "include final.rules"))+      ruleIncludes OtherSupported root "include  shopping categories.rules\r\n" @?= Reusable ["shopping categories.rules"]+      write root "include middle.rules\r\n"+      write mid "include main.rules\n"+      native <- discoverRules base V1524 root+      case native of+        Fallback CyclicRules{} -> assertFailure "top-level and native reads have different include targets"+        Fallback _ -> pure ()+        _ -> assertFailure "expected ordinary conversion for native line endings"+      write root mainBytes+      write mid "include missing.rules"+      discoverRules base V1996 root >>= fallback+      write mid "include main.rules"+      discoverRules base V1996 root >>= fallback+      write mid (T.encodeUtf8 ("include " <> T.pack (toFilePath leaf)))+      discoverRules base V1996 root >>= fallback+#ifndef mingw32_HOST_OS+      D.removeFile (toFilePath mid)+      D.createFileLink (toFilePath leaf) (toFilePath mid)+      tree <- discoverRules base V1996 root+      tree @?= Reusable [(root,digestBytes mainBytes),(mid,digestBytes ""),(leaf,digestBytes "")]+      D.removeFile (toFilePath mid)+      D.createFileLink (toFilePath root) (toFilePath mid)+      write root "include middle.rules"+      cycleResult <- discoverRules base V1524 root+      case cycleResult of+        Fallback CyclicRules{} -> pure ()+        _ -> assertFailure "a nested symlink resolving the same relative include repeats the cycle"+#endif+      let imports = base </> [reldir|import|]+          csv = imports </> [relfile|alice/bank/acct/1-in/2026/01_rfo-special.csv|]+          bank = imports </> [relfile|bank.rules|]+          override = imports </> [relfile|alice/bank/acct/rfo-special.rules|]+      dirs <- either (fail . T.unpack) pure (extractImportDirs imports csv)+      P.createDirIfMissing True (parent csv)+      write bank ""+      selectRules csv dirs >>= (@?= Reusable (Just bank))+      write override ""+      D.getModificationTime (toFilePath bank) >>= D.setModificationTime (toFilePath override) . addUTCTime (-1000)+      selectRules csv dirs >>= (@?= Reusable (Just override))+      D.removeFile (toFilePath override)+      selectRules csv dirs >>= (@?= Reusable (Just bank)))+  , "config syntax and whole-file digest" ~: withTestDir (\base -> do+      let file = base </> [relfile|hledger.conf|]+          ok release s = checkConfig release file s @?= Reusable ()+          no release s = fallback (checkConfig release file s)+      forM_ [V1524,V1996,OtherSupported] $ \release -> do+        ok release "# comment\n[balance] --period=thismonth\n[register]\n@file\n[general]\n[print]\n"+        forM_ ["--period=thismonth", "[general] --debug", "[print]\n@indirect", "[txns]", "[bal]", "[b]", "[alias]", "[missing", "[general", "[balance]\n[print] --period=thismonth"] (no release)+      ok V1996 "[print] --layout=hledger1 # layout\n"+      ok OtherSupported "[print] --layout=hledger1 # layout\n"+      no OtherSupported "[print]\n [balance]\n--empty"+      no OtherSupported "[holdings]"+      no OtherSupported "[demo]"+      no V1524 "[print]\n--layout=hledger1"+      no V1996 "[general]\n--layout=hledger1"+      ok V1996 "[holdings]"+      no V1524 "[print]\n [balance]\n--empty"+      no V1524 "[holdings]"+      ok V1524 "[demo]"+      no V1996 "[demo]"+      forM_ ["[balance]\n--period=thismonth", "[balance]\n--period=thisyear"] $ \bytes -> do+        B.writeFile (toFilePath file) bytes+        discoverConfig V1996 (Just (GivenConf (toFilePath file))) >>= (@?= Reusable (Just (file,digestBytes bytes)))+      fallback (checkArguments ["print","@file"])+      discoverConfig V1996 (Just (GivenConf (toFilePath base))) >>= fallback+#ifndef mingw32_HOST_OS+      D.removeFile (toFilePath file)+      D.createFileLink "hledger.conf" (toFilePath file)+      D.withCurrentDirectory (toFilePath base) (discoverConfig V1996 Nothing >>= fallback)+#endif+      )+  , "directives eligibility" ~: withTestDir (\base -> do+      let file = base </> [relfile|directives.journal|]+          check = checkDirectives V1996 file . T.encodeUtf8+      forM_ ["account assets:bank:current  ; type: C","commodity 1,000.00000000 BTC  ; lots:","commodity €1,000.00","P 2000-01-01 EUR €1","P 2000/01/01 BTC 12.30 EUR","; tags\n# prices\n* prices\n\n"] $ \s -> check s @?= Reusable ()+      forM_ ["account assets:any\n    note", "comment\ncommodity €1\nend comment","include *.journal","P today EUR €1","P 2000-01-01 EUR €1 trailing","commodity 1,234 ABC extra", "2025/01/01 A"] (fallback . check))+  , "converter identity" ~: bracket getLocaleEncoding setLocaleEncoding (\_ -> withTestDir (\base -> do+      setLocaleEncoding utf8+      let real = base </> [relfile|actual|]+          second = base </> [relfile|other|]+          native = B.pack [127,69,76,70]+      B.writeFile (toFilePath real) (native <> "build A")+      B.writeFile (toFilePath second) native+#ifndef mingw32_HOST_OS+      let link = base </> [relfile|hledger|]+      D.createFileLink (toFilePath real) (toFilePath link)+      let executable = link+#else+      let executable = real+#endif+      probes <- newIORef (0 :: Int)+      let version _ = modifyIORef' probes (+1) >> pure (ExitSuccess,"hledger 1.99.6-gc3a608c2b-20261101, mac-aarch64")+      probe <- converterProbeWith executable version+      first <- probe+      case first of+        Reusable c -> converterRelease c @?= V1996+        Fallback why -> assertFailure (show why)+      probe >>= (@?= first)+      readIORef probes >>= (@?= 1)+      let replacement = toFilePath (base </> [relfile|replacement|])+      previousTime <- D.getModificationTime (toFilePath real)+      B.writeFile replacement (native <> "build B")+      D.setModificationTime replacement previousTime+      D.renameFile replacement (toFilePath real)+      -- Compare the key persisted between imports, which excludes file stamps.+      -- Both executables have the same path, version, length and mtime.+      let checkReplacement result = case (first, result) of+            (Reusable old, Reusable new) ->+              assertBool "replacement changes saved converter key" (converterKey old /= converterKey new)+            _ -> assertFailure ("expected two eligible builds: " ++ show (first, result))+      probe >>= checkReplacement+      freshProbe <- converterProbeWith executable version+      freshProbe >>= checkReplacement+#ifndef mingw32_HOST_OS+      D.removeFile (toFilePath link)+      D.createFileLink (toFilePath second) (toFilePath link)+      retargeted <- probe+      assertBool "retargeted symlink" (retargeted /= first)+#endif+      forM_ ["h.cmd","h.BAT","scoop.exe","wrapped","unknown"] $ \name -> do+        name' <- parseRelFile name+        let path = base </> name'+        B.writeFile (toFilePath path) (if name == "wrapped" then "#!/bin/sh\n" else if name == "unknown" then "hledger" else native)+        if name == "scoop.exe" then B.writeFile (toFilePath (base </> [relfile|scoop.shim|])) "path = hidden" else pure ()+        converterProbeWith path version >>= (>>= fallback)+      let identify v = converterProbeWith real (const (pure (ExitSuccess,"hledger " <> v))) >>= id+      forM_ [("1.52.4",V1524),("1.99.6",V1996),("1.52.40",OtherSupported),+             ("1.99-g01a4b4866-20261113, mac-aarch64",OtherSupported),("1.100",OtherSupported),("2.0",OtherSupported)] $ \(v, expected) ->+        (fmap converterRelease <$> identify v) >>= (@?= Reusable expected)+      forM_ ["1.52.3","1.52","1.9.40","1.99.x","1.99.","1..99","unknown"] $ \v -> identify v >>= fallback+      setLocaleEncoding latin1+      converterProbeWith real version >>= (>>= fallback)))+  ]
+ test/Incremental/Platform.hs view
@@ -0,0 +1,157 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Incremental.Platform (tests) where++import Control.Monad (forM_)+import qualified Data.ByteString as BS+import Data.Time (addUTCTime)+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.PrivateDirectory+import Path+import qualified Path.IO as PathIO+import qualified System.Directory as Dir+import System.IO.Error (tryIOError)+import Test.HUnit hiding (Path)+import TestHelpers (withTestDir)+#ifdef mingw32_HOST_OS+import System.Process (callProcess)+import System.Environment (setEnv, unsetEnv)+import Control.Exception (bracket_)+#else+import Control.Exception (bracket)+import Data.Bits ((.&.))+import qualified System.Posix.Files as Posix+#endif+#ifdef darwin_HOST_OS+import Data.List (isInfixOf)+import qualified System.Posix.Directory as Posix+import System.Process (callProcess, readProcess)+#endif++tests :: Test+tests = TestLabel "Incremental platform primitives" $ TestList+  [ TestLabel "SHA-256 over exact file bytes" $ TestCase $ withTestDir $ \root -> do+      let file = root </> [relfile|input|]+      forM_ [(0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"),+             (1000000, "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0")] $+        \(len, expected) -> do+          BS.writeFile (toFilePath file) (BS.replicate len 97)+          assertEqual "in-memory digest" expected (digestHex (digestBytes (BS.replicate len 97)))+          digest <- digestFile file+          assertEqual "fixed SHA-256 vector" expected (digestHex digest)+          let copy = root </> [relfile|copied|]+          writeFile (toFilePath copy) "old contents"+          copied <- copyDigest file copy+          assertEqual "hash of copied bytes" digest copied+          bytes <- BS.readFile (toFilePath copy)+          assertEqual "copied all bytes" (BS.replicate len 97) bytes+          observation <- observeFile file+          stamp <- statFile file+          assertEqual "same file metadata" stamp (fileStamp observation)+          assertEqual "observed bytes" digest (observationDigest observation)+          assertEqual "observed size" (fromIntegral len) (stampSize (fileStamp observation))+  , TestLabel "same-byte rewrite changes transient observation" $ TestCase $ withTestDir $ \root -> do+      let file = root </> [relfile|input|]+          name = toFilePath file+      writeFile name "same bytes"+      first <- observeFile file+      originalTime <- Dir.getModificationTime name+      writeFile name "same bytes"+      Dir.setModificationTime name (addUTCTime 10 originalTime)+      second <- observeFile file+      assertBool "observed rewrite" (first /= second)+      assertEqual "unchanged content key" (observationDigest first) (observationDigest second)+  , TestLabel "file identity distinguishes copies" $ TestCase $ withTestDir $ \root -> do+      let file = root </> [relfile|input|]+          copy = root </> [relfile|copy|]+      writeFile (toFilePath file) "shared bytes"+      Dir.copyFile (toFilePath file) (toFilePath copy)+      a <- observeFile file+      b <- observeFile copy+      assertBool "different files" (stampIdentity (fileStamp a) /= stampIdentity (fileStamp b))+  , TestLabel "private staging under a shared directory" $ TestCase $ withTestDir $ \root -> do+      let run = root </> [reldir|run|]+#ifdef mingw32_HOST_OS+      withAclPaths root run $ do+        powershell $ unlines+          [ "$acl = [IO.Directory]::GetAccessControl($env:HF_ACL_PARENT)"+          , "$everyone = [System.Security.Principal.SecurityIdentifier]'S-1-1-0'"+          , "$acl.AddAccessRule([System.Security.AccessControl.FileSystemAccessRule]::new($everyone,'ReadAndExecute','ContainerInherit,ObjectInherit','None','Allow'))"+          , "[IO.Directory]::SetAccessControl($env:HF_ACL_PARENT, $acl)"+          ]+        createPrivateDirectory run+        let nested = run </> [reldir|nested|]+            copied = nested </> [relfile|statement|]+            source = root </> [relfile|source|]+        PathIO.createDir nested+        writeFile (toFilePath source) "private"+        _ <- copyDigest source copied+        powershell $ unlines+          [ "$sid = [System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value"+          , "$dir = [IO.Directory]::GetAccessControl($env:HF_ACL_RUN)"+          , "if (-not $dir.AreAccessRulesProtected) { throw 'inheriting cache parent DACL' }"+          , "$file = [IO.File]::GetAccessControl([IO.Path]::Combine($env:HF_ACL_RUN, 'nested/statement'))"+          , "foreach ($acl in @($dir, $file)) {"+          , "  $rules = $acl.GetAccessRules($true,$true,[System.Security.Principal.SecurityIdentifier])"+          , "  if ($rules.Count -ne 1 -or $rules[0].IdentityReference.Value -ne $sid -or $rules[0].AccessControlType -ne 'Allow' -or $rules[0].FileSystemRights -ne 'FullControl') { throw 'expected only current user with full access' }"+          , "  if ($acl -ne $dir -and -not $rules[0].IsInherited) { throw 'file lacks inherited access' }"+          , "}"+          ]+#else+      Posix.setFileMode (toFilePath root) 0o777+      createPrivateDirectory run+      mode <- Posix.fileMode <$> Posix.getFileStatus (toFilePath run)+      assertEqual "owner-only directory" 0o700 (mode .&. 0o777)+#endif+#ifndef mingw32_HOST_OS+  , TestLabel "unusable creation mode rejects the empty directory" $ TestCase $ withTestDir $ \root -> do+      let run = root </> [reldir|run|]+      result <- bracket (Posix.setFileCreationMask 0o177) Posix.setFileCreationMask $+        \_ -> tryIOError (createPrivateDirectory run)+      assertRejectedDirectory run result+#endif+#ifdef darwin_HOST_OS+  , TestLabel "inherited macOS ACL rejects staging" $ TestCase $ withTestDir $ \root -> do+      let probe = root </> [reldir|probe|]+          run = root </> [reldir|run|]+      callProcess "/bin/chmod" ["+a", "everyone allow list,search,directory_inherit", toFilePath root]+      Posix.createDirectory (toFilePath probe) 0o700+      acl <- readProcess "/bin/ls" ["-lde", toFilePath probe] ""+      assertBool "the filesystem inherited the test grant" ("inherited allow" `isInfixOf` acl)+      result <- tryIOError (createPrivateDirectory run)+      assertRejectedDirectory run result+#endif+  , TestLabel "occupied path stops staging before copying" $ TestCase $ withTestDir $ \root -> do+      let run = root </> [reldir|occupied|]+      PathIO.createDir run+      result <- tryIOError $ do+        createPrivateDirectory run+        writeFile (toFilePath (run </> [relfile|statement|])) "private"+      case result of+        Left _ -> return ()+        Right () -> assertFailure "used a pre-existing directory"+      entries <- Dir.listDirectory (toFilePath run)+      assertEqual "no copies made" [] entries+  ]++#ifndef mingw32_HOST_OS+assertRejectedDirectory :: Path Abs Dir -> Either IOError () -> Assertion+assertRejectedDirectory run result = do+  case result of+    Left _ -> pure ()+    Right () -> assertFailure "accepted unverified staging permissions"+  exists <- PathIO.doesDirExist run+  assertBool "removed the rejected empty directory" (not exists)+#endif++#ifdef mingw32_HOST_OS+withAclPaths :: Path Abs Dir -> Path Abs Dir -> IO a -> IO a+withAclPaths root run =+  bracket_ (setEnv "HF_ACL_PARENT" (toFilePath root) >> setEnv "HF_ACL_RUN" (toFilePath run))+           (unsetEnv "HF_ACL_PARENT" >> unsetEnv "HF_ACL_RUN")++powershell :: String -> IO ()+powershell command = callProcess "powershell.exe" ["-NoProfile", "-NonInteractive", "-Command", "$ErrorActionPreference = 'Stop'; " ++ command]+#endif
+ test/Incremental/Snapshot.hs view
@@ -0,0 +1,313 @@+{-# LANGUAGE CApiFFI #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Incremental.Snapshot (tests, runWorker) where++import Control.Concurrent+import Control.Concurrent.STM (newTChanIO)+import Control.Exception (bracket, try, IOException, throwIO)+import Control.Monad (unless)+import GHC.Conc (threadStatus, ThreadStatus(..))+import GHC.IO.Encoding (getLocaleEncoding, setLocaleEncoding, utf8)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Char8 as B8+import qualified Data.Map.Strict as Map+import qualified Data.Text as T+import Hledger.Flow.Import.Cache+import Hledger.Flow.Import.Dependencies+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.ImportHelpers (extractImportDirs)+import Hledger.Flow.Import.Incremental+import Hledger.Flow.Import.PrivateDirectory+import Hledger.Flow.Import.Snapshot+import Hledger.Flow.RuntimeOptions+import Path+import qualified Path.IO as P+import qualified System.Directory as D+#ifdef mingw32_HOST_OS+import Data.Word (Word32)+import Data.Text.Encoding (encodeUtf8)+import Foreign.C.String (CWString, peekCWString, withCWString)+import Foreign.Marshal.Array (allocaArray)+#endif+import qualified System.FilePath as FP+import System.Environment (getExecutablePath)+import System.IO (hFlush, hGetLine, stdout)+import System.Process+import System.Timeout (timeout)+import Test.HUnit+import TestHelpers (defaultOpts, withTestDir)++tests :: Test+tests = TestLabel "Private input snapshots" $ TestList+  [ TestLabel "no snapshot area for a hit" $ TestCase $ withTestDir $ \base -> do+      withSnapshotRun base $ \_ -> pure ()+      P.doesDirExist (base </> [reldir|.hledger-flow/cache/snapshots|]) >>= assertBool "created lazily" . not+  , TestLabel "copy selected bytes and translate locations" $ TestCase $ withTestDir $ \base -> do+      let statement = base </> [relfile|account/2026/input.csv|]+          rules = base </> [relfile|account/bank.rules|]+          category = base </> [relfile|shared/coffee.rules|]+          directives = base </> [relfile|directives.journal|]+          a = "account2 expenses:cafe\n"+      P.createDirIfMissing True (parent statement)+      P.createDir (parent category)+      BS.writeFile (toFilePath statement) "2026-01-01,Coffee,3\n"+      BS.writeFile (toFilePath rules) "include ../shared/coffee.rules\n"+      BS.writeFile (toFilePath category) a+      BS.writeFile (toFilePath directives) "account expenses:cafe\n"+      result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (RulesInput statement rules (Just directives) Nothing) $ \snapshot -> do+        RulesInput dataCopy rulesCopy (Just directivesCopy) Nothing <- pure (snapshotInputs snapshot)+        let deps = snapshotManifest snapshot+        assertEqual "manifest includes nested rule" (Just (digestBytes a)) (Map.lookup category deps)+        assertEqual "all four files" 4 (Map.size deps)+        assertEqual "CSV basename" (filename statement) (filename dataCopy)+        canonicalCopy <- D.canonicalizePath (toFilePath rulesCopy)+        let location = ":1:2:\n"+            excerpt = "1 | " <> T.pack (toFilePath rulesCopy) <> "\n"+        assertEqual "maps location, preserves excerpt" ("hledger: Error: " <> T.pack (toFilePath rules) <> location <> excerpt)+           =<< translateDiagnostic snapshot ("hledger: Error: " <> T.pack canonicalCopy <> location <> excerpt)+        let leafCopy = parent (parent rulesCopy) </> [relfile|shared/coffee.rules|]+        pure [dataCopy, rulesCopy, leafCopy, directivesCopy]+      case result of+        Reusable files -> mapM_ (\p -> P.doesFileExist p >>= assertBool "copy released" . not) files+        Fallback reason -> assertFailure (show reason)+  , TestLabel "publish only when converted snapshot matches live inputs" $ TestCase snapshotPublication+  , TestLabel "construct pins an external config" $ TestCase $ withTestDir $ \base -> withTestDir $ \home -> do+      let config = home </> [relfile|hledger.conf|]+      BS.writeFile (toFilePath config) "[balance]\n--period=thismonth\n"+      result <- withSnapshotRun base $ \run -> withSnapshot run V1996 (ConstructInput (Just config)) $ \snapshot -> do+        ConstructInput (Just copied) <- pure (snapshotInputs snapshot)+        assertBool "separate private copy" (config /= copied)+        assertEqual "config path" (T.pack (toFilePath config) <> ":6:3:")+           =<< translateDiagnostic snapshot (T.pack (toFilePath copied) <> ":6:3:")+        let windowsError p = "hledger.exe: Error: " <> T.pack (toFilePath p) <> ":6:3:"+        assertEqual "Windows hledger diagnostic" (windowsError config)+           =<< translateDiagnostic snapshot (windowsError copied)+        BS.readFile (toFilePath copied) >>= assertEqual "whole-file content" "[balance]\n--period=thismonth\n"+      case result of+        Reusable () -> pure ()+        Fallback reason -> assertFailure (show reason)+  , TestLabel "conversion errors propagate and clean up" $ TestCase $ withTestDir $ \base -> do+      result <- try $ withSnapshotRun base $ \run ->+        withSnapshot run V1524 (ConstructInput Nothing) $ \_ -> ioError (userError "converter failed")+      case result :: Either IOException (ReuseDecision ()) of+        Left _ -> pure ()+        Right _ -> assertFailure "swallowed converter error"+      names <- D.listDirectory (toFilePath (base </> [reldir|.hledger-flow/cache/snapshots|]))+      assertBool "only registration lock survives" (length names <= 1)+#ifndef mingw32_HOST_OS+  , TestLabel "canonical snapshot below symlink maps to original spelling" $ TestCase $ withTestDir $ \root -> do+      let base = root </> [reldir|ledger alias|]+          real = root </> [reldir|ledger|]+          config = base </> [relfile|config with spaces|]+      P.createDir real+      D.createDirectoryLink (toFilePath real) (FP.dropTrailingPathSeparator (toFilePath base))+      BS.writeFile (toFilePath config) "# ok\n"+      result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (ConstructInput (Just config)) $ \snap -> do+        ConstructInput (Just copied) <- pure (snapshotInputs snap)+        canonical <- D.canonicalizePath (toFilePath copied)+        assertBool "release canonicalized the ancestor" (canonical /= toFilePath copied)+        let ruleLine fileName = "the date rule is:   %1                   (" <> T.pack fileName <> ":7)"+        let err name = "hledger: Error: " <> T.dropEnd 1 (T.pack (show name)) <> ":2:2:"+            warning name = "Warning: " <> T.pack name <> ": replaced delimiter"+        assertEqual "quoted config diagnostic" (err (toFilePath config)) =<< translateDiagnostic snap (err canonical)+        assertEqual "CSV warning" (warning (toFilePath config)) =<< translateDiagnostic snap (warning canonical)+        assertEqual "diagnostic location" (ruleLine (toFilePath config))+          =<< translateDiagnostic snap (ruleLine canonical)+      assertEqual "converted" (Reusable ()) result+#else+  , TestLabel "260 UTF-16 units in data, include and pinned config" $ TestCase $ withTestDir $ \base -> do+      let csv = base </> [relfile|in.csv|]+          rules = base </> [relfile|in.rules|]+          fixed copied = toFilePath base FP.</> ".hledger-flow" FP.</> "cache" FP.</> "snapshots"+            FP.</> "r00000000" FP.</> "t00000000" FP.</> copied+          runFor part lengthWanted = do+            let folder = if part == "config" then "c" else "l"+                suffix = if part == "CSV" then "😀.csv" else "😀.rules"+                units = sum . map (\c -> if fromEnum c > 65535 then 2 else 1)+                padding = lengthWanted - units (fixed (folder FP.</> suffix))+            assertBool "fixture has room for the filename" (padding >= 0)+            file <- (base </>) <$> parseRelFile (replicate padding 'x' ++ suffix)+            BS.writeFile (toFilePath file) ""+            BS.writeFile (toFilePath csv) "2026-01-01,coffee,3"+            if part == "include"+              then BS.writeFile (toFilePath rules) (encodeUtf8 ("include " <> T.pack (toFilePath (filename file)) <> "\n"))+              else writeFile (toFilePath rules) ""+            let input = case part of+                  "CSV" -> RulesInput file rules Nothing Nothing+                  "include" -> RulesInput csv rules Nothing Nothing+                  _ -> ConstructInput (Just file)+            result <- withSnapshotRun base $ \run -> withSnapshot run V1996 input (const (pure ()))+            case (lengthWanted, result) of+              (259, Reusable ()) -> pure ()+              (260, Fallback reason) -> assertEqual part "overlong Windows snapshot path" (reasonScope reason)+              _ -> assertFailure (part ++ " at " ++ show lengthWanted ++ ": " ++ show result)+      mapM_ (\part -> mapM_ (runFor part) [259,260]) ["CSV","include","config"]+  , TestLabel "short filename diagnostic when available" $ TestCase $ withTestDir $ \base -> do+      let config = base </> [relfile|config containing spaces|]+      BS.writeFile (toFilePath config) "# config\n"+      result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (ConstructInput (Just config)) $ \snap -> do+        ConstructInput (Just copied) <- pure (snapshotInputs snap)+        withCWString (toFilePath copied) $ \long -> allocaArray 260 $ \buffer -> do+          n <- getShortPath long buffer 260+          if n == 0 || n >= 260 then pure () else do+            short <- peekCWString buffer+            assertEqual "resolved file identity" (T.pack (toFilePath config) <> ":4:2:")+              =<< translateDiagnostic snap (T.pack short <> ":4:2:")+      assertEqual "converted" (Reusable ()) result+#endif+  , TestLabel "permission failure makes no copies" $ TestCase $ withTestDir $ \base -> do+      let deny _ = ioError (userError "permission denied")+      result <- withSnapshotRunUsing deny base $ \run ->+        withSnapshot run V1524 (ConstructInput Nothing) $ \_ -> assertFailure "ran a private conversion"+      case result of+        Fallback r -> assertBool "private staging reason" ("private staging unavailable" `T.isPrefixOf` reasonScope r)+        Reusable _ -> assertFailure "admitted unprotected staging"+      D.listDirectory (toFilePath (base </> [reldir|.hledger-flow/cache/snapshots|])) >>= assertEqual "no financial data" ["registration.lock"]+  , TestLabel "registration protects an initializing directory" $ TestCase $ withTestDir $ \base -> do+      created <- newEmptyMVar+      proceed <- newEmptyMVar+      initializer <- newEmptyMVar+      cleaner <- newEmptyMVar+      let suspended dir = createPrivateDirectory dir >> putMVar created () >> takeMVar proceed+      _ <- forkFinally (withSnapshotRunUsing suspended base $ \run ->+        withSnapshot run V1524 (ConstructInput Nothing) (const (pure ()))) (putMVar initializer)+      takeMVar created+      tid <- forkFinally (withSnapshotRun base (const (pure ()))) (putMVar cleaner)+      let blocked = do+            status <- threadStatus tid+            case status of+              ThreadBlocked _ -> pure True+              ThreadFinished -> pure False+              ThreadDied -> pure False+              ThreadRunning -> yield >> blocked+      waiting <- timeout 5000000 blocked+      putMVar proceed ()+      first <- takeMVar initializer >>= either throwIO pure+      takeMVar cleaner >>= either throwIO pure+      assertEqual "cleanup waited for registration" (Just True) waiting+      assertEqual "initializer consumed its own tree" (Reusable ()) first+  , TestLabel "reap killed worker, preserve live and unknown directories" $ TestCase $ withTestDir $ \base -> do+      let config = base </> [relfile|hledger.conf|]+      BS.writeFile (toFilePath config) "# private config\n"+      exe <- getExecutablePath+      let child = (proc exe ["--snapshot-test-worker", toFilePath base]) {std_in = CreatePipe, std_out = CreatePipe}+      result <- withSnapshotRun base $ \active -> withSnapshot active V1524 (ConstructInput (Just config)) $ \snap -> do+        ConstructInput (Just live) <- pure (snapshotInputs snap)+        let unknown = base </> [reldir|.hledger-flow/cache/snapshots/r01234567|]+            keep = unknown </> [relfile|unrelated|]+        P.createDir unknown+        writeFile (toFilePath keep) "preserve"+        withCreateProcess child $ \_ out _ pid -> do+          Just stdoutPipe <- pure out+          copiedName <- hGetLine stdoutPipe+          abandoned <- parseAbsFile copiedName+          terminateProcess pid+          _ <- waitForProcess pid+          P.doesFileExist abandoned >>= assertBool "private data survived unhandled termination"+          withSnapshotRun base (const (pure ()))+          P.doesFileExist abandoned >>= assertBool "abandoned copy reclaimed" . not+          P.doesFileExist keep >>= assertBool "unknown directory preserved"+          BS.readFile (toFilePath live) >>= assertEqual "other run untouched" "# private config\n"+      assertEqual "live run completes" (Reusable ()) result+  , TestLabel "different include spellings cannot overwrite a snapshot file" $ TestCase $ withTestDir $ \base -> do+      let csv = base </> [relfile|input.csv|]+          rules = base </> [relfile|bank.rules|]+          upper = [relfile|A.rules|]+          lower = [relfile|a.rules|]+      BS.writeFile (toFilePath csv) "2026-01-01,Coffee,3\n"+      BS.writeFile (toFilePath rules) "include A.rules\ninclude a.rules\n"+      BS.writeFile (toFilePath (base </> upper)) "account1 assets:bank\n"+      sameFile <- P.doesFileExist (base </> lower)+      BS.writeFile (toFilePath (base </> lower)) "account2 expenses:cafe\n"+      result <- withSnapshotRun base $ \run -> withSnapshot run V1996 (RulesInput csv rules Nothing Nothing) $ \snap -> do+        RulesInput _ staged _ _ <- pure (snapshotInputs snap)+        mapM (BS.readFile . toFilePath . (parent staged </>)) [upper,lower]+      case result of+        Fallback reason -> assertEqual "fallback reason" "snapshot filenames collide" (reasonScope reason)+        Reusable contents -> do+          assertBool "aliased destinations must fall back" (not sameFile)+          assertEqual "distinct copies preserve each set of rules" ["account1 assets:bank\n","account2 expenses:cafe\n"] contents+  , TestLabel "Windows uses UTF-16 units without the NUL" $ TestCase $ do+      assertBool "259" (windowsSnapshotPathFits (replicate 257 'a' ++ "😀"))+      assertBool "260" (not (windowsSnapshotPathFits (replicate 258 'a' ++ "😀")))+  ]++-- Run the real importer with this native test executable as its converter.+-- Its print command changes the live include while consuming the supplied rules.+snapshotPublication :: IO ()+snapshotPublication = withTestDir $ \base -> P.withCurrentDir base $+  bracket getLocaleEncoding setLocaleEncoding $ \_ -> do+    setLocaleEncoding utf8+    exe <- getExecutablePath >>= parseAbsFile+    let opts = (defaultOpts base) {hledgerExecutable = exe, importPolicy = RunScripts}+        root = base </> [reldir|import|]+        source = root </> [relfile|o/b/a/1-in/2026/in.csv|]+        leaf = root </> [relfile|shared/category.rules|]+        a = "account2 expenses:cafe\n"+    P.createDirIfMissing True (parent source)+    P.createDir (parent leaf)+    BS.writeFile (toFilePath source) "2026-01-01,Coffee,3\n"+    BS.writeFile (toFilePath (root </> [relfile|b.rules|])) "include shared/category.rules\n"+    BS.writeFile (toFilePath leaf) a+    BS.writeFile "hledger.conf" ""+    dirs <- either (assertFailure . T.unpack) pure (extractImportDirs root source)+    ch <- newTChanIO+    let run = withIncrementalRun opts ch 1 $ \ctx -> importStatement ctx opts ch dirs source+        journal = "2026-01-01 Coffee\n    expenses:cafe  3\n    assets:bank\n"+    record <- recordPath base source+    BS.writeFile "snapshot-converter-test" "restore"+    (out,_,_) <- run+    readFile (toFilePath out) >>= assertEqual "journal built from copied rule A" journal+    entry <- readRecord record >>= maybe (assertFailure "missing journal record") pure . (>>= journalRecord)+    lookup (T.pack (toFilePath leaf)) (stageInputs entry) @?= Just (digestHex (digestBytes a))+    digestFile out >>= assertEqual "record identifies installed journal bytes" (stageOutput entry)+    -- Force a rebuild with an existing record. Leaving B in place must also+    -- discard the old A record, even though the snapshot produced A again.+    P.removeFile out+    BS.writeFile "snapshot-converter-test" "keep"+    _ <- run+    readFile (toFilePath out) >>= assertEqual "kept journal still built from A" journal+    BS.readFile (toFilePath leaf) >>= assertEqual "live include remains B" "account2 expenses:travel\n"+    readRecord record >>= assertEqual "invalidated journal record" Nothing . (>>= journalRecord)++-- The real test executable owns the lifetime lock. Its parent terminates it+-- abruptly, so no in-process finalizer can remove the abandoned tree.+runWorker :: [String] -> IO Bool+runWorker ["--snapshot-test-worker", baseName] = do+  base <- parseAbsDir baseName+  let config = base </> [relfile|hledger.conf|]+  result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (ConstructInput (Just config)) $ \snapshot -> do+    ConstructInput (Just copied) <- pure (snapshotInputs snapshot)+    putStrLn (toFilePath copied)+    hFlush stdout+    _ <- getLine+    pure ()+  case result of+    Fallback reason -> ioError (userError (show reason))+    Reusable () -> pure True+runWorker args = do+  fixture <- D.doesFileExist "snapshot-converter-test"+  if fixture then fixtureConverter args >> pure True else pure False++fixtureConverter :: [String] -> IO ()+fixtureConverter ["--version"] = putStrLn "hledger 1.99.6"+fixtureConverter ["--conf",_,"print","--explicit","--file",_,"--rules-file",rules] = do+  D.renameDirectory "import/shared" "import/held"+  D.createDirectory "import/shared"+  BS.writeFile "import/shared/category.rules" "account2 expenses:travel\n"+  ["include",relative] <- words . B8.unpack <$> BS.readFile rules+  ["account2",account] <- words . B8.unpack <$> BS.readFile (FP.takeDirectory rules FP.</> relative)+  keep <- (== "keep") <$> BS.readFile "snapshot-converter-test"+  unless keep $ do+    D.removeDirectoryRecursive "import/shared"+    D.renameDirectory "import/held" "import/shared"+  putStr $ "2026-01-01 Coffee\n    " ++ account ++ "  3\n    assets:bank\n"+fixtureConverter args = ioError (userError ("Unexpected converter arguments: " ++ show args))++#ifdef mingw32_HOST_OS+foreign import capi unsafe "windows.h GetShortPathNameW"+  getShortPath :: CWString -> CWString -> Word32 -> IO Word32+#endif
test/ReleaseNotice/Unit.hs view
@@ -4,21 +4,39 @@  import Control.Concurrent.MVar import Control.Exception (AsyncException (UserInterrupt), throwIO, try)-import Control.Monad (forM_)+import Control.Monad (forM_, join) import Data.Aeson (encode, object, (.=))+import qualified Data.ByteString.Lazy as BL import Data.IORef import qualified Data.Text as T+import Data.Time (UTCTime, addUTCTime, getCurrentTime) import Data.Version (makeVersion, showVersion) import Paths_hledger_flow (version)-import Hledger.Flow.ReleaseNotice (ReleaseResult (..), checkRelease, finishCommand, renderUpdate, startCheck)+import Hledger.Flow.Cache (initializeCache)+import Hledger.Flow.PathHelpers (AbsDir)+import Hledger.Flow.ReleaseNotice import Path (toFilePath)-import System.Directory (createDirectoryIfMissing)+import Path.IO (withSystemTempDir)+import System.Directory (createDirectoryIfMissing, doesDirectoryExist, getModificationTime, setModificationTime) import System.Exit (ExitCode (ExitFailure)) import System.FilePath ((</>)) import System.Timeout (timeout) import Test.HUnit import TestHelpers (withTestDir) +-- Exercise a complete, deterministic refresh without the CLI's exit deadline.+checkRelease :: (String -> IO (Maybe BL.ByteString)) -> IO FilePath -> IO ReleaseResult+checkRelease get executable = withSystemTempDir "release-check" $ \base -> do+  initializeCache base+  refreshIfDue base getCurrentTime get executable+  cachedRelease base executable++recordFile :: AbsDir -> FilePath+recordFile base = toFilePath base </> ".hledger-flow/cache/release-check.json"++testTime :: UTCTime+testTime = read "2026-11-16 12:00:00 UTC"+ tests :: Test tests = TestLabel "release notice" $ TestList   [ "invalid or missing responses cannot confirm release status" ~: TestCase $@@ -62,16 +80,64 @@         check releases Nothing         check releases (Just "  tag = \"v999.9.0\"\n")         check "Upgrade with: brew update && brew upgrade ferntrail/tap/hledger-flow" published-  , "command finishes with worker still pending" ~: TestCase $ do-      started <- newEmptyMVar-      release <- newEmptyMVar-      finish <- startCheck (putMVar started () >> takeMVar release)-      takeMVar started-      success <- timeout 1000000 $ finishCommand finish (pure ())-      failure <- timeout 1000000 $ try (finishCommand finish (throwIO (ExitFailure 37)))-      Just () @=? success-      Just (Left (ExitFailure 37)) @=? failure-      putMVar release UpToDate+  , "cached formula advice follows the current executable" ~: TestCase $+      withTestDir $ \base -> do+        let cache = toFilePath base </> ".hledger-flow/cache"+        createDirectoryIfMissing True cache+        writeFile (recordFile base) "{\"lastSuccess\":{\"latestVersion\":\"999\",\"checkedAt\":\"2026-11-16T12:00:00Z\",\"formulaConfirmed\":true}}"+        notice <- cachedRelease base (pure $ toFilePath base </> "hledger-flow")+        UpdateAvailable (makeVersion [999]) "https://github.com/apauley/hledger-flow/releases/latest" @=? notice+  , "missing cache stays absent on read and corrupt JSON can be refreshed" ~: TestCase $+      withTestDir $ \base -> do+        let cache = toFilePath base </> ".hledger-flow/cache"+            readOnly = cachedRelease base (pure "hledger-flow")+        assertEqual "no local notice" CheckUnavailable =<< readOnly+        assertEqual "read-only lookup" False =<< doesDirectoryExist cache+        createDirectoryIfMissing True cache+        writeFile (recordFile base) "partial JSON"+        assertEqual "corrupt cache" CheckUnavailable =<< readOnly+        notice <- join $ prepareCheck base (pure testTime) (const $ pure $ Just "{\"tag_name\":\"v0\"}") (pure "hledger-flow")+        UpToDate @=? notice+        let ignore = cache </> ".gitignore"+        assertEqual "shared ignore file repaired" "*\n" =<< readFile ignore+        setModificationTime ignore testTime+        timestamp <- getModificationTime ignore+        initializeCache base+        assertEqual "incremental initializer preserves ignore mtime" timestamp =<< getModificationTime ignore+  , "refresh intervals and offline fallback" ~: TestCase $ withTestDir $ \base -> do+      calls <- newIORef (0 :: Int)+      let executable = pure (toFilePath base </> "hledger-flow")+          fetch body _ = modifyIORef' calls (+1) >> pure body+          newer = Just "{\"tag_name\":\"v999\"}"+          at seconds = pure (addUTCTime seconds testTime)+          refresh seconds body = refreshIfDue base (at seconds) (fetch body) executable+          notice = UpdateAvailable (makeVersion [999]) "https://github.com/apauley/hledger-flow/releases/latest"+      first <- join $ prepareCheck base (at 0) (fetch newer) executable+      notice @=? first+      assertEqual "fresh result" notice =<< join (prepareCheck base (at 86399) (fetch Nothing) executable)+      assertEqual "only initial request" 1 =<< readIORef calls+      refresh 86400 Nothing+      assertEqual "offline result" notice =<< cachedRelease base executable+      assertEqual "status reads and failure cooldown" notice =<< join (prepareCheck base (at 87299) (fetch newer) executable)+      assertEqual "one failed refresh" 2 =<< readIORef calls+      refresh 87300 (Just "{\"tag_name\":\"v0\"}")+      assertEqual "successful retry" UpToDate =<< cachedRelease base executable+      assertEqual "retry after 15 minutes" 3 =<< readIORef calls+  , "cold command finishes while another refresh is pending" ~: TestCase $+      withTestDir $ \base -> do+        started <- newEmptyMVar+        release <- newEmptyMVar+        calls <- newIORef (0 :: Int)+        let fetch _ = modifyIORef' calls (+1) >> putMVar started () >> takeMVar release+            start = prepareCheck base (pure testTime) fetch (pure "hledger-flow")+        finish <- start+        takeMVar started+        overlapping <- start+        assertEqual "contending check finishes" (Just CheckUnavailable) =<< timeout 1000000 overlapping+        assertEqual "cold wait is bounded" (Just CheckUnavailable) =<< timeout 1000000 finish+        assertEqual "one fetch" 1 =<< readIORef calls+        putMVar release (Just "{\"tag_name\":\"v0\"}")+        assertEqual "finished worker published" UpToDate =<< finish   , "ready notice follows diagnostics and preserves the exit code" ~: TestCase $ do       events <- newIORef ([] :: [String])       let record msg = modifyIORef' events (++ [msg])
test/RuntimeOptions/Unit.hs view
@@ -20,4 +20,4 @@   assertEqual "verbose alone" (True, False, False) (flags opts {verbose = True})   assertEqual "sequential alone" (False, True, False) (flags opts {sequential = True})   assertEqual "pretty reports alone" (False, False, True) (flags opts {prettyReports = True})-  assertEqual "unrelated flags" (False, False, False) (flags opts {onlyNewFiles = True, showOptions = True})+  assertEqual "unrelated flags" (False, False, False) (flags opts {importPolicy = TimestampOnly, showOptions = True})
test/Spec.hs view
@@ -10,6 +10,10 @@ import qualified Common.Unit import qualified Environment.Unit import qualified ImportHelpers.Integration+import qualified Incremental.Cache+import qualified Incremental.Dependencies+import qualified Incremental.Platform+import qualified Incremental.Snapshot import qualified Init.Integration import qualified PathHelpers.Unit import qualified ReleaseNotice.Unit@@ -18,6 +22,8 @@ import qualified Status.Integration import qualified Status.Unit import Test.HUnit+import Control.Monad (unless, when)+import System.Environment (getArgs) import System.Exit (exitFailure)  tests :: Test@@ -32,6 +38,10 @@       CSVImport.Integration.tests,       Status.Unit.tests,       Status.Integration.tests,+      Incremental.Cache.tests,+      Incremental.Dependencies.tests,+      Incremental.Platform.tests,+      Incremental.Snapshot.tests,       Init.Integration.tests,       ReleaseNotice.Unit.tests,       RulesTemplate.Unit.tests,@@ -39,9 +49,9 @@       Environment.Unit.tests     ] -main :: IO Counts+main :: IO () main = do-  errCounts <- runTestTT tests-  if (errors errCounts > 0 || failures errCounts > 0)-    then exitFailure-    else return errCounts+  child <- getArgs >>= Incremental.Snapshot.runWorker+  unless child $ do+    results <- runTestTT tests+    when (errors results > 0 || failures results > 0) exitFailure
test/TestHelpers.hs view
@@ -49,7 +49,7 @@     { baseDir = bd,       importRunDir = [reldir|./|],       importStartYear = Nothing,-      onlyNewFiles = False,+      importPolicy = Full,       hfVersion = versionInfo testSystemInfo,       hledgerExecutable = FlowTypes.hlPath defaultHlInfo,       hledgerVersion = return (FlowTypes.hlVersion defaultHlInfo),