hledger-flow 0.20.0 → 0.20.1
raw patch · 38 files changed
+3115/−496 lines, 38 filesdep +Win32dep +cryptohash-sha256dep +unixPVP: major bump suggested
API removals or changes: PVP suggests a major version bump
Dependencies added: Win32, cryptohash-sha256, unix
API changes (from Hackage documentation)
- Hledger.Flow.Common: descriptiveOutput :: Text -> Text -> Text
- Hledger.Flow.Common: logTimedAction :: HasVerbosity o => o -> TChan LogMessage -> Text -> [Text] -> Text -> [Text] -> (TChan LogMessage -> Text -> IO ()) -> (TChan LogMessage -> Text -> IO ()) -> IO FullOutput -> IO FullTimedOutput
- Hledger.Flow.Common: timeAndExitOnErr :: HasVerbosity o => o -> TChan LogMessage -> Text -> (TChan LogMessage -> Text -> IO ()) -> (TChan LogMessage -> Text -> IO ()) -> ProcInput -> IO FullTimedOutput
- Hledger.Flow.Common: timeAndExitOnErr' :: HasVerbosity o => o -> TChan LogMessage -> Text -> [Text] -> [Text] -> (TChan LogMessage -> Text -> IO ()) -> (TChan LogMessage -> Text -> IO ()) -> ProcInput -> IO FullTimedOutput
- Hledger.Flow.Logging: dummyLogger :: TChan LogMessage -> Text -> IO ()
- Hledger.Flow.Logging: timestampPrefix :: Text -> IO Text
- Hledger.Flow.ReleaseNotice: checkRelease :: (String -> IO (Maybe ByteString)) -> IO FilePath -> IO ReleaseResult
- Hledger.Flow.ReleaseNotice: startCheck :: IO ReleaseResult -> IO (IO ())
- Hledger.Flow.RuntimeOptions: [onlyNewFiles] :: RuntimeOptions -> Bool
- Hledger.Flow.Types: type FullTimedOutput = (FullOutput, NominalDiffTime)
+ Hledger.Flow.Cache: cacheDir :: AbsDir -> AbsDir
+ Hledger.Flow.Cache: initializeCache :: AbsDir -> IO ()
+ Hledger.Flow.Common: External :: Text -> Text -> [[Text]] -> [Text] -> External
+ Hledger.Flow.Common: [extCommand] :: External -> [[Text]]
+ Hledger.Flow.Common: [extHints] :: External -> [Text]
+ Hledger.Flow.Common: [extName] :: External -> Text
+ Hledger.Flow.Common: [extPurpose] :: External -> Text
+ Hledger.Flow.Common: announceExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> IO ()
+ Hledger.Flow.Common: checkExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> Bool -> NominalDiffTime -> FullOutput -> IO ()
+ Hledger.Flow.Common: data External
+ Hledger.Flow.Common: externalCommand :: HasBaseDir o => o -> External -> IO Text
+ Hledger.Flow.Common: findFirstFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> Text -> [AbsFile] -> IO (Maybe AbsFile)
+ Hledger.Flow.Common: firstConfirmedFile :: [FilePath] -> IO (Maybe FilePath)
+ Hledger.Flow.Common: hledgerConfCandidates :: IO [FilePath]
+ Hledger.Flow.Common: lookupHledgerConf :: Maybe FilePath -> IO (Maybe HledgerConf)
+ Hledger.Flow.Common: outputSection :: Bool -> Bool -> Text -> Text -> [Text]
+ Hledger.Flow.Common: pasteCommand :: Shell -> FilePath -> [FilePath] -> [[Text]] -> Text
+ Hledger.Flow.Common: runExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> Bool -> ProcInput -> IO Text
+ Hledger.Flow.Common: showDuration :: NominalDiffTime -> Text
+ Hledger.Flow.Common: writeStagedText :: AbsFile -> Text -> AbsFile -> IO ()
+ Hledger.Flow.DocHelpers: IncrementalImportFallback :: DocLink
+ Hledger.Flow.DocHelpers: Reports :: DocLink
+ Hledger.Flow.DocHelpers: StatusCommand :: DocLink
+ Hledger.Flow.Import.Cache: StageRecord :: StageInputs -> Digest -> StageRecord
+ Hledger.Flow.Import.Cache: StatementRecord :: Maybe StageRecord -> Maybe StageRecord -> StatementRecord
+ Hledger.Flow.Import.Cache: [journalRecord] :: StatementRecord -> Maybe StageRecord
+ Hledger.Flow.Import.Cache: [preprocessRecord] :: StatementRecord -> Maybe StageRecord
+ Hledger.Flow.Import.Cache: [stageInputs] :: StageRecord -> StageInputs
+ Hledger.Flow.Import.Cache: [stageOutput] :: StageRecord -> Digest
+ Hledger.Flow.Import.Cache: data StageRecord
+ Hledger.Flow.Import.Cache: data StatementRecord
+ Hledger.Flow.Import.Cache: digestIfPresent :: AbsFile -> IO (Maybe Digest)
+ Hledger.Flow.Import.Cache: installOutput :: Bool -> AbsFile -> (AbsFile -> IO ()) -> IO Digest
+ Hledger.Flow.Import.Cache: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: instance GHC.Classes.Eq Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance GHC.Classes.Eq Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: instance GHC.Internal.Generics.Generic Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance GHC.Internal.Generics.Generic Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: instance GHC.Internal.Show.Show Hledger.Flow.Import.Cache.StageRecord
+ Hledger.Flow.Import.Cache: instance GHC.Internal.Show.Show Hledger.Flow.Import.Cache.StatementRecord
+ Hledger.Flow.Import.Cache: readRecord :: AbsFile -> IO (Maybe StatementRecord)
+ Hledger.Flow.Import.Cache: recordPath :: AbsDir -> AbsFile -> IO AbsFile
+ Hledger.Flow.Import.Cache: saveRecord :: AbsFile -> StatementRecord -> IO ()
+ Hledger.Flow.Import.Cache: stageMiss :: StageInputs -> AbsFile -> Maybe StageRecord -> IO (Maybe Text)
+ Hledger.Flow.Import.Cache: type StageInputs = [(Text, Text)]
+ Hledger.Flow.Import.Conversion: RulesPaths :: AbsFile -> AbsFile -> Maybe AbsFile -> RulesPaths
+ Hledger.Flow.Import.Conversion: constructArguments :: [Text] -> Text -> [Text]
+ Hledger.Flow.Import.Conversion: constructWith :: RuntimeOptions -> TChan LogMessage -> AbsFile -> [Text] -> (Text -> IO Text) -> AbsFile -> ImportDirs Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: convertRules :: RuntimeOptions -> TChan LogMessage -> AbsFile -> RulesPaths -> RulesPaths -> [Text] -> (Text -> IO Text) -> IO Text
+ Hledger.Flow.Import.Conversion: customConstruct :: RuntimeOptions -> TChan LogMessage -> AbsFile -> ImportDirs Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: data RulesPaths
+ Hledger.Flow.Import.Conversion: findHook :: RuntimeOptions -> TChan LogMessage -> Text -> AbsFile -> IO (Maybe AbsFile)
+ Hledger.Flow.Import.Conversion: hledgerImport :: RuntimeOptions -> TChan LogMessage -> ImportDirs Abs -> AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: hookDirArgs :: ImportDirs b -> [Text]
+ Hledger.Flow.Import.Conversion: logNewFileSkip :: RuntimeOptions -> TChan LogMessage -> Text -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: pinnedConfig :: Maybe Text -> [Text]
+ Hledger.Flow.Import.Conversion: preprocess :: RuntimeOptions -> TChan LogMessage -> AbsFile -> ImportDirs Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.Import.Conversion: rulesArguments :: [Text] -> Text -> Text -> Maybe Text -> [Text]
+ Hledger.Flow.Import.Conversion: savedInput :: RuntimeOptions -> TChan LogMessage -> AbsFile -> AbsFile -> IO AbsFile
+ Hledger.Flow.Import.Dependencies: Converter :: Release -> AbsFile -> FileStamp -> [Text] -> Converter
+ Hledger.Flow.Import.Dependencies: CyclicRules :: Text -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: Fallback :: FallbackReason -> ReuseDecision a
+ Hledger.Flow.Import.Dependencies: FallbackReason :: Text -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: FileReason :: Text -> FilePath -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: OrdinaryInputError :: Text -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: OtherSupported :: Release
+ Hledger.Flow.Import.Dependencies: Reusable :: a -> ReuseDecision a
+ Hledger.Flow.Import.Dependencies: V1524 :: Release
+ Hledger.Flow.Import.Dependencies: V1996 :: Release
+ Hledger.Flow.Import.Dependencies: [converterExecutable] :: Converter -> AbsFile
+ Hledger.Flow.Import.Dependencies: [converterKey] :: Converter -> [Text]
+ Hledger.Flow.Import.Dependencies: [converterRelease] :: Converter -> Release
+ Hledger.Flow.Import.Dependencies: [converterStamp] :: Converter -> FileStamp
+ Hledger.Flow.Import.Dependencies: [reasonDetail] :: FallbackReason -> Text
+ Hledger.Flow.Import.Dependencies: [reasonFile] :: FallbackReason -> FilePath
+ Hledger.Flow.Import.Dependencies: [reasonScope] :: FallbackReason -> Text
+ Hledger.Flow.Import.Dependencies: atPath :: Text -> FilePath -> Text -> ReuseDecision a
+ Hledger.Flow.Import.Dependencies: checkArguments :: [Text] -> ReuseDecision ()
+ Hledger.Flow.Import.Dependencies: checkConfig :: Release -> AbsFile -> ByteString -> ReuseDecision ()
+ Hledger.Flow.Import.Dependencies: checkDirectives :: Release -> AbsFile -> ByteString -> ReuseDecision ()
+ Hledger.Flow.Import.Dependencies: checkedIO :: Text -> FilePath -> IO (ReuseDecision a) -> IO (ReuseDecision a)
+ Hledger.Flow.Import.Dependencies: converterProbeWith :: AbsFile -> (AbsFile -> IO (ExitCode, Text)) -> IO (IO (ReuseDecision Converter))
+ Hledger.Flow.Import.Dependencies: data Converter
+ Hledger.Flow.Import.Dependencies: data FallbackReason
+ Hledger.Flow.Import.Dependencies: data Release
+ Hledger.Flow.Import.Dependencies: data ReuseDecision a
+ Hledger.Flow.Import.Dependencies: discoverConfig :: Release -> Maybe HledgerConf -> IO (ReuseDecision (Maybe (AbsFile, Digest)))
+ Hledger.Flow.Import.Dependencies: discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile, Digest)))
+ Hledger.Flow.Import.Dependencies: discoverRules :: AbsDir -> Release -> AbsFile -> IO (ReuseDecision [(AbsFile, Digest)])
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq Hledger.Flow.Import.Dependencies.Converter
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq Hledger.Flow.Import.Dependencies.FallbackReason
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq Hledger.Flow.Import.Dependencies.Release
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq a => GHC.Classes.Eq (Hledger.Flow.Import.Dependencies.ReuseDecision a)
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Ord Hledger.Flow.Import.Dependencies.FallbackReason
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Base.Applicative Hledger.Flow.Import.Dependencies.ReuseDecision
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Base.Functor Hledger.Flow.Import.Dependencies.ReuseDecision
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Base.Monad Hledger.Flow.Import.Dependencies.ReuseDecision
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show Hledger.Flow.Import.Dependencies.Converter
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show Hledger.Flow.Import.Dependencies.FallbackReason
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show Hledger.Flow.Import.Dependencies.Release
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show a => GHC.Internal.Show.Show (Hledger.Flow.Import.Dependencies.ReuseDecision a)
+ Hledger.Flow.Import.Dependencies: isSeparatedValues :: AbsFile -> Bool
+ Hledger.Flow.Import.Dependencies: nestedRuleIncludes :: Release -> AbsFile -> ByteString -> ReuseDecision [FilePath]
+ Hledger.Flow.Import.Dependencies: newConverterProbe :: RuntimeOptions -> IO (IO (ReuseDecision Converter))
+ Hledger.Flow.Import.Dependencies: resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)
+ Hledger.Flow.Import.Dependencies: ruleIncludes :: Release -> AbsFile -> ByteString -> ReuseDecision [FilePath]
+ Hledger.Flow.Import.Dependencies: selectRules :: AbsFile -> ImportDirs Abs -> IO (ReuseDecision (Maybe AbsFile))
+ Hledger.Flow.Import.Fingerprint: Digest :: Text -> Digest
+ Hledger.Flow.Import.Fingerprint: FileObservation :: !FileStamp -> !Digest -> FileObservation
+ Hledger.Flow.Import.Fingerprint: FileStamp :: !(Integer, Integer) -> !Integer -> !Rational -> !Maybe Rational -> FileStamp
+ Hledger.Flow.Import.Fingerprint: [digestHex] :: Digest -> Text
+ Hledger.Flow.Import.Fingerprint: [fileStamp] :: FileObservation -> !FileStamp
+ Hledger.Flow.Import.Fingerprint: [observationDigest] :: FileObservation -> !Digest
+ Hledger.Flow.Import.Fingerprint: [stampChanged] :: FileStamp -> !Maybe Rational
+ Hledger.Flow.Import.Fingerprint: [stampIdentity] :: FileStamp -> !(Integer, Integer)
+ Hledger.Flow.Import.Fingerprint: [stampModified] :: FileStamp -> !Rational
+ Hledger.Flow.Import.Fingerprint: [stampSize] :: FileStamp -> !Integer
+ Hledger.Flow.Import.Fingerprint: copyDigest :: AbsFile -> AbsFile -> IO Digest
+ Hledger.Flow.Import.Fingerprint: data FileObservation
+ Hledger.Flow.Import.Fingerprint: data FileStamp
+ Hledger.Flow.Import.Fingerprint: digestBytes :: ByteString -> Digest
+ Hledger.Flow.Import.Fingerprint: digestFile :: AbsFile -> IO Digest
+ Hledger.Flow.Import.Fingerprint: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance GHC.Classes.Eq Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance GHC.Classes.Eq Hledger.Flow.Import.Fingerprint.FileObservation
+ Hledger.Flow.Import.Fingerprint: instance GHC.Classes.Eq Hledger.Flow.Import.Fingerprint.FileStamp
+ Hledger.Flow.Import.Fingerprint: instance GHC.Classes.Ord Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance GHC.Internal.Show.Show Hledger.Flow.Import.Fingerprint.Digest
+ Hledger.Flow.Import.Fingerprint: instance GHC.Internal.Show.Show Hledger.Flow.Import.Fingerprint.FileObservation
+ Hledger.Flow.Import.Fingerprint: instance GHC.Internal.Show.Show Hledger.Flow.Import.Fingerprint.FileStamp
+ Hledger.Flow.Import.Fingerprint: newtype Digest
+ Hledger.Flow.Import.Fingerprint: observeFile :: AbsFile -> IO FileObservation
+ Hledger.Flow.Import.Fingerprint: statFile :: AbsFile -> IO FileStamp
+ Hledger.Flow.Import.Incremental: data IncrementalRun
+ Hledger.Flow.Import.Incremental: importStatement :: IncrementalRun -> RuntimeOptions -> TChan LogMessage -> ImportDirs Abs -> AbsFile -> IO (AbsFile, Bool, Bool)
+ Hledger.Flow.Import.Incremental: instance GHC.Classes.Eq Hledger.Flow.Import.Incremental.HookInputs
+ Hledger.Flow.Import.Incremental: instance GHC.Classes.Eq Hledger.Flow.Import.Incremental.JournalInputs
+ Hledger.Flow.Import.Incremental: instance GHC.Internal.Show.Show Hledger.Flow.Import.Incremental.HookInputs
+ Hledger.Flow.Import.Incremental: instance GHC.Internal.Show.Show Hledger.Flow.Import.Incremental.JournalInputs
+ Hledger.Flow.Import.Incremental: withIncrementalRun :: RuntimeOptions -> TChan LogMessage -> Int -> (IncrementalRun -> IO a) -> IO a
+ Hledger.Flow.Import.PrivateDirectory: createPrivateDirectory :: AbsDir -> IO ()
+ Hledger.Flow.Import.Snapshot: ConstructInput :: Maybe AbsFile -> SnapshotInput
+ Hledger.Flow.Import.Snapshot: RulesInput :: AbsFile -> AbsFile -> Maybe AbsFile -> Maybe AbsFile -> SnapshotInput
+ Hledger.Flow.Import.Snapshot: data PreparedSnapshot
+ Hledger.Flow.Import.Snapshot: data SnapshotInput
+ Hledger.Flow.Import.Snapshot: data SnapshotRun
+ Hledger.Flow.Import.Snapshot: instance GHC.Classes.Eq Hledger.Flow.Import.Snapshot.SnapshotInput
+ Hledger.Flow.Import.Snapshot: instance GHC.Internal.Exception.Type.Exception Hledger.Flow.Import.Snapshot.PreparationFailure
+ Hledger.Flow.Import.Snapshot: instance GHC.Internal.Show.Show Hledger.Flow.Import.Snapshot.PreparationFailure
+ Hledger.Flow.Import.Snapshot: instance GHC.Internal.Show.Show Hledger.Flow.Import.Snapshot.SnapshotInput
+ Hledger.Flow.Import.Snapshot: snapshotInputs :: PreparedSnapshot -> SnapshotInput
+ Hledger.Flow.Import.Snapshot: snapshotManifest :: PreparedSnapshot -> Map AbsFile Digest
+ Hledger.Flow.Import.Snapshot: translateDiagnostic :: PreparedSnapshot -> Text -> IO Text
+ Hledger.Flow.Import.Snapshot: windowsSnapshotPathFits :: FilePath -> Bool
+ Hledger.Flow.Import.Snapshot: withSnapshot :: SnapshotRun -> Release -> SnapshotInput -> (PreparedSnapshot -> IO a) -> IO (ReuseDecision a)
+ Hledger.Flow.Import.Snapshot: withSnapshotRun :: AbsDir -> (SnapshotRun -> IO a) -> IO a
+ Hledger.Flow.Import.Snapshot: withSnapshotRunUsing :: (AbsDir -> IO ()) -> AbsDir -> (SnapshotRun -> IO a) -> IO a
+ Hledger.Flow.Logging: bold :: Bool -> Text -> Text
+ Hledger.Flow.Logging: cyan :: Bool -> Text -> Text
+ Hledger.Flow.Logging: dim :: Bool -> Text -> Text
+ Hledger.Flow.Logging: green :: Bool -> Text -> Text
+ Hledger.Flow.Logging: paint :: Bool -> [SGR] -> Text -> Text
+ Hledger.Flow.Logging: red :: Bool -> Text -> Text
+ Hledger.Flow.Logging: resetCode :: Text
+ Hledger.Flow.Logging: synchronous :: SomeException -> Maybe SomeException
+ Hledger.Flow.Logging: useColor :: Handle -> IO Bool
+ Hledger.Flow.Logging: verboseLine :: Text -> IO Text
+ Hledger.Flow.Logging: withLogBlock :: HasVerbosity o => o -> Bool -> TChan LogMessage -> Text -> (TChan LogMessage -> IO a) -> IO a
+ Hledger.Flow.Logging: yellow :: Bool -> Text -> Text
+ Hledger.Flow.ReleaseNotice: cachedRelease :: AbsDir -> IO FilePath -> IO ReleaseResult
+ Hledger.Flow.ReleaseNotice: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReleaseNotice.Observation
+ Hledger.Flow.ReleaseNotice: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReleaseNotice.ReleaseCache
+ Hledger.Flow.ReleaseNotice: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReleaseNotice.Observation
+ Hledger.Flow.ReleaseNotice: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReleaseNotice.ReleaseCache
+ Hledger.Flow.ReleaseNotice: instance GHC.Internal.Generics.Generic Hledger.Flow.ReleaseNotice.Observation
+ Hledger.Flow.ReleaseNotice: instance GHC.Internal.Generics.Generic Hledger.Flow.ReleaseNotice.ReleaseCache
+ Hledger.Flow.ReleaseNotice: prepareCheck :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> IO (IO ReleaseResult)
+ Hledger.Flow.ReleaseNotice: refreshIfDue :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> IO ()
+ Hledger.Flow.ReleaseNotice: withCachedReleaseNotice :: AbsDir -> IO () -> IO ()
+ Hledger.Flow.RuntimeOptions: Full :: ImportPolicy
+ Hledger.Flow.RuntimeOptions: RunScripts :: ImportPolicy
+ Hledger.Flow.RuntimeOptions: SkipUnchangedScripts :: ImportPolicy
+ Hledger.Flow.RuntimeOptions: TimestampOnly :: ImportPolicy
+ Hledger.Flow.RuntimeOptions: [importPolicy] :: RuntimeOptions -> ImportPolicy
+ Hledger.Flow.RuntimeOptions: data ImportPolicy
+ Hledger.Flow.RuntimeOptions: instance GHC.Classes.Eq Hledger.Flow.RuntimeOptions.ImportPolicy
+ Hledger.Flow.RuntimeOptions: instance GHC.Internal.Show.Show Hledger.Flow.RuntimeOptions.ImportPolicy
+ Hledger.Flow.RuntimeOptions: onlyNewFiles :: RuntimeOptions -> Bool
- Hledger.Flow.Common: verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> AbsFile -> IO Bool
+ Hledger.Flow.Common: verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> Text -> AbsFile -> IO Bool
- Hledger.Flow.ReleaseNotice: withReleaseNotice :: IO () -> IO ()
+ Hledger.Flow.ReleaseNotice: withReleaseNotice :: AbsDir -> IO () -> IO ()
- Hledger.Flow.RuntimeOptions: RuntimeOptions :: BaseDir -> RunDir -> Maybe Integer -> Bool -> Text -> AbsFile -> IO Text -> Maybe HledgerConf -> SystemInfo -> Bool -> Bool -> Bool -> Int -> Bool -> RuntimeOptions
+ Hledger.Flow.RuntimeOptions: RuntimeOptions :: BaseDir -> RunDir -> Maybe Integer -> ImportPolicy -> Text -> AbsFile -> IO Text -> Maybe HledgerConf -> SystemInfo -> Bool -> Bool -> Bool -> Int -> Bool -> RuntimeOptions
Files
- ChangeLog.md +22/−2
- README.md +47/−19
- app/Main.hs +126/−69
- cbits/private_directory_darwin.c +24/−0
- cbits/private_directory_windows.c +88/−0
- cbits/sha256_darwin.c +23/−0
- hledger-flow.cabal +38/−2
- site/go-links.json +4/−1
- src/Hledger/Flow/Cache.hs +30/−0
- src/Hledger/Flow/Common.hs +182/−92
- src/Hledger/Flow/DocHelpers.hs +6/−0
- src/Hledger/Flow/Environment.hs +6/−1
- src/Hledger/Flow/Import/CSVImport.hs +41/−158
- src/Hledger/Flow/Import/Cache.hs +100/−0
- src/Hledger/Flow/Import/Conversion.hs +193/−0
- src/Hledger/Flow/Import/Dependencies.hs +360/−0
- src/Hledger/Flow/Import/Fingerprint.hs +136/−0
- src/Hledger/Flow/Import/IncludeFiles.hs +12/−6
- src/Hledger/Flow/Import/Incremental.hs +237/−0
- src/Hledger/Flow/Import/PrivateDirectory.hs +53/−0
- src/Hledger/Flow/Import/Snapshot.hs +329/−0
- src/Hledger/Flow/Logging.hs +57/−17
- src/Hledger/Flow/ReleaseNotice.hs +129/−43
- src/Hledger/Flow/Reports.hs +26/−24
- src/Hledger/Flow/RuntimeOptions.hs +7/−1
- src/Hledger/Flow/Status/Render.hs +2/−19
- src/Hledger/Flow/Types.hs +0/−3
- test/Common/Integration.hs +16/−17
- test/Common/Unit.hs +11/−2
- test/Environment/Unit.hs +3/−0
- test/Incremental/Cache.hs +56/−0
- test/Incremental/Dependencies.hs +185/−0
- test/Incremental/Platform.hs +157/−0
- test/Incremental/Snapshot.hs +313/−0
- test/ReleaseNotice/Unit.hs +79/−13
- test/RuntimeOptions/Unit.hs +1/−1
- test/Spec.hs +15/−5
- test/TestHelpers.hs +1/−1
ChangeLog.md view
@@ -31,13 +31,33 @@ existing destination belongs in release-procedure.md. --> +## 0.20.1++- Use `hledger-flow import --incremental` with hledger 1.52.4 or later to reuse journals when their inputs, including nested rules files, still match. Choose `--incremental=skip-unchanged-scripts` to also reuse the outputs of unchanged scripts. See the [incremental import reference](https://hledger-flow.github.io/reference/incremental-import/).+- Interactive `import` and `report` share a [release check](https://hledger-flow.github.io/reference/release-notices/) for 24 hours. `status` can show a previously discovered upgrade while offline.+- On macOS, incremental import checks the hledger executable faster and still detects a replacement build that reports the same version.+- `hledger-flow import --verbose` and `hledger-flow report --verbose` are+ easier to read. Each statement's lines appear together, files are labelled+ by what import looked for, and every hledger or hook command is shown ready+ to paste into a shell at the base directory, with paths there starting+ with `./`. On a terminal, the output uses colour unless+ `NO_COLOR` is set.+- `--help` is easier to scan. Each command's help lists the global options,+ describes its examples and links to its docs. `import --help` shows each+ `--incremental` mode on its own line. On a terminal, examples and modes use+ colour unless `NO_COLOR` is set.+- When hledger or a preprocess or construct script fails, the message names+ that program, keeps its exit code and marks its output as its own. With+ `--verbose`, the full error output is shown instead of the first 20 lines.+ A construct script's error output now appears once the script finishes.+ ## 0.20.0 - If you have statements with brackets in their filenames, such as `April [1].csv`, run `hledger-flow import` to regenerate their journals. Each now imports exactly once alongside `April 1.csv`.-- If a statement is newer than its cached CSV and its preprocess hook is- missing, import stops and names both files. Restore the hook or move the+- If a statement is newer than its cached CSV and its preprocess script is+ missing, import stops and names both files. Restore the script or move the cached CSV aside, then run `hledger-flow import`. - The documentation has moved to the [project site](https://hledger-flow.github.io/). Links in hledger-flow's messages and `--help` open the relevant section
README.md view
@@ -2,31 +2,59 @@ ## What is it? -hledger-flow organises bank statements by owner, bank, account and year.-It uses [hledger](https://hledger.org/) and its CSV rules to generate one-journal per statement, then includes those journals and your manual entries-in `all-years.journal`. You keep the original statements so you can rebuild-the imported entries when your rules change.+Keep every statement. Rebuild your journals from them. +hledger-flow is a command-line workflow for [hledger](https://hledger.org/),+the plain-text accounting tool. hledger keeps your accounts in journals:+human-readable text files you can open in any editor, in a format that+hledger reads and that+[Ledger](https://hledger-flow.github.io/guides/ledger/) and other plain-text+accounting tools largely share.++hledger-flow is opinionated about where files live. You save each statement,+exactly as your bank issued it, in a folder for its owner, institution,+account and year. `hledger-flow import` converts each one with your hledger+rules, or a script for formats rules can't read, and links the journals+through `all-years.journal`. The statements stay as permanent input, so+every journal can be rebuilt from its source and every entry traced to the+statement it came from.+ It grew out of scripts I wrote while using [adept's Full-fledged Hledger](https://github.com/adept/full-fledged-hledger/wiki). ## Who should use this? -**Choose hledger-flow** if you keep non-overlapping statements and want to-apply better rules to several years of transactions. It is especially useful-when you manage several accounts or owners and want to review the changes-to each statement's journal before committing them.+There are many ways to keep your finances with hledger, and+[hledger's workflows page](https://hledger.org/workflows.html) describes them.+Three of them handle bank statements in quite different ways: -**Choose plain `hledger import`** if you want to append new transactions-and then review, split or annotate them directly in your journal. Your edits-survive the next import. This also suits rolling downloads such as “the last-90 days” from the same account. hledger remembers how far it got for each-source, which works well when dates and the order of same-day transactions-are stable. hledger-flow processes each statement independently, so overlapping-files can count the same purchase twice. You can keep the original CSVs with-either workflow. See [hledger's import guide](https://hledger.org/dev/hledger.html#import).+- **`hledger import`.** Your journal is the primary record. You classify,+ split and annotate entries by editing the journal, and a better rule changes+ future imports only. Overlapping downloads are fine: hledger skips entries+ it has already imported, as long as their dates and same-day order stay+ stable. [How hledger import works](https://hledger.org/hledger.html#import)+- **Read the CSV directly.** The CSV files are the record and nothing is+ generated. You classify by editing the rules, and a better rule applies the+ next time hledger reads the file. Suits a few files you name yourself, as in+ `hledger -f checking.csv bal`. The hledger 1.99 previews can also `include`+ a CSV in a journal. [Reading CSV with hledger](https://hledger.org/hledger.html#csv)+- **hledger-flow.** The original statements are the primary record, and+ journals are rebuilt from them. You classify by editing rules files, and a+ better rule rebuilds every journal that uses it, in every year. A fixed+ folder for each owner, institution, account and year keeps years of+ statements maintainable. An account's statements must not overlap, or a+ purchase appears twice. +Choose `hledger import` if your journal is your primary record and you want+to edit it by hand. Choose hledger-flow if you want to generate your journals+from statements exactly as they came from the source, classify by editing+rules, and keep years of files in a structure that stays maintainable and+auditable.++Full-fledged hledger teaches this last approach with build scripts you can adapt.+hledger-flow packages the folder conventions, import and standard reports as CLI commands.+[Full-fledged hledger’s principles and practices](https://github.com/adept/full-fledged-hledger/wiki/Key-principles-and-practices)+ ## Overview of the Basic Workflow 1. Save each statement under its [account and year](https://hledger-flow.github.io/reference/input-files/).@@ -68,13 +96,13 @@ ## How do I install it? -Install with Homebrew on macOS or Linux:+Install with Homebrew on Linux or macOS: ```sh brew install ferntrail/tap/hledger-flow ``` -Or download a release for Windows, macOS or Linux, or build from source.+Or download a release for Linux, macOS, or Windows, or build from source. The [quick start](https://hledger-flow.github.io/quickstart/) covers each option, how to verify a download, and your first import.
app/Main.hs view
@@ -11,32 +11,43 @@ import Prelude hiding (putStrLn) import qualified Options.Applicative as Opts (command)-import Options.Applicative.Help.Pretty (Doc, pretty, vsep)+import Options.Applicative.Help.Pretty (Color (Cyan), Doc, annotate, bold, color, fillSep, indent, pretty, vsep, (<+>)) import Options.Applicative ( auto,+ eitherReader, argument,- customExecParser,+ execParserPure,+ handleParseResult, headerDoc, helper,+ helpShowGlobals, hsubparser, info, prefs, progDesc,+ renderFailure,+ ParserResult (Failure), footerDoc, hidden,+ internal, optional, Alternative(many, (<|>)), Parser, flag', help,+ helpDoc, long, metavar, option, short, str,+ style, switch )+-- Exposed by optparse-applicative, though not re-exported by Options.Applicative.+import Options.Applicative.Builder.Internal (noGlobal)+import System.Environment (getArgs) -import Hledger.Flow.Types (HledgerInfo (..))+import Hledger.Flow.Types (BaseDir, RunDir, HledgerInfo (..)) import Hledger.Flow.Common ( deferredHledgerVersion, findHledgerConf, hledgerPathFromOption, hledgerInfoFromPath ) import Hledger.Flow.Status ( Situation (..) ) import Hledger.Flow.Status.Gather ( classifyDir )@@ -44,13 +55,14 @@ import qualified Data.Text as Text import Path.IO ( getCurrentDir, resolveDir' ) import Hledger.Flow.Init ( runInit )-import Hledger.Flow.DocHelpers ( DocLink (Docs), docURL, siteURL )+import Hledger.Flow.DocHelpers ( DocLink (..), docURL, siteURL )+import Hledger.Flow.Logging ( useColor ) import Hledger.Flow.RulesTemplate ( rulesTemplateFor ) import Data.Maybe (fromMaybe) import Hledger.Flow.Internals (availableCores, versionInfo, systemInfo) import Hledger.Flow.BaseDir ( determineBaseDir, findBaseDir ) import qualified Hledger.Flow.RuntimeOptions as RT-import Hledger.Flow.ReleaseNotice ( withReleaseNotice )+import Hledger.Flow.ReleaseNotice ( withReleaseNotice, withCachedReleaseNotice ) import Hledger.Flow.Reports ( generateReports ) import Hledger.Flow.Import.CSVImport ( importCSVs ) @@ -66,7 +78,7 @@ data ImportParams = ImportParams { maybeImportBaseDir :: Maybe FilePath , importStartYear :: Maybe StartYear- , onlyNewFiles :: Bool+ , filePolicy :: RT.ImportPolicy } deriving (Show) data ReportParams = ReportParams { maybeReportBaseDir :: Maybe FilePath@@ -123,17 +135,26 @@ runCommand :: IO () runCommand = do+ colour <- useColor stdout let description = pretty ("An hledger workflow focusing on automated statement import and classification:\n" <> Text.unpack siteURL)- cmd <- customExecParser (prefs mempty)- (info (helper <*> baseCommandParser) (headerDoc (Just description) <> footerDoc (Just topExamples)))+ cli = info (helper <*> baseCommandParser colour) (headerDoc (Just description) <> footerDoc (Just (topExamples colour)))+ args <- getArgs+ -- Subcommand help lists the global options. Keep them out of usage errors,+ -- where they would bury the message.+ let parse p = execParserPure (prefs p) cli args+ cmd <- handleParseResult $ case parse helpShowGlobals of+ Failure failure | (_, ExitFailure _) <- renderFailure failure "" -> parse mempty+ result -> result case cmd of Version -> do sysInfo <- systemInfo T.putStrLn $ versionInfo sysInfo - Command mainParams' (Just (Import subParams)) ->- withReleaseNotice $ (toRuntimeOptionsImport mainParams' subParams >>= importCSVs) `catch` reportFatal- Command mainParams' (Just (Report subParams)) ->- withReleaseNotice $ (toRuntimeOptionsReport mainParams' subParams >>= generateReports) `catch` reportFatal+ Command mainParams' (Just (Import subParams)) -> do+ (base, runDir) <- determineBaseDir (maybeImportBaseDir subParams)+ withReleaseNotice base $ (toRuntimeOptionsImport base runDir mainParams' subParams >>= importCSVs) `catch` reportFatal+ Command mainParams' (Just (Report subParams)) -> do+ (base, _) <- determineBaseDir (maybeReportBaseDir subParams)+ withReleaseNotice base $ (toRuntimeOptionsReport base mainParams' subParams >>= generateReports) `catch` reportFatal Command mainParams' (Just (Status subParams)) -> runStatus mainParams' subParams Command mainParams' (Just (Init subParams)) -> do cwd <- getCurrentDir@@ -168,15 +189,19 @@ base <- findBaseDir dir >>= maybe (throwIO (MissingBaseDir dir)) return case statusRulesTemplate params of Just account -> rulesTemplateFor base (Text.pack account) >>= either (ioError . userError . Text.unpack) T.putStr- Nothing -> printStatus (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') (statusJson params) base+ Nothing | statusJson params -> printStatus (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') True base+ | otherwise -> printInteractiveStatus mainParams' base +printInteractiveStatus :: MainParams -> BaseDir -> IO ()+printInteractiveStatus params base = withCachedReleaseNotice base $+ printStatus (hledgerPathOpt params) (hledgerConfOpt params) False base+ -- | Running hledger-flow with no command: say where we are and what to do. runBare :: MainParams -> IO () runBare mainParams' = do situation <- getCurrentDir >>= classifyDir case situation of- InLedger base ->- printStatus (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') False base+ InLedger base -> printInteractiveStatus mainParams' base EmptyDir -> T.putStrLn "This folder is empty. Run `hledger-flow init` to start your ledger here." Elsewhere -> T.putStr $@@ -184,11 +209,9 @@ <> "\nNo hledger-flow ledger here or in any parent folder.\n" <> "Run `hledger-flow init` in an empty folder to start one.\n" -toRuntimeOptionsImport :: MainParams -> ImportParams -> IO RT.RuntimeOptions-toRuntimeOptionsImport mainParams' subParams' = do+toRuntimeOptionsImport :: BaseDir -> RunDir -> MainParams -> ImportParams -> IO RT.RuntimeOptions+toRuntimeOptionsImport bd runDir mainParams' subParams' = do startYear <- traverse resolveStartYear (importStartYear subParams')- let maybeBD = maybeImportBaseDir subParams' :: Maybe FilePath- (bd, runDir) <- determineBaseDir maybeBD hlp <- hledgerPathFromOption $ hledgerPathOpt mainParams' version <- deferredHledgerVersion hlp conf <- findHledgerConf $ hledgerConfOpt mainParams'@@ -196,7 +219,7 @@ return RT.RuntimeOptions { RT.baseDir = bd , RT.importRunDir = runDir , RT.importStartYear = startYear- , RT.onlyNewFiles = onlyNewFiles subParams'+ , RT.importPolicy = filePolicy subParams' , RT.hfVersion = versionInfo sysInfo , RT.hledgerExecutable = hlp , RT.hledgerVersion = version@@ -209,17 +232,15 @@ , RT.prettyReports = True } -toRuntimeOptionsReport :: MainParams -> ReportParams -> IO RT.RuntimeOptions-toRuntimeOptionsReport mainParams' subParams' = do- let maybeBD = maybeReportBaseDir subParams' :: Maybe FilePath- (bd, _) <- determineBaseDir maybeBD+toRuntimeOptionsReport :: BaseDir -> MainParams -> ReportParams -> IO RT.RuntimeOptions+toRuntimeOptionsReport bd mainParams' subParams' = do HledgerInfo hlp version <- hledgerInfoFromPath $ hledgerPathOpt mainParams' conf <- findHledgerConf $ hledgerConfOpt mainParams' sysInfo <- systemInfo return RT.RuntimeOptions { RT.baseDir = bd , RT.importRunDir = [reldir|.|] , RT.importStartYear = Nothing- , RT.onlyNewFiles = False+ , RT.importPolicy = RT.Full , RT.hfVersion = versionInfo sysInfo , RT.hledgerExecutable = hlp , RT.hledgerVersion = return version@@ -232,65 +253,101 @@ , RT.prettyReports = not(asciiReports subParams') } -baseCommandParser :: Parser BaseCommand-baseCommandParser = (Command <$> verboseParser <*> commandParser)- <|> flag' Version (long "version" <> short 'V' <> help "Display version information")+baseCommandParser :: Bool -> Parser BaseCommand+baseCommandParser colour = (Command <$> verboseParser <*> commandParser colour)+ <|> flag' Version (long "version" <> short 'V' <> noGlobal <> help "Display version information.") -commandParser :: Parser (Maybe Command)-commandParser = optional $ hsubparser $- subcommand "status" (Status <$> subcommandParserStatus)- <> subcommand "init" (Init . InitParams <$> optional (argument str (metavar "DIR" <> help "The folder for the new ledger, created if missing. Defaults to the current folder.")))- <> subcommand "import" (Import <$> subcommandParserImport)- <> subcommand "report" (Report <$> subcommandParserReport)+commandParser :: Bool -> Parser (Maybe Command)+commandParser colour = optional $ hsubparser $+ subcommand "status" StatusCommand (Status <$> subcommandParserStatus)+ [ ("hledger-flow status --json", "The same status, for scripts and agents"),+ ("hledger-flow status --rules-template me/mybank/checking", "Print a starter rules file for one account") ]+ <> subcommand "init" GettingStarted (Init . InitParams <$> optional (argument str (metavar "DIR" <> help "The folder for the new ledger, created if missing. Default: the current folder.")))+ [ ("hledger-flow init ~/finances", "Start a ledger in ~/finances") ]+ <> subcommand "import" InputFiles (Import <$> subcommandParserImport colour)+ [ ("hledger-flow import", "Import the current folder"),+ ("hledger-flow import import/me/mybank", "Import one bank's statements"),+ ("hledger-flow import --start-year current", "Import this year only"),+ ("hledger-flow import --incremental", "Rebuild only what changed") ]+ <> subcommand "report" Reports (Report <$> subcommandParserReport)+ [ ("hledger-flow report", "Write reports to the reports folder") ] <> metavar "COMMAND" where- subcommand name parser = Opts.command name (info parser (progDesc (summaryOf name) <> footerDoc (lines' <$> lookup name subcommandExamples)))+ subcommand name link parser examples =+ Opts.command name (info parser (progDesc (summaryOf name) <> footerDoc (Just (examplesDoc colour link examples)))) -lines' :: [String] -> Doc-lines' = vsep . map pretty+-- optparse-applicative writes ANSI styles into help even when stdout is a+-- pipe, so style only after useColor has checked the terminal and NO_COLOR.+literal, heading :: Bool -> Doc -> Doc+literal colour = if colour then annotate (color Cyan) else id+heading colour = if colour then annotate bold else id -topExamples :: Doc-topExamples = lines'- [ "Examples:",- " hledger-flow init Start a ledger in the current folder",- " hledger-flow status See what's there and what to do next",- " hledger-flow import Turn statements into journals",- " hledger-flow report Generate reports",- " hledger-flow status --json The same status, for scripts and agents",- "",- "Docs: " <> Text.unpack (docURL Docs)- ]+-- | Commands with descriptions in a column, or below each command when a+-- column would not fit in 80 characters, then the command's docs link.+examplesDoc :: Bool -> DocLink -> [(String, String)] -> Doc+examplesDoc colour link examples = vsep $+ heading colour (pretty (if length examples == 1 then "Example:" else "Examples:" :: String))+ : concatMap row examples+ ++ [mempty, heading colour (pretty ("Docs:" :: String)) <+> pretty (docURL link)]+ where+ width = maximum (map (length . fst) examples)+ fits = 2 + width + 3 + maximum (map (length . snd) examples) <= 80+ row (cmd, desc)+ | fits = [indent 2 (literal colour (pretty cmd) <> pretty (replicate (width - length cmd + 3) ' ' ++ desc))]+ | otherwise = [indent 2 (literal colour (pretty cmd)), indent 4 (pretty desc)] -subcommandExamples :: [(String, [String])]-subcommandExamples =- [ ("status", ["Examples:", " hledger-flow status --json", " hledger-flow status --rules-template me/mybank/checking"]),- ("init", ["Example:", " hledger-flow init ~/finances"]),- ("import", ["Examples:", " hledger-flow import import/me/mybank", " hledger-flow import --start-year current"]),- ("report", ["Example:", " hledger-flow report"])+topExamples :: Bool -> Doc+topExamples colour = examplesDoc colour Docs+ [ ("hledger-flow init", "Start a ledger in the current folder"),+ ("hledger-flow status", "See what's there and what to do next"),+ ("hledger-flow import", "Turn statements into journals"),+ ("hledger-flow report", "Generate reports"),+ ("hledger-flow status --json", "The same status, for scripts and agents") ] subcommandParserStatus :: Parser StatusParams subcommandParserStatus = StatusParams- <$> optional (argument str (metavar "DIR" <> help "A folder inside the ledger. Defaults to the current folder."))- <*> switch (long "json" <> help "Print the status as JSON, for scripts and agents")- <*> optional (option str (long "rules-template" <> metavar "ACCOUNT" <> help "Print a starter rules file for owner/bank/account, built from its newest statement's header row"))+ <$> optional (argument str (metavar "DIR" <> help "A folder inside the ledger. Default: the current folder."))+ <*> switch (long "json" <> help "Print the status as JSON, for scripts and agents.")+ <*> optional (option str (long "rules-template" <> metavar "ACCOUNT" <> help "Print a starter rules file for owner/bank/account, built from its newest statement's header row.")) verboseParser :: Parser MainParams verboseParser = MainParams- <$> (length <$> many (flag' () (long "verbose" <> short 'v' <> help "Print more verbose output")))- <*> optional (option str (long "hledger-path" <> short 'H' <> metavar "HLEDGER-PATH" <> help "The full path to an hledger executable"))- <*> optional (option str (long "hledger-conf" <> metavar "PATH" <> hidden <> help "The hledger config file to pass to hledger --conf"))- <*> switch (long "show-options" <> hidden <> help "Print the hledger-flow version, hledger path and settings import or report runs with. status always shows them.")+ <$> (length <$> many (flag' () (long "verbose" <> short 'v' <> help "Show each step and the commands hledger-flow runs.")))+ <*> optional (option str (long "hledger-path" <> short 'H' <> metavar "FILE" <> help "Use this hledger executable instead of the one on PATH."))+ <*> optional (option str (long "hledger-conf" <> metavar "FILE" <> hidden <> help "Pass this config file to hledger. Default: the one hledger finds itself."))+ <*> switch (long "show-options" <> hidden <> help "Print the version, hledger path and settings before import or report runs. status always shows them.") <*> optional (option auto (long "batch-size" <> metavar "SIZE" <> hidden <> help "Number of files to process in parallel. Must be positive. Default: the number of cores in use. Ignored during sequential processing."))- <*> switch (long "sequential" <> hidden <> help "Disable parallel processing")+ <*> switch (long "sequential" <> hidden <> help "Disable parallel processing.") -subcommandParserImport :: Parser ImportParams-subcommandParserImport = ImportParams- <$> optional (argument str (metavar "DIR" <> help "The directory to import. Use the base directory for a full import or a sub-directory for a partial import. Defaults to the current directory."))- <*> optional (option startYearReader (long "start-year" <> metavar "YEAR" <> help "Import only from the specified year and onwards, ignoring previous years. By default all available years are imported. Use a 4-digit year or 'current' for the current year"))- <*> switch (long "new-files-only" <> help "Skip regenerating output files that are at least as new as their source. This applies to preprocessed files and hledger journal files.")+subcommandParserImport :: Bool -> Parser ImportParams+subcommandParserImport colour = ImportParams <$> optional (argument str (metavar "DIR" <> help "The folder to import: the ledger root for everything, or a folder inside it, such as an owner, bank or account, for part of it. Default: the current folder."))+ <*> optional (option startYearReader (long "start-year" <> metavar "YEAR" <> help "Import statements from YEAR onwards only. YEAR is a four-digit year or current. Default: all years."))+ <*> filePolicyParser colour +incrementalModes :: [(String, RT.ImportPolicy)]+incrementalModes = [("run-scripts", RT.RunScripts), ("skip-unchanged-scripts", RT.SkipUnchangedScripts)]++filePolicyParser :: Bool -> Parser RT.ImportPolicy+filePolicyParser colour =+ flag' RT.TimestampOnly (long "new-files-only" <> help "Rebuild a file only when its statement or cached CSV is newer. Fast, but misses other changes, such as an edited rules file.")+ <|> flag' RT.RunScripts (long "incremental" <> style (const (pretty ("--incremental[=MODE]" :: String))) <> helpDoc (Just incrementalHelp))+ <|> option (eitherReader readPolicy) (long "incremental" <> internal)+ <|> pure RT.Full+ where+ readPolicy value = maybe (Left ("unknown mode '" ++ value ++ "'. Use run-scripts or skip-unchanged-scripts.")) Right (lookup value incrementalModes)+ para = fillSep . map pretty . words+ mode name = literal colour (pretty (name :: String))+ incrementalHelp = vsep+ [ para "Rebuild only the journals whose inputs changed, comparing file contents. Accounts without scripts behave the same in both modes.",+ mempty,+ mode "run-scripts" <+> pretty ("(default)" :: String),+ indent 2 (para "Run every preprocess and construct script. Reuse a journal when its inputs, including a preprocess script's output, still match."),+ mode "skip-unchanged-scripts",+ indent 2 (para "Also skip a script when the script and its statement are unchanged. Misses edits to other files the script reads, such as a lookup table.")+ ]+ subcommandParserReport :: Parser ReportParams subcommandParserReport = ReportParams- <$> optional (argument str (metavar "BASEDIR" <> help "The hledger-flow base directory"))- <*> switch (long "ascii-reports" <> help "If to avoid using hledger --pretty-tables flag when generating reports.")+ <$> optional (argument str (metavar "DIR" <> help "The ledger folder. Default: the ledger containing the current folder."))+ <*> switch (long "ascii-reports" <> help "Draw report tables with plain ASCII instead of box-drawing characters.")
+ cbits/private_directory_darwin.c view
@@ -0,0 +1,24 @@+#include <stddef.h>+#include <errno.h>+#include <fcntl.h>+#include <sys/stat.h>++/* Darwin ACL grants can authorize access independently of the Unix mode. */+int hf_verify_private_acl(int fd)+{+ filesec_t security = filesec_init();+ if (security == NULL)+ return -1;+ struct stat status;+ int has_acl = 0;+ int result = fstatx_np(fd, &status, security);+ if (result == 0)+ result = filesec_query_property(security, FILESEC_ACL, &has_acl);+ int error = errno;+ filesec_free(security);+ if (result != 0 || has_acl) {+ errno = result != 0 ? error : EACCES;+ return -1;+ }+ return 0;+}
+ cbits/private_directory_windows.c view
@@ -0,0 +1,88 @@+#ifndef _WIN32_WINNT+#define _WIN32_WINNT 0x0600+#endif+#include <windows.h>+#include <sddl.h>+#include <stdlib.h>+#include <wchar.h>++/* Check the volume of the newly created directory before any copies are made. */+static DWORD check_persistent_acls(LPCWSTR path)+{+ DWORD flags = 0;+ HANDLE dir = CreateFileW(path, FILE_READ_ATTRIBUTES,+ FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE,+ NULL, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT, NULL);+ if (dir == INVALID_HANDLE_VALUE)+ return GetLastError();++ BOOL ok = GetVolumeInformationByHandleW(dir, NULL, 0, NULL, NULL, &flags, NULL, 0);+ DWORD error = ok ? ERROR_SUCCESS : GetLastError();+ CloseHandle(dir);+ if (error != ERROR_SUCCESS)+ return error;+ return (flags & FILE_PERSISTENT_ACLS) ? ERROR_SUCCESS : ERROR_NOT_SUPPORTED;+}++DWORD hf_create_private_directory(LPCWSTR path)+{+ DWORD error;++ HANDLE token = NULL;+ TOKEN_USER *user = NULL;+ LPWSTR sid = NULL;+ LPWSTR sddl = NULL;+ PSECURITY_DESCRIPTOR descriptor = NULL;+ SECURITY_ATTRIBUTES attributes = {sizeof(SECURITY_ATTRIBUTES), NULL, FALSE};+ DWORD size = 0;+ const WCHAR prefix[] = L"D:P(A;OICI;FA;;;";++ if (!OpenProcessToken(GetCurrentProcess(), TOKEN_QUERY, &token))+ return GetLastError();++ GetTokenInformation(token, TokenUser, NULL, 0, &size);+ error = GetLastError();+ if (error != ERROR_INSUFFICIENT_BUFFER)+ goto done;+ user = malloc(size);+ if (user == NULL) {+ error = ERROR_OUTOFMEMORY;+ goto done;+ }+ if (!GetTokenInformation(token, TokenUser, user, size, &size) ||+ !ConvertSidToStringSidW(user->User.Sid, &sid)) {+ error = GetLastError();+ goto done;+ }+ sddl = malloc(sizeof(WCHAR) * (wcslen(prefix) + wcslen(sid) + 2));+ if (sddl == NULL) {+ error = ERROR_OUTOFMEMORY;+ goto done;+ }+ wcscpy(sddl, prefix);+ wcscat(sddl, sid);+ wcscat(sddl, L")");+ if (!ConvertStringSecurityDescriptorToSecurityDescriptorW(+ sddl, SDDL_REVISION_1, &descriptor, NULL)) {+ error = GetLastError();+ goto done;+ }+ attributes.lpSecurityDescriptor = descriptor;+ if (CreateDirectoryW(path, &attributes)) {+ error = check_persistent_acls(path);+ if (error != ERROR_SUCCESS)+ RemoveDirectoryW(path);+ } else {+ error = GetLastError();+ }++done:+ if (descriptor != NULL)+ LocalFree(descriptor);+ free(sddl);+ if (sid != NULL)+ LocalFree(sid);+ free(user);+ CloseHandle(token);+ return error;+}
+ cbits/sha256_darwin.c view
@@ -0,0 +1,23 @@+#include <CommonCrypto/CommonDigest.h>+#include <errno.h>+#include <unistd.h>++/* Read the already-open input with a bounded buffer. CommonCrypto selects+ the SHA-256 implementation for the running Mac. */+int hf_sha256_fd(int fd, unsigned char digest[CC_SHA256_DIGEST_LENGTH])+{+ CC_SHA256_CTX ctx;+ unsigned char buffer[64 * 1024];+ CC_SHA256_Init(&ctx);+ for (;;) {+ ssize_t count = read(fd, buffer, sizeof(buffer));+ if (count > 0) {+ CC_SHA256_Update(&ctx, buffer, (CC_LONG)count);+ } else if (count == 0) {+ CC_SHA256_Final(digest, &ctx);+ return 0;+ } else if (errno != EINTR) {+ return -1;+ }+ }+}
hledger-flow.cabal view
@@ -5,9 +5,13 @@ -- see: https://github.com/sol/hpack name: hledger-flow-version: 0.20.0+version: 0.20.1 synopsis: An hledger workflow focusing on automated statement import and classification.-description: Documentation and a quick start are at <https://hledger-flow.github.io/>+description: hledger-flow uses hledger's CSV rules to turn each bank statement into a+ separate journal. It combines those journals with your manual entries for+ reporting. When your rules change, you can rebuild the journals from the+ original statements and review which transactions changed.+ See <https://hledger-flow.github.io/> for the quick start and tutorial. category: Finance, Console homepage: https://github.com/apauley/hledger-flow#readme bug-reports: https://github.com/apauley/hledger-flow/issues@@ -29,14 +33,22 @@ library exposed-modules: Hledger.Flow.BaseDir+ Hledger.Flow.Cache Hledger.Flow.Common Hledger.Flow.DateTime Hledger.Flow.DocHelpers Hledger.Flow.Environment+ Hledger.Flow.Import.Cache+ Hledger.Flow.Import.Conversion Hledger.Flow.Import.CSVImport+ Hledger.Flow.Import.Dependencies+ Hledger.Flow.Import.Fingerprint Hledger.Flow.Import.ImportHelpers Hledger.Flow.Import.IncludeFiles+ Hledger.Flow.Import.Incremental Hledger.Flow.Import.Inventory+ Hledger.Flow.Import.PrivateDirectory+ Hledger.Flow.Import.Snapshot Hledger.Flow.Import.Types Hledger.Flow.Init Hledger.Flow.Internals@@ -62,6 +74,7 @@ , base >=4.20 && <4.22 , bytestring >=0.12.2.0 && <0.13 , containers ==0.7.*+ , cryptohash-sha256 ==0.11.* , directory >=1.3.8.5 && <1.4 , exceptions >=0.10.9 && <0.11 , filepath >=1.5.4.0 && <1.6@@ -75,6 +88,20 @@ , text >=2.1.3 && <2.2 , time >=1.12.2 && <1.13 || >=1.14 && <1.15 default-language: Haskell2010+ if os(darwin)+ c-sources:+ cbits/private_directory_darwin.c+ cbits/sha256_darwin.c+ if os(windows)+ c-sources:+ cbits/private_directory_windows.c+ extra-libraries:+ advapi32+ build-depends:+ Win32 ==2.14.*+ else+ build-depends:+ unix ==2.8.* executable hledger-flow main-is: Main.hs@@ -110,6 +137,10 @@ CSVImport.Unit Environment.Unit ImportHelpers.Integration+ Incremental.Cache+ Incremental.Dependencies+ Incremental.Platform+ Incremental.Snapshot Init.Integration PathHelpers.Unit ReleaseNotice.Unit@@ -126,13 +157,18 @@ HUnit >=1.6.2.0 && <1.7 , aeson >=2.2.5.1 && <2.3 , base >=4.20 && <4.22+ , bytestring >=0.12.2.0 && <0.13 , containers ==0.7.* , directory >=1.3.8.5 && <1.4 , filepath >=1.5.4.0 && <1.6 , hledger-flow , path >=0.9.6 && <0.10 , path-io >=1.8.2 && <1.9+ , process >=1.6.26.1 && <1.7 , stm >=2.5.3.1 && <2.6 , text >=2.1.3 && <2.2 , time >=1.12.2 && <1.13 || >=1.14 && <1.15 default-language: Haskell2010+ if !os(windows)+ build-depends:+ unix ==2.8.*
site/go-links.json view
@@ -2,7 +2,10 @@ "docs": "/quickstart/", "getting-started": "/quickstart/#import-your-first-statement", "input-files": "/reference/input-files/",+ "incremental-import-fallback": "/reference/incremental-import/#when-reuse-is-unavailable", "rules-files": "/reference/rules-files/", "preprocess-script": "/reference/preprocess/",- "set-ledger-file": "/quickstart/#import-your-first-statement"+ "set-ledger-file": "/quickstart/#import-your-first-statement",+ "status": "/reference/status/",+ "reports": "/tutorial/part-2/#reports" }
+ src/Hledger/Flow/Cache.hs view
@@ -0,0 +1,30 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Hledger.Flow.Cache (cacheDir, initializeCache) where++import Control.Monad (unless)+import qualified Data.ByteString as BS+import GHC.IO.Handle.Lock (LockMode (ExclusiveLock), hLock)+import Hledger.Flow.PathHelpers (AbsDir, withStagedOutput)+import qualified Path as P+import qualified Path.IO as P+import System.IO (IOMode (AppendMode), withBinaryFile)++cacheDir :: AbsDir -> AbsDir+cacheDir base = base P.</> [P.reldir|.hledger-flow/cache|]++initializeCache :: AbsDir -> IO ()+initializeCache base = do+ let cache = cacheDir base+ ignore = cache P.</> [P.relfile|.gitignore|]+ lock = cache P.</> [P.relfile|initialize.lock|]+ exists <- P.doesFileExist ignore+ unless exists $ do+ P.createDirIfMissing True cache+ -- Serialize first use across processes. Closing the handle releases the lock.+ withBinaryFile (P.toFilePath lock) AppendMode $ \handle -> do+ hLock handle ExclusiveLock+ initialized <- P.doesFileExist ignore+ unless initialized $ withStagedOutput ignore $ \stage ->+ BS.writeFile (P.toFilePath stage) "*\n"
src/Hledger/Flow/Common.hs view
@@ -11,7 +11,10 @@ import Control.Concurrent.STM import Data.Char (isAlphaNum, isAscii, isDigit) import Data.Either-import qualified Data.List as List (isPrefixOf, isSuffixOf, nub, null)+import Data.Maybe (fromMaybe)+import Data.Time.Clock (NominalDiffTime)+import Text.Printf (printf)+import qualified Data.List as List (take, isPrefixOf, isSuffixOf, nub) import qualified Data.Map.Strict as Map import qualified Data.Text as T import qualified Data.Text.IO as T@@ -33,7 +36,7 @@ import System.IO (hClose) import qualified System.Info as Sys import qualified System.Process as Process-import System.IO.Error (tryIOError)+import System.IO.Error (isDoesNotExistError, tryIOError) import System.FilePath ((<.>)) import Prelude hiding (putStrLn, readFile, writeFile) @@ -139,20 +142,43 @@ -- by itself, like possibleConfFiles in hledger's Hledger.Cli.Conf: hledger.conf in the current -- directory or a parent, ~/.hledger.conf, or hledger.conf in the XDG config directory. findHledgerConf :: Maybe FilePath -> IO (Maybe HledgerConf)-findHledgerConf (Just given) = return (Just (GivenConf given))+findHledgerConf (Just given) = pure (Just (GivenConf given)) findHledgerConf Nothing = do- found <- tryIOError $ do- here <- Directory.getCurrentDirectory- home <- Directory.getHomeDirectory- xdg <- Directory.getXdgDirectory Directory.XdgConfig "hledger"- let file d = d FilePath.</> if d == home then ".hledger.conf" else "hledger.conf"- filterM Directory.doesFileExist (map file (List.nub (upToRoot here ++ [home, xdg])))- return $ case found of+ result <- tryIOError (hledgerConfCandidates >>= filterM Directory.doesFileExist)+ pure $ case result of Right (f : _) -> Just (FoundConf f) _ -> Nothing++-- | Fresh lookup for callers that need to distinguish absence from failed discovery.+lookupHledgerConf :: Maybe FilePath -> IO (Maybe HledgerConf)+lookupHledgerConf (Just given) = return (Just (GivenConf given))+lookupHledgerConf Nothing = fmap FoundConf <$> (hledgerConfCandidates >>= firstConfirmedFile)++hledgerConfCandidates :: IO [FilePath]+hledgerConfCandidates = do+ here <- Directory.getCurrentDirectory+ home <- Directory.getHomeDirectory+ xdg <- Directory.getXdgDirectory Directory.XdgConfig "hledger"+ let file d = d FilePath.</> if d == home then ".hledger.conf" else "hledger.conf"+ pure (map file (List.nub (List.take 100 (upToRoot here) ++ [home, xdg]))) where upToRoot d = let up = FilePath.takeDirectory d in d : if up == d then [] else upToRoot up +-- doesFileExist also returns False on permission errors. Verify a negative result+-- before advancing to the next candidate so an uncertain selection cannot be cached.+firstConfirmedFile :: [FilePath] -> IO (Maybe FilePath)+firstConfirmedFile [] = pure Nothing+firstConfirmedFile (file : rest) = do+ exists <- Directory.doesFileExist file+ if exists then pure (Just file) else do+ status <- tryIOError (Directory.getPermissions file)+ case status of+ Left e | isDoesNotExistError e -> firstConfirmedFile rest+ | otherwise -> ioError e+ Right _ -> do+ dir <- Directory.doesDirectoryExist file+ if dir then firstConfirmedFile rest else ioError (userError ("Could not determine the file type of " ++ file))+ showCmdArgs :: [T.Text] -> T.Text showCmdArgs args = T.intercalate " " (map escapeArg args) @@ -161,8 +187,14 @@ homeDirs :: IO [FilePath] homeDirs = either (const []) id <$> tryIOError (do home <- Directory.getHomeDirectory- real <- Directory.canonicalizePath home- return (List.nub [home, real]))+ -- PowerShell expands $HOME from USERPROFILE. The Windows profile API can+ -- still return the original profile after USERPROFILE has been overridden.+ shellHome <- if nativeShell == PowerShell then lookupEnv "USERPROFILE" else pure (Just home)+ case shellHome of+ Just value | FilePath.equalFilePath value home -> do+ real <- Directory.canonicalizePath home+ pure (List.nub [home, real])+ _ -> pure []) -- | The path's components below the home folder; @Just []@ for the home folder itself. belowHome :: [FilePath] -> FilePath -> Maybe [FilePath]@@ -237,35 +269,121 @@ logFun errorMessage exitWith (ExitFailure exitStatus) -descriptiveOutput :: T.Text -> T.Text -> T.Text-descriptiveOutput outputLabel outTxt = do- if not (T.null outTxt)- then outputLabel <> ":\n" <> outTxt <> "\n"- else ""+-- | An external program hledger-flow runs, described for the people reading its output.+data External = External+ { -- | Who printed the output and who failed: @hledger@, or a hook script's path.+ extName :: T.Text,+ -- | What it runs for, after "to": @convert import/…/a.csv with …@.+ extPurpose :: T.Text,+ -- | The command line to show, as one or more commands piped together.+ extCommand :: [[T.Text]],+ -- | What to check when it fails.+ extHints :: [T.Text]+ } -logTimedAction ::- (HasVerbosity o) =>- o ->- TChan LogMessage ->- T.Text ->- [T.Text] ->- T.Text ->- [T.Text] ->- (TChan LogMessage -> T.Text -> IO ()) ->- (TChan LogMessage -> T.Text -> IO ()) ->- IO FullOutput ->- IO FullTimedOutput-logTimedAction opts ch cmdLabel extraCmdLabels cmd args stdoutLogger stderrLogger action = do- logVerbose opts ch $ "Begin: " <> cmdLabel+-- | A command line to paste into this platform's shell from the given directory: paths below it+-- start with @./@, and other paths under the home folder with @~@ or @$HOME@.+pasteCommand :: Shell -> FilePath -> [FilePath] -> [[T.Text]] -> T.Text+pasteCommand shell base homes = T.intercalate " | " . map command+ where+ command argv = case shell of+ Posix -> T.unwords (map posixArg argv)+ PowerShell -> case map powerShellArg argv of+ -- PowerShell reads a quoted first word as a string, not a command to run.+ first : rest | "'" `T.isPrefixOf` first || "\"" `T.isPrefixOf` first -> T.unwords ("&" : first : rest)+ ws -> T.unwords ws+ belowBase a = case belowHome [base] (T.unpack a) of+ Just [] -> Just "."+ Just rest -> Just (T.pack ('.' : FilePath.pathSeparator : FilePath.joinPath rest))+ Nothing -> Nothing+ posixArg a = maybe (showCmdArgsAt homes [a]) escapeArg (belowBase a)+ powerShellArg a = case belowBase a of+ Just rel -> shellArgument PowerShell rel+ Nothing -> fromMaybe (shellArgument PowerShell a) (homeShellPath PowerShell homes (T.unpack a))++-- hledger discovers config files from its working directory, and scripts can read+-- relative paths. Preserve that directory when replaying a command at the ledger root.+externalCommand :: HasBaseDir o => o -> External -> IO T.Text+externalCommand opts ext = do homes <- homeDirs- logVerbose opts ch $ showCmdArgsAt homes (cmd : args)- if (List.null extraCmdLabels) then return () else logVerbose opts ch $ T.intercalate "\n" extraCmdLabels- timed@((ec, stdOut, stdErr), diff) <- timeAction action- stdoutLogger ch stdOut- stderrLogger ch stdErr- logVerbose opts ch $ "End: " <> cmdLabel <> " " <> T.pack (show ec) <> " (" <> T.pack (show diff) <> ")"- return timed+ cwd <- Directory.getCurrentDirectory+ let command = pasteCommand nativeShell cwd homes (extCommand ext)+ location = fromMaybe (shellArgument nativeShell (T.pack cwd)) (homeShellPath nativeShell homes cwd)+ pure $ if FilePath.equalFilePath cwd (toFilePath (baseDir opts))+ then command+ else case nativeShell of+ Posix -> "(cd " <> location <> " && " <> command <> ")"+ PowerShell -> "& { Push-Location -LiteralPath " <> location <> " -ErrorAction Stop; try { "+ <> command <> " } finally { Pop-Location } }" +-- | In verbose mode, say which program is about to run and show its command line.+announceExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> IO ()+announceExternal opts ch ext = when (verbose opts) $ do+ command <- externalCommand opts ext+ color <- useColor H.stderr+ logToChannel ch $+ "Running " <> extName ext <> " to " <> extPurpose ext <> ":\n"+ <> " " <> dim color "$" <> " " <> cyan color command++-- | Run an external program, show what it printed, and stop with its exit code when it fails.+-- Returns its standard output, which goes to the console only when 'showStdout' is set.+runExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> Bool -> ProcInput -> IO T.Text+runExternal opts ch ext showStdout input = do+ announceExternal opts ch ext+ (result@(_, out, _), duration) <- timeAction (runProcessCaptured input)+ checkExternal opts ch ext showStdout duration result+ return out++-- | Show what an external program printed. When it failed, stop with its exit code and+-- say plainly that the error came from that program.+checkExternal :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> External -> Bool -> NominalDiffTime -> FullOutput -> IO ()+checkExternal opts ch ext showStdout duration (status, out, err) = do+ outColor <- useColor H.stdout+ errColor <- useColor H.stderr+ case status of+ ExitSuccess -> do+ logVerbose opts ch (green errColor (extName ext <> " finished") <> " in " <> showDuration duration)+ when showStdout $ passOn (channelOut ch) outColor "Output" out+ passOn (channelErr ch) errColor "Messages" err+ ExitFailure code -> do+ when showStdout $ passOn (channelOut ch) outColor "Output" out+ commandLines <- if verbose opts then pure [] else do+ command <- externalCommand opts ext+ pure ["Command:", " " <> command]+ let section label txt = outputSection errColor (verbose opts) (label <> " from " <> extName ext) txt+ errExit code ch $ T.intercalate "\n" $+ [ red errColor (bold errColor (extName ext <> " failed with exit code " <> T.pack (show code)))+ <> " while trying to " <> extPurpose ext <> "."+ ]+ -- Verbose output showed the command just before it ran.+ ++ commandLines+ ++ (if showStdout then [] else section "Output" out)+ ++ section "Error output" err+ ++ extHints ext+ where+ -- In verbose mode, mark the output as the program's; otherwise pass it on as it is.+ passOn write color label txt+ | T.null txt = return ()+ | verbose opts = case outputSection color True (label <> " from " <> extName ext) txt of+ heading : lns -> logToChannel ch heading >> write (T.unlines lns)+ [] -> return ()+ | otherwise = write txt++-- | Another program's output under a label, each line behind a bar. Without 'complete',+-- long output is trimmed.+outputSection :: Bool -> Bool -> T.Text -> T.Text -> [T.Text]+outputSection color complete label txt+ | T.null (T.strip txt) = []+ | otherwise = (label <> ":") : map bar (T.lines (if complete then txt else trimToolOutput txt))+ where+ -- Keep the program's own colours from spilling into the lines that follow.+ bar l = dim color " │" <> " " <> l <> (if color && "\ESC" `T.isInfixOf` l then resetCode else "")++showDuration :: NominalDiffTime -> T.Text+showDuration d+ | d < 1 = T.pack (show (round (d * 1000) :: Integer)) <> " ms"+ | otherwise = T.pack (printf "%.1f s" (realToFrac d :: Double))+ -- Drain both output pipes concurrently so a full stderr pipe cannot stall stdout. runProcessCaptured :: ProcInput -> IO FullOutput runProcessCaptured (cmd, args, input) = do@@ -285,47 +403,6 @@ status <- Process.waitForProcess handle return (status, stdoutText, stderrText) -timeAndExitOnErr ::- (HasVerbosity o) =>- o ->- TChan LogMessage ->- T.Text ->- (TChan LogMessage -> T.Text -> IO ()) ->- (TChan LogMessage -> T.Text -> IO ()) ->- ProcInput ->- IO FullTimedOutput-timeAndExitOnErr opts ch cmdLabel = timeAndExitOnErr' opts ch cmdLabel [] []--timeAndExitOnErr' ::- (HasVerbosity o) =>- o ->- TChan LogMessage ->- T.Text ->- [T.Text] ->- [T.Text] ->- (TChan LogMessage -> T.Text -> IO ()) ->- (TChan LogMessage -> T.Text -> IO ()) ->- ProcInput ->- IO FullTimedOutput-timeAndExitOnErr' opts ch cmdLabel extraCmdLabels hints stdoutLogger stderrLogger input@(cmd, args, _) = do- let action = runProcessCaptured input- -- On failure the error output goes into the (trimmed) message, not straight to the console.- timed@((ec, stdOut, stdErr), _) <- logTimedAction opts ch cmdLabel extraCmdLabels cmd args stdoutLogger (\_ _ -> return ()) action- case ec of- ExitFailure i -> do- homes <- homeDirs- let cmdText = showCmdArgsAt homes (cmd : args)- let msgOut = descriptiveOutput "Standard output" (trimToolOutput stdOut)- let msgErr = descriptiveOutput "Error output" (trimToolOutput stdErr)- let exitMsg =- "Failed " <> cmdLabel <> " (exit code " <> T.pack (show i) <> ").\n"- <> "Command: " <> cmdText <> "\n" <> msgOut <> msgErr- <> T.unlines hints- errExit i ch exitMsg- ExitSuccess -> do- stderrLogger ch stdErr- return timed- -- | Keep a tool's output readable: at most 20 lines of at most 300 characters. -- hledger can echo a whole statement on one line when parsing fails. trimToolOutput :: T.Text -> T.Text@@ -338,7 +415,7 @@ maxWidth = 300 cut l = if T.length l > maxWidth then T.take maxWidth l <> "…" else l dropped = case length ls - maxLines of- n | n > 0 -> ["… (" <> T.pack (show n) <> " more lines)"]+ n | n > 0 -> ["… (" <> T.pack (show n) <> " more lines; run with --verbose to see them all)"] _ -> [] parAwareActions :: (HasSequential o, HasBatchSize o) => o -> [IO a] -> IO [a]@@ -350,15 +427,25 @@ | null (drop batch todo) = (done ++) <$> mapConcurrently id todo | otherwise = (done ++) <$> mapConcurrentlyBounded batch id todo -verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> AbsFile -> IO Bool-verboseTestFile opts ch p = do+-- | Whether the file exists, saying in verbose mode what was looked for and where.+verboseTestFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> T.Text -> AbsFile -> IO Bool+verboseTestFile opts ch what p = do fileExists <- Path.doesFileExist p+ color <- useColor H.stderr let rel = T.pack (relativeToBase opts (toFilePath p))- if fileExists- then logVerbose opts ch ("Found '" <> rel <> "'")- else logVerbose opts ch ("Looked for but did not find '" <> rel <> "'")+ logVerbose opts ch $+ if fileExists+ then green color ("Found " <> what) <> ": " <> rel+ else "No " <> what <> " at " <> rel return fileExists +-- | The first of these files that exists, saying in verbose mode which ones were looked for.+findFirstFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> T.Text -> [AbsFile] -> IO (Maybe AbsFile)+findFirstFile _ _ _ [] = return Nothing+findFirstFile opts ch what (file : rest) = do+ exists <- verboseTestFile opts ch what file+ if exists then return (Just file) else findFirstFile opts ch what rest+ needsRegeneration :: AbsFile -> AbsFile -> IO Bool needsRegeneration src target = do -- A missing or unreadable timestamp also requires regeneration.@@ -422,13 +509,16 @@ writeGeneratedText path contents = do -- Replace the file behind an existing symlink and keep its permissions. target <- Directory.canonicalizePath path >>= Path.parseAbsFile- withStagedOutput target $ \staged -> do- let tmp = toFilePath staged- exists <- Path.doesFileExist target- when exists $ do- T.writeFile tmp ""- Directory.copyPermissions (toFilePath target) tmp- T.writeFile tmp contents+ withStagedOutput target (writeStagedText target contents)++writeStagedText :: AbsFile -> T.Text -> AbsFile -> IO ()+writeStagedText target contents staged = do+ let tmp = toFilePath staged+ exists <- Path.doesFileExist target+ when exists $ do+ T.writeFile tmp ""+ Directory.copyPermissions (toFilePath target) tmp+ T.writeFile tmp contents textFileNeedsUpdate :: FilePath -> T.Text -> IO Bool textFileNeedsUpdate path contents = either (const True) (/= contents) <$> tryIOError (T.readFile path)
src/Hledger/Flow/DocHelpers.hs view
@@ -18,19 +18,25 @@ data DocLink = Docs | GettingStarted+ | IncrementalImportFallback | InputFiles | RulesFiles | PreprocessScript | SetLedgerFile+ | StatusCommand+ | Reports deriving (Bounded, Enum, Eq, Show) docSlug :: DocLink -> Text docSlug Docs = "docs" docSlug GettingStarted = "getting-started"+docSlug IncrementalImportFallback = "incremental-import-fallback" docSlug InputFiles = "input-files" docSlug RulesFiles = "rules-files" docSlug PreprocessScript = "preprocess-script" docSlug SetLedgerFile = "set-ledger-file"+docSlug StatusCommand = "status"+docSlug Reports = "reports" siteURL :: Text siteURL = "https://hledger-flow.github.io/"
src/Hledger/Flow/Environment.hs view
@@ -68,10 +68,15 @@ optsEnvironment homes info o ++ [ setting "Run from" runFrom "", setting "Start year" (maybe "all years" (\y -> tshow y <> " onwards") (importStartYear o)) "--start-year",- setting "Files" (if onlyNewFiles o then "only new or changed" else "regenerate all") "--new-files-only",+ files, parallelism o ] where+ files = case importPolicy o of+ Full -> setting "Files" "regenerate all" "--new-files-only"+ TimestampOnly -> setting "Files" "only new or changed" "--new-files-only"+ RunScripts -> setting "Files" "incremental (run-scripts)" "--incremental"+ SkipUnchangedScripts -> setting "Files" "incremental (skip-unchanged-scripts)" "--incremental" runFrom = case dropTrailingPathSeparator (toFilePath (importRunDir o)) of "." -> ". (whole ledger)" dir -> T.pack dir
src/Hledger/Flow/Import/CSVImport.hs view
@@ -14,7 +14,8 @@ import Hledger.Flow.Environment (importSettings, renderSettings) import Hledger.Flow.BaseDir (effectiveRunDir, relativeToBase) import Hledger.Flow.Common-import Hledger.Flow.DocHelpers (DocLink (..), docURL)+import Hledger.Flow.Import.Incremental+import Hledger.Flow.Import.Conversion import Hledger.Flow.Import.ImportHelpers import Hledger.Flow.Import.IncludeFiles (writeIncludesFromInventory, writeToplevelAllYearsInclude) import Hledger.Flow.Import.Inventory@@ -25,12 +26,7 @@ import qualified Hledger.Flow.Types as FlowTypes import qualified Path import qualified Path.IO as PathIO-import System.Environment (getEnv)-import System.Exit (ExitCode (..), exitWith)-import System.IO (hClose)-import qualified System.Info as Sys-import qualified System.Process as Process-import System.FilePath (dropTrailingPathSeparator, takeExtension, (</>))+import System.FilePath (dropTrailingPathSeparator) import qualified Data.List as List import Hledger.Flow.Status (plural) import Hledger.Flow.Status.Gather (accountName)@@ -40,18 +36,19 @@ importCSVs :: RuntimeOptions -> IO () importCSVs opts = withConsoleLogging $ \ch -> do+ homes <- homeDirs when (showOptions opts) $ do- homes <- homeDirs info <- hledgerInfo opts channelOutLn ch (T.unlines (renderSettings id (importSettings homes info opts)))- logVerbose opts ch "Starting import"+ logVerbose opts ch $ "Base directory: " <> T.pack (tildePath homes (dropTrailingPathSeparator (Path.toFilePath (baseDir opts)))) (journals, diff) <- timeAction $ importCSVs' opts ch let importRoot = baseDir opts Path.</> [Path.reldir|import|] effectiveDir = effectiveRunDir (baseDir opts) (importRunDir opts) scope = T.pack (dropTrailingPathSeparator (relativeToBase opts (Path.toFilePath effectiveDir))) accounts = List.nub (map (accountName importRoot . fst) journals)+ when (verbose opts) $ channelErr ch "\n" channelOutLn ch (importSummary scope (length journals) (length (filter snd journals)) accounts)- logVerbose opts ch ("Import took " <> T.pack (show diff))+ logVerbose opts ch ("Import finished in " <> showDuration diff) -- | One line saying what import did. importSummary :: T.Text -> Int -> Int -> [T.Text] -> T.Text@@ -71,42 +68,50 @@ let effectiveDir = effectiveRunDir (baseDir opts) (importRunDir opts) importRoot = baseDir opts Path.</> [Path.reldir|import|] selected = inImportScope effectiveDir (fromMaybe 0 (importStartYear opts))- let startYearMsg = maybe " " (\y -> " (for the year " <> T.pack (show y) <> " and onwards) ") (importStartYear opts)- let dirText = T.pack (Path.toFilePath effectiveDir)- logVerbose opts ch $ "Collecting input files" <> startYearMsg <> "from " <> dirText+ let startYearMsg = maybe "" (\y -> " from " <> T.pack (show y) <> " onwards") (importStartYear opts)+ let relative = T.pack . relativeToBase opts . Path.toFilePath+ logVerbose opts ch $ "Looking for statements" <> startYearMsg <> " in " <> relative effectiveDir+ <> " (import/<owner>/<bank>/<account>/1-in/<year>/)" (inventory, diff) <- timeAction $ findImportFiles importRoot let inputFiles = filter selected (stageFiles importRoot [Path.reldir|1-in|] inventory) fileCount = length inputFiles collisions <- conflictingOutputs inputFiles- let relative = T.pack . relativeToBase opts . Path.toFilePath unless (null collisions) $ errExit 1 ch ("Multiple input files map to the same output:\n" <> T.intercalate "\n" [relative output <> " <- " <> T.intercalate ", " (map relative sources) | (output, sources) <- collisions])- logVerbose opts ch $ "Found " <> T.pack (show fileCount) <> " input files" <> startYearMsg <> "in " <> T.pack (show diff) <> ". Proceeding with import..."+ logVerbose opts ch $ "Found " <> plural fileCount "statement" <> " in " <> showDuration diff -- Glob escaping depends on the hledger version. Check it before any worker -- writes output; conversions reuse this result. directives.journal and -- cached CSVs share these paths' base and statement names. when (any (any isGlobMetacharacter . Path.toFilePath) inputFiles) $ void (hledgerVersion opts)- importedJournals <- parAwareActions opts (map (extractAndImport opts ch importRoot) inputFiles)-- -- Running a preprocess hook also forces conversion. When every journal was- -- skipped, neither a hook nor a conversion could have changed the inventory.- let cached = if any snd importedJournals then Nothing else Just inventory+ let run tracking = parAwareActions opts (map (extractAndImport tracking opts ch importRoot) inputFiles)+ outcomes <- case importPolicy opts of+ RunScripts -> withIncrementalRun opts ch fileCount (run . Just)+ SkipUnchangedScripts -> withIncrementalRun opts ch fileCount (run . Just)+ _ -> run Nothing+ let importedJournals = [(journal, generated) | (journal, generated, _) <- outcomes]+ cached = if or [refresh | (_, _, refresh) <- outcomes] then Nothing else Just inventory ((refreshed, orphans, journalsOnDisk), journalFindTime) <- timeAction $ reconciliationInventory importRoot cached forM_ (filter selected orphans) $ \file -> do target <- moveToTrash file channelOutLn ch $ "Moved orphaned output " <> relative file <> " to " <> relative target- (_, writeIncludeTime1) <- timeAction $ writeIncludesFromInventory opts ch importRoot refreshed journalsOnDisk- (_, writeIncludeTime2) <- timeAction $ writeToplevelAllYearsInclude opts- let includeGenTime = journalFindTime + writeIncludeTime1 + writeIncludeTime2- logVerbose opts ch $ "Wrote include files for " <> T.pack (show (length journalsOnDisk)) <> " generated journals in " <> T.pack (show includeGenTime)+ withLogBlock opts True ch "Include files" $ \_ -> do+ (_, writeIncludeTime1) <- timeAction $ writeIncludesFromInventory opts ch importRoot refreshed journalsOnDisk+ (_, writeIncludeTime2) <- timeAction $ writeToplevelAllYearsInclude opts+ let includeGenTime = journalFindTime + writeIncludeTime1 + writeIncludeTime2+ logVerbose opts ch $ "Updated include files for " <> plural (length journalsOnDisk) "generated journal" <> " in " <> showDuration includeGenTime return importedJournals -extractAndImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsDir -> AbsFile -> IO (AbsFile, FileWasGenerated)-extractAndImport opts ch root inputFile = do+extractAndImport :: Maybe IncrementalRun -> RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsDir -> AbsFile -> IO (AbsFile, Bool, Bool)+extractAndImport tracking opts parent root inputFile =+ withLogBlock opts (sequential opts) parent ("Statement " <> T.pack (relativeToBase opts (Path.toFilePath inputFile))) $ \ch -> case extractImportDirs root inputFile of- Right importDirs -> importCSV opts ch importDirs inputFile+ Right importDirs -> case tracking of+ Just run -> importStatement run opts ch importDirs inputFile+ Nothing -> do+ (output, generated) <- importCSV opts ch importDirs inputFile+ pure (output, generated, generated) Left errorMessage -> do errExit 1 ch errorMessage @@ -122,148 +127,26 @@ PathIO.createDirIfMissing True (Path.parent journalOut) if shouldImport then do- hook <- findHook opts ch (accountDir importDirs Path.</> [Path.relfile|construct|])+ hook <- findHook opts ch "construct script" (accountDir importDirs Path.</> [Path.relfile|construct|]) case hook of- Just script -> withStagedOutput journalOut (customConstruct opts ch script importDirs csvFile)+ Just script -> withStagedOutput journalOut (customConstruct opts ch script importDirs csvFile journalOut) Nothing -> hledgerImport opts ch importDirs csvFile journalOut+ logVerbose opts ch ("Wrote " <> T.pack (relativeToBase opts (Path.toFilePath journalOut))) else logNewFileSkip opts ch "import" journalOut return (journalOut, shouldImport) preprocessIfNeeded :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO (AbsFile, Bool) preprocessIfNeeded opts ch dirs src csvOut = do- hook <- findHook opts ch (accountDir dirs Path.</> [Path.relfile|preprocess|])+ hook <- findHook opts ch "preprocess script" (accountDir dirs Path.</> [Path.relfile|preprocess|]) case hook of Nothing -> do- cached <- verboseTestFile opts ch csvOut- if cached- then do- stale <- needsRegeneration src csvOut- when stale $ ioError $ userError $- relativeToBase opts (Path.toFilePath src) ++ " is newer than its cached CSV "- ++ relativeToBase opts (Path.toFilePath csvOut) ++ ". Restore the preprocess hook to regenerate it,"- ++ " or move the cached CSV aside to use the source directly. Then run hledger-flow import."- return (csvOut, False)- else return (src, False)+ effective <- savedInput opts ch src csvOut+ return (effective, False) Just script -> do regenerate <- if onlyNewFiles opts then needsRegeneration src csvOut else return True if regenerate- then withStagedOutput csvOut (preprocess opts ch script dirs src)+ then do+ withStagedOutput csvOut (preprocess opts ch script dirs src csvOut)+ logVerbose opts ch ("Wrote " <> T.pack (relativeToBase opts (Path.toFilePath csvOut))) else logNewFileSkip opts ch "preprocess" csvOut return (csvOut, regenerate)--findHook :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> IO (Maybe AbsFile)-findHook opts ch base = do- alternatives <- if Sys.os == "mingw32"- then mapM (`Path.addExtension` base) [".ps1", ".exe"]- else return []- first (base : alternatives)- where- first [] = return Nothing- first (file : rest) = do- exists <- verboseTestFile opts ch file- if exists then return (Just file) else first rest--hookCommand :: AbsFile -> [T.Text] -> IO (T.Text, [T.Text])-hookCommand script args- | Sys.os == "mingw32" && takeExtension path == ".ps1" = do- root <- getEnv "SystemRoot"- let shell = root </> "System32" </> "WindowsPowerShell" </> "v1.0" </> "powershell.exe"- -- Windows PowerShell -File treats construct's literal "-" as an option.- -- Invoke the script with literal strings and propagate its exit status.- let command = "[Console]::OutputEncoding = New-Object Text.UTF8Encoding($false); & "- <> T.unwords (map powerShellLiteral (T.pack path : args))- <> "; if (!$?) { if ($LASTEXITCODE) { exit $LASTEXITCODE }; exit 1 }"- return (T.pack shell, ["-NoProfile", "-NonInteractive", "-OutputFormat", "Text", "-Command", command])- | otherwise = return (T.pack path, args)- where- path = Path.toFilePath script--logNewFileSkip :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> AbsFile -> IO ()-logNewFileSkip opts ch logIdentifier absTarget =- Control.Monad.when (onlyNewFiles opts) $ do- let relativeTarget = relativeToBase opts (Path.toFilePath absTarget)- logVerbose opts ch $- "Skipping " <> logIdentifier <> " - output file is at least as new as source: '" <> T.pack relativeTarget <> "'"--preprocess :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()-preprocess opts ch script dirs src csvOut = do- let args = map (T.pack . Path.toFilePath) [src, csvOut] ++ hookDirArgs dirs- let relScript = relativeToBase opts (Path.toFilePath script)- let relSrc = relativeToBase opts (Path.toFilePath src)- let cmdLabel = "executing '" <> T.pack relScript <> "' on '" <> T.pack relSrc <> "'"- (cmd, hookArgs) <- hookCommand script args- _ <- timeAndExitOnErr opts ch cmdLabel channelOut channelErr (cmd, hookArgs, Nothing)- return ()--hledgerImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()-hledgerImport opts ch importDirs csvFile journalOut = do- let csvSrc = Path.toFilePath csvFile- candidates <- rulesFileCandidates csvFile importDirs- maybeRulesFile <- firstExistingFile candidates- let relCSV = relativeToBase opts csvSrc- case maybeRulesFile of- Just rf -> do- let relRules = relativeToBase opts (Path.toFilePath rf)- let hledger = T.pack (Path.toFilePath (hledgerExecutable opts))- let directives = directivesFile opts- directivesExist <- PathIO.doesFileExist directives- let paths = csvSrc : [Path.toFilePath directives | directivesExist]- fileArgument <- if any (any isGlobMetacharacter) paths- then hledgerFileArgument <$> hledgerInfo opts- else return T.pack- let directivesArgs = if directivesExist then ["--file", fileArgument (Path.toFilePath directives)] else []- let csvArgs = ["--file", fileArgument csvSrc, "--rules-file", T.pack (Path.toFilePath rf)]- let args = hledgerConfArgs opts ++ ["print", "--explicit"] ++ directivesArgs ++ csvArgs-- let cmdLabel = "importing '" <> T.pack relCSV <> "' using rules file '" <> T.pack relRules <> "'"- let hints = ["Check the skip, fields and separator lines in " <> T.pack relRules <> ".", "Docs: " <> docURL RulesFiles]- ((_, stdOut, _), _) <- timeAndExitOnErr' opts ch cmdLabel [] hints dummyLogger channelErr (hledger, args, Nothing)- writeGeneratedText (Path.toFilePath journalOut) stdOut- Nothing ->- do- let candidatesTxt = T.intercalate "\n" $ map (T.pack . relativeToBase opts . Path.toFilePath) candidates- let msg =- "I couldn't find an hledger rules file while trying to import\n" <> T.pack relCSV- <> "\n\nI will happily use the first rules file I can find from any one of these "- <> T.pack (show (length candidates)) <> " files:\n" <> candidatesTxt- <> "\n\nHere is a bit of documentation about rules files that you may find helpful:\n"- <> docURL RulesFiles- errExit 1 ch msg--hookDirArgs :: ImportDirs b -> [T.Text]-hookDirArgs dirs = map (T.concat . importDirLines) [bankDir dirs, accountDir dirs, ownerDir dirs]--customConstruct :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()-customConstruct opts ch constructScript dirs csvFile journalOut = do- let csvSrc = Path.toFilePath csvFile- let relScript = relativeToBase opts (Path.toFilePath constructScript)- let constructArgs = [T.pack csvSrc, "-"] ++ hookDirArgs dirs- homes <- homeDirs- let constructCmdText = "Running: " <> T.pack relScript <> " " <> showCmdArgsAt homes constructArgs- (cmd, hookArgs) <- hookCommand constructScript constructArgs- let hook = (Process.proc (T.unpack cmd) (map T.unpack hookArgs))- { Process.std_in = Process.CreatePipe, Process.std_out = Process.CreatePipe }- let hledger = T.pack (Path.toFilePath (hledgerExecutable opts))- let args =- hledgerConfArgs opts- ++ [ "print",- "--ignore-assertions",- "--file",- "-",- "--output-file",- T.pack (Path.toFilePath journalOut)- ]-- let relSrc = relativeToBase opts csvSrc- let cmdLabel = "executing '" <> T.pack relScript <> "' on '" <> T.pack relSrc <> "'"- Process.withCreateProcess hook $ \input output _ handle -> do- Just stdinPipe <- return input- Just stdoutPipe <- return output- hClose stdinPipe- _ <- timeAndExitOnErr' opts ch cmdLabel [constructCmdText] [] channelOut channelErr- (hledger, args, Just stdoutPipe)- hClose stdoutPipe- status <- Process.waitForProcess handle- case status of- ExitSuccess -> return ()- ExitFailure _ -> exitWith status
+ src/Hledger/Flow/Import/Cache.hs view
@@ -0,0 +1,100 @@+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Hledger.Flow.Import.Cache+ ( StageRecord (..), StatementRecord (..), StageInputs+ , recordPath, readRecord, saveRecord+ , stageMiss, installOutput, digestIfPresent+ ) where++import Control.Monad (unless)+import Data.Aeson (FromJSON, ToJSON, eitherDecodeStrict', encode)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Lazy as BL+import Data.Text (Text)+import qualified Data.Text as T+import GHC.Generics (Generic)+import Hledger.Flow.Cache (cacheDir)+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.PathHelpers+import qualified Path as P+import qualified Path.IO as P+import System.IO.Error (isDoesNotExistError, tryIOError)++type StageInputs = [(Text, Text)]++data StageRecord = StageRecord+ { stageInputs :: StageInputs+ , stageOutput :: Digest+ } deriving (Eq, Show, Generic, FromJSON, ToJSON)++data StatementRecord = StatementRecord+ { preprocessRecord :: Maybe StageRecord+ , journalRecord :: Maybe StageRecord+ } deriving (Eq, Show, Generic, FromJSON, ToJSON)++-- Bump when a conversion's semantics, cache key or serialization changes.+recordVersion :: Int+recordVersion = 2 -- Earlier snapshots could overwrite a differently spelled input.++recordPath :: AbsDir -> AbsFile -> IO AbsFile+recordPath base source = do+ let year = P.parent source+ account = P.parent (P.parent year)+ group <- P.stripProperPrefix (base P.</> [P.reldir|import|]) account+ pure $ cacheDir base P.</> [P.reldir|import|] P.</> group+ P.</> P.dirname year P.</> P.filename source++readRecord :: AbsFile -> IO (Maybe StatementRecord)+readRecord path = do+ result <- tryIOError (BS.readFile (P.toFilePath path))+ pure $ case result of+ Right bytes | Right (version, record) <- eitherDecodeStrict' bytes+ , version == recordVersion -> Just record+ _ -> Nothing++saveRecord :: AbsFile -> StatementRecord -> IO ()+saveRecord path record = withStagedOutput path $ \stage ->+ BL.writeFile (P.toFilePath stage) (encode (recordVersion, record))++digestIfPresent :: AbsFile -> IO (Maybe Digest)+digestIfPresent path = either (const Nothing) Just <$> tryIOError (digestFile path)++-- Return the first difference in the same ordered key used for cache lookup.+-- File dependencies use their resolved path as the key, including nested rules.+stageMiss :: StageInputs -> AbsFile -> Maybe StageRecord -> IO (Maybe Text)+stageMiss _ _ Nothing = pure (Just "no valid record for this stage")+stageMiss inputs output (Just record) =+ case changedInput (stageInputs record) inputs of+ Just key -> pure (Just ("changed input: " <> key))+ Nothing -> do+ result <- tryIOError (digestFile output)+ pure $ case result of+ Left err | isDoesNotExistError err -> Just ("missing output: " <> path)+ | otherwise -> Just ("could not read output: " <> path)+ Right digest | digest /= stageOutput record -> Just ("changed output: " <> path)+ | otherwise -> Nothing+ where+ path = T.pack (P.toFilePath output)+ changedInput [] [] = Nothing+ changedInput ((key,_):_) [] = Just key+ changedInput [] ((key,_):_) = Just key+ changedInput (old:xs) (new:ys)+ | old == new = changedInput xs ys+ | otherwise = Just (fst new)++-- The writer owns its staged file until it returns. Hash it before publishing:+-- another process can install a different output as soon as the rename finishes.+installOutput :: Bool -> AbsFile -> (AbsFile -> IO ()) -> IO Digest+installOutput keepIdentical target writeOutput = do+ let dir = P.parent target+ P.createDirIfMissing True dir+ P.withTempDir dir ".hledger-flow-" $ \tmp -> do+ let staged = tmp P.</> P.filename target+ writeOutput staged+ digest <- digestFile staged+ existing <- if keepIdentical then digestIfPresent target else pure Nothing+ unless (existing == Just digest) $ P.renameFile staged target+ pure digest
+ src/Hledger/Flow/Import/Conversion.hs view
@@ -0,0 +1,193 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+module Hledger.Flow.Import.Conversion+ ( findHook, preprocess, savedInput, logNewFileSkip, hledgerImport, customConstruct+ , hookDirArgs, RulesPaths(..), convertRules, constructWith, rulesArguments, constructArguments, pinnedConfig+ ) where++import Control.Concurrent.STM+import Control.Monad+import qualified Data.Text as T+import Hledger.Flow.DateTime (timeAction)+import Hledger.Flow.BaseDir (relativeToBase)+import Hledger.Flow.Common+import Hledger.Flow.DocHelpers (DocLink (..), docURL)+import Hledger.Flow.Import.ImportHelpers+import Hledger.Flow.Import.Types+import Hledger.Flow.Logging+import Hledger.Flow.PathHelpers (AbsFile, isGlobMetacharacter)+import Hledger.Flow.RuntimeOptions+import qualified Hledger.Flow.Types as FlowTypes+import qualified Path+import qualified Path.IO as PathIO+import System.Environment (getEnv)+import System.Exit (ExitCode (..))+import Control.Concurrent.Async (concurrently)+import qualified Data.Text.IO as T+import System.IO (hClose, stderr)+import qualified System.Info as Sys+import qualified System.Process as Process+import System.FilePath (takeExtension, (</>))+import Prelude hiding (putStrLn, take, writeFile)++findHook :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> AbsFile -> IO (Maybe AbsFile)+findHook opts ch what base = do+ alternatives <- if Sys.os == "mingw32"+ then mapM (`Path.addExtension` base) [".ps1", ".exe"]+ else return []+ findFirstFile opts ch what (base : alternatives)++hookCommand :: AbsFile -> [T.Text] -> IO (T.Text, [T.Text])+hookCommand script args+ | Sys.os == "mingw32" && takeExtension path == ".ps1" = do+ root <- getEnv "SystemRoot"+ let shell = root </> "System32" </> "WindowsPowerShell" </> "v1.0" </> "powershell.exe"+ -- Windows PowerShell -File treats construct's literal "-" as an option.+ -- Invoke the script with literal strings and propagate its exit status.+ let command = "[Console]::OutputEncoding = New-Object Text.UTF8Encoding($false); & "+ <> T.unwords (map powerShellLiteral (T.pack path : args))+ <> "; if (!$?) { if ($LASTEXITCODE) { exit $LASTEXITCODE }; exit 1 }"+ return (T.pack shell, ["-NoProfile", "-NonInteractive", "-OutputFormat", "Text", "-Command", command])+ | otherwise = return (T.pack path, args)+ where+ path = Path.toFilePath script++logNewFileSkip :: RuntimeOptions -> TChan FlowTypes.LogMessage -> T.Text -> AbsFile -> IO ()+logNewFileSkip opts ch logIdentifier absTarget =+ Control.Monad.when (onlyNewFiles opts) $ do+ let relativeTarget = relativeToBase opts (Path.toFilePath absTarget)+ logVerbose opts ch $+ "Skipping " <> logIdentifier <> ": " <> T.pack relativeTarget <> " is at least as new as its source"++-- | Run the preprocess script, writing to the staged file. The command shown names the final+-- output instead, so a pasted command writes where hledger-flow does.+preprocess :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()+preprocess opts ch script dirs src csvOut staged = do+ let args out = map (T.pack . Path.toFilePath) [src, out] ++ hookDirArgs dirs+ relSrc = relativeToBase opts (Path.toFilePath src)+ external = External (relativeHook opts script) ("preprocess " <> T.pack relSrc)+ [T.pack (Path.toFilePath script) : args csvOut] []+ (cmd, hookArgs) <- hookCommand script (args staged)+ void $ runExternal opts ch external True (cmd, hookArgs, Nothing)++relativeHook :: RuntimeOptions -> AbsFile -> T.Text+relativeHook opts = T.pack . relativeToBase opts . Path.toFilePath++savedInput :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> AbsFile -> IO AbsFile+savedInput opts ch src csvOut = do+ cached <- verboseTestFile opts ch "preprocessed CSV from an earlier run" csvOut+ if cached+ then do+ stale <- needsRegeneration src csvOut+ when stale $ ioError $ userError $+ relativeToBase opts (Path.toFilePath src) ++ " is newer than its cached CSV "+ ++ relativeToBase opts (Path.toFilePath csvOut) ++ ". Restore the preprocess script to regenerate it,"+ ++ " or move the cached CSV aside to use the source directly. Then run hledger-flow import."+ return csvOut+ else return src++hledgerImport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> IO ()+hledgerImport opts ch importDirs csvFile journalOut = do+ let csvSrc = Path.toFilePath csvFile+ candidates <- rulesFileCandidates csvFile importDirs+ maybeRulesFile <- findFirstFile opts ch "rules file" candidates+ let relCSV = relativeToBase opts csvSrc+ case maybeRulesFile of+ Just rf -> do+ let directives = directivesFile opts+ exists <- PathIO.doesFileExist directives+ let paths = RulesPaths csvFile rf (if exists then Just directives else Nothing)+ stdOut <- convertRules opts ch (hledgerExecutable opts) paths paths (hledgerConfArgs opts) pure+ writeGeneratedText (Path.toFilePath journalOut) stdOut+ Nothing ->+ do+ let candidatesTxt = T.intercalate "\n" $ map (T.pack . relativeToBase opts . Path.toFilePath) candidates+ let msg =+ "I couldn't find an hledger rules file while trying to import\n" <> T.pack relCSV+ <> "\n\nI will happily use the first rules file I can find from any one of these "+ <> T.pack (show (length candidates)) <> " files:\n" <> candidatesTxt+ <> "\n\nHere is a bit of documentation about rules files that you may find helpful:\n"+ <> docURL RulesFiles+ errExit 1 ch msg++data RulesPaths = RulesPaths AbsFile AbsFile (Maybe AbsFile)++-- The journal cache uses these same argument builders with stable path labels.+pinnedConfig :: Maybe T.Text -> [T.Text]+pinnedConfig = maybe ["--no-conf"] (\path -> ["--conf", path])++rulesArguments :: [T.Text] -> T.Text -> T.Text -> Maybe T.Text -> [T.Text]+rulesArguments conf input rules directives =+ conf ++ ["print", "--explicit"] ++ maybe [] (\path -> ["--file", path]) directives+ ++ ["--file", input, "--rules-file", rules]++constructArguments :: [T.Text] -> T.Text -> [T.Text]+constructArguments conf out = conf ++ ["print", "--ignore-assertions", "--file", "-", "--output-file", out]++convertRules :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> RulesPaths -> RulesPaths -> [T.Text]+ -> (T.Text -> IO T.Text) -> IO T.Text+convertRules opts ch executable original@(RulesPaths csv rules _) actual config translate = do+ let shown = T.pack . relativeToBase opts . Path.toFilePath+ hledger = T.pack (Path.toFilePath (hledgerExecutable opts))+ args conf (RulesPaths input rf directives) = do+ let paths = input : maybe [] pure directives+ fileArg <- if any (any isGlobMetacharacter . Path.toFilePath) paths+ then hledgerFileArgument <$> hledgerInfo opts else pure T.pack+ pure $ rulesArguments conf (fileArg (Path.toFilePath input)) (T.pack (Path.toFilePath rf))+ (fileArg . Path.toFilePath <$> directives)+ command <- args (hledgerConfArgs opts) original+ invocation <- args config actual+ let ext = External "hledger" ("convert " <> shown csv <> " with the rules in " <> shown rules)+ [hledger : command] ["Check the skip, fields and separator lines in " <> shown rules <> ".", "Docs: " <> docURL RulesFiles]+ announceExternal opts ch ext+ ((status,out,err),duration) <- timeAction $ runProcessCaptured (T.pack (Path.toFilePath executable),invocation,Nothing)+ diagnostic <- translate err+ diagnosticOut <- if status == ExitSuccess then pure out else translate out+ checkExternal opts ch ext False duration (status,diagnosticOut,diagnostic)+ pure out++hookDirArgs :: ImportDirs b -> [T.Text]+hookDirArgs dirs = map (T.concat . importDirLines) [bankDir dirs, accountDir dirs, ownerDir dirs]++-- | Pipe the construct script's output through hledger into the staged journal. The command+-- shown names the final journal instead, so a pasted command writes where hledger-flow does.+customConstruct :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()+customConstruct opts ch = constructWith opts ch (hledgerExecutable opts) (hledgerConfArgs opts) pure++constructWith :: RuntimeOptions -> TChan FlowTypes.LogMessage -> AbsFile -> [T.Text] -> (T.Text -> IO T.Text) -> AbsFile -> ImportDirs Path.Abs -> AbsFile -> AbsFile -> AbsFile -> IO ()+constructWith opts ch executable config translate constructScript dirs csvFile journalOut staged = do+ let csvSrc = Path.toFilePath csvFile+ relScript = relativeHook opts constructScript+ relSrc = T.pack (relativeToBase opts csvSrc)+ constructArgs = [T.pack csvSrc, "-"] ++ hookDirArgs dirs+ hledger = T.pack (Path.toFilePath (hledgerExecutable opts))+ hledgerArgs conf = constructArguments conf . T.pack . Path.toFilePath+ shown = [T.pack (Path.toFilePath constructScript) : constructArgs, hledger : hledgerArgs (hledgerConfArgs opts) journalOut]+ constructExt = External relScript ("construct a journal from " <> relSrc) shown []+ hledgerExt = External "hledger" ("read the journal " <> relScript <> " constructed from " <> relSrc) shown []+ (cmd, hookArgs) <- hookCommand constructScript constructArgs+ let hook = (Process.proc (T.unpack cmd) (map T.unpack hookArgs))+ { Process.std_in = Process.CreatePipe, Process.std_out = Process.CreatePipe, Process.std_err = Process.CreatePipe }+ announceExternal opts ch constructExt+ Process.withCreateProcess hook $ \input output errors handle -> do+ Just stdinPipe <- return input+ Just stdoutPipe <- return output+ Just stderrPipe <- return errors+ hClose stdinPipe+ (((hledgerStatus, converterOut, converterErr), duration), hookErr) <- concurrently+ (timeAction (runProcessCaptured (T.pack (Path.toFilePath executable), hledgerArgs config staged, Just stdoutPipe)))+ (T.hGetContents stderrPipe)+ mappedErr <- translate converterErr+ let converted = (hledgerStatus, converterOut, mappedErr)+ status <- Process.waitForProcess handle+ color <- useColor stderr+ -- When hledger stops reading early, the script fails with a broken pipe, so hledger's error+ -- comes first, along with anything the script said.+ let scriptNote = case status of+ ExitFailure code -> [relScript <> " also failed, with exit code " <> T.pack (show code) <> "."]+ ExitSuccess -> []+ scriptOutput = outputSection color (verbose opts) ("Error output from " <> relScript) hookErr+ when (hledgerStatus /= ExitSuccess) $+ checkExternal opts ch hledgerExt {extHints = scriptNote ++ scriptOutput} True duration converted+ checkExternal opts ch constructExt True duration (status, "", hookErr)+ checkExternal opts ch hledgerExt True duration converted
+ src/Hledger/Flow/Import/Dependencies.hs view
@@ -0,0 +1,360 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE DeriveFunctor #-}+{-# LANGUAGE OverloadedStrings #-}++module Hledger.Flow.Import.Dependencies+ ( Release (..), FallbackReason (..), ReuseDecision (..), Converter (..),+ ruleIncludes, nestedRuleIncludes, resolveRuleInclude, discoverRules, selectRules, isSeparatedValues,+ checkedIO, atPath, checkArguments, discoverConfig, discoverDirectives, checkConfig, checkDirectives,+ newConverterProbe, converterProbeWith+ ) where++import Control.Concurrent.MVar (modifyMVar, newMVar)+import Control.Monad (ap, foldM)+import qualified Data.ByteString as B+import Data.Char (isAlpha, isAlphaNum, isAscii, isControl, isDigit, isSpace, toLower, GeneralCategory(CurrencySymbol), generalCategory)+import Data.List (isPrefixOf)+import Data.Maybe (mapMaybe)+import qualified Data.Set as Set+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import qualified Data.Text.Read as TR+import Data.Time (getCurrentTimeZone, timeZoneMinutes)+import GHC.IO.Encoding (getLocaleEncoding, getFileSystemEncoding, getForeignEncoding)+import Hledger.Flow.Common (firstConfirmedFile, lookupHledgerConf, probeHledgerVersion, hledgerVersionNumber)+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.ImportHelpers (rulesFileCandidates)+import Hledger.Flow.Import.Types (ImportDirs)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile)+import qualified Hledger.Flow.RuntimeOptions as RT+import Hledger.Flow.Types (HledgerConf(..), hledgerConfPath)+import Path (Abs, parseAbsFile, toFilePath)+import qualified System.Directory as D+import qualified System.FilePath as F+import System.Exit (ExitCode(..))+import System.IO (IOMode(ReadMode), withBinaryFile)+import System.IO.Error (tryIOError, ioeGetFileName, ioeGetErrorString)+import qualified System.Info as Sys+#ifdef mingw32_HOST_OS+import System.Win32.Console (getConsoleCP, getConsoleOutputCP)+import System.Win32.NLS (getACP)+#endif++-- Exact releases have known reader quirks. Other eligible versions use syntax+-- shared by both readers, leaving ambiguous inputs to the ordinary converter.+data Release = V1524 | V1996 | OtherSupported deriving (Eq, Show)++data FallbackReason+ = FallbackReason {reasonScope :: T.Text, reasonDetail :: T.Text}+ | FileReason {reasonScope :: T.Text, reasonFile :: FilePath, reasonDetail :: T.Text}+ | OrdinaryInputError {reasonScope :: T.Text, reasonDetail :: T.Text}+ | CyclicRules {reasonScope :: T.Text, reasonDetail :: T.Text}+ deriving (Eq, Ord, Show)+data ReuseDecision a = Reusable a | Fallback FallbackReason deriving (Functor, Eq, Show)++instance Applicative ReuseDecision where+ pure = Reusable+ (<*>) = ap++instance Monad ReuseDecision where+ Reusable a >>= f = f a+ Fallback r >>= _ = Fallback r++atPath :: T.Text -> FilePath -> T.Text -> ReuseDecision a+atPath reason path detail = Fallback (FileReason reason path detail)++checkedIO :: T.Text -> FilePath -> IO (ReuseDecision a) -> IO (ReuseDecision a)+checkedIO reason path action = do+ result <- tryIOError action+ pure $ case result of+ Right a -> a+ Left e -> atPath reason (maybe path id (ioeGetFileName e)) (T.pack (ioeGetErrorString e))++physicalLines :: AbsFile -> B.ByteString -> ReuseDecision [T.Text]+physicalLines path bytes = case TE.decodeUtf8' bytes of+ Left _ -> atPath "unrecognised text encoding" (toFilePath path) "expected UTF-8"+ Right s -> pure (T.lines (T.replace "\r\n" "\n" s))++ruleIncludes :: Release -> AbsFile -> B.ByteString -> ReuseDecision [FilePath]+ruleIncludes release path bytes = do+ ls <- physicalLines path (dropUtf8Bom bytes)+ sequence (mapMaybe include (zip [1 :: Int ..] ls))+ where+ include (n,line)+ | Just raw <- T.stripPrefix "include " line = Just (checkPath n raw)+ | release == OtherSupported && looksLikeInclude line = Just (uncertain n)+ | otherwise = Nothing+ looksLikeInclude line = case T.words line of+ word:_ -> T.toCaseFold word == "include"+ [] -> False+ checkPath n raw+ | T.null name || T.any isControl name = uncertain n+ | release == OtherSupported && name /= T.stripEnd name = uncertain n+ | otherwise = pure (T.unpack name)+ where name = if release == V1996 then T.strip raw else T.stripStart raw+ uncertain n = atPath "uncertain rules include" (toFilePath path) ("line " <> T.pack (show n))++-- Older readers open nested rules through the native text handle. On POSIX a+-- CR at the end of an include line may be part of the filename.+nestedRuleIncludes :: Release -> AbsFile -> B.ByteString -> ReuseDecision [FilePath]+nestedRuleIncludes release path bytes+ | release /= V1996, any ambiguous (B.split 10 bytes) =+ atPath "uncertain nested rules include" (toFilePath path) "native line ending or byte-order mark"+ | otherwise = ruleIncludes release path bytes+ where+ ambiguous line = "include " `B.isPrefixOf` dropUtf8Bom line &&+ (B.elem 13 line || dropUtf8Bom line /= line)++dropUtf8Bom :: B.ByteString -> B.ByteString+dropUtf8Bom b = maybe b id (B.stripPrefix (B.pack [0xef,0xbb,0xbf]) b)++-- Walk leading .. components relative to the including file's spelled path.+-- Verify the result against the filesystem's resolution, which can differ when a+-- symlink precedes .. . An internal symlink is mirrored under its original name.+resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)+resolveRuleInclude base parent child = checkedIO "uncertain rules include" raw $+ if F.hasDrive child || not (F.isRelative child) || any F.isPathSeparator (take 1 child)+ then pure (atPath "absolute rules include" child (T.pack (toFilePath parent)))+ else do+ let (ups, rest) = span (== "..") (filter (/= ".") (F.splitDirectories child))+ lexical = foldr (const F.takeDirectory) (F.takeDirectory (toFilePath parent)) ups F.</> F.joinPath rest+ if ".." `elem` rest || null rest || not (inside (toFilePath base) lexical)+ then pure outside+ else do+ actual <- D.canonicalizePath raw+ mirror <- D.canonicalizePath lexical+ root <- D.canonicalizePath (toFilePath base)+ if F.equalFilePath actual mirror && inside root actual+ then pure (maybe outside Reusable (parseAbsFile lexical))+ else pure outside+ where+ raw = F.takeDirectory (toFilePath parent) F.</> child+ outside = atPath "rules include escapes the ledger or cannot be mirrored" raw (T.pack (toFilePath parent))+ inside root path = F.splitDirectories (F.dropTrailingPathSeparator root) `isPrefixOf` F.splitDirectories path++discoverRules :: AbsDir -> Release -> AbsFile -> IO (ReuseDecision [(AbsFile,Digest)])+discoverRules base release = walk Set.empty+ where+ walk ancestors path = checkedIO "could not read rules file" (toFilePath path) $ do+ canonical <- D.canonicalizePath (toFilePath path)+ directory <- D.canonicalizePath (F.takeDirectory (toFilePath path))+ -- The same rules bytes can resolve includes in different directories.+ -- In 1.52.4 the first read also decodes line endings differently.+ let node = (canonical, directory, release /= V1996 && Set.null ancestors)+ if Set.member node ancestors && release /= V1996+ then pure $ Fallback (CyclicRules "cyclic rules include" (T.pack (toFilePath path)))+ else if Set.member node ancestors || Set.size ancestors >= 256+ then pure (atPath "cyclic or excessively deep rules include" (toFilePath path) "")+ else do+ bytes <- B.readFile (toFilePath path)+ let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes+ case scan release path bytes of+ Fallback r -> pure (Fallback r)+ Reusable children -> do+ nested <- descend (Set.insert node ancestors) path children+ pure (((path,digestBytes bytes):) <$> nested)+ descend _ _ [] = pure (pure [])+ descend ancestors path (c:cs) = do+ selected <- resolveRuleInclude base path c+ tree <- case selected of+ Fallback r -> pure (Fallback r)+ Reusable file -> walk ancestors file+ case tree of+ Fallback r -> pure (Fallback r)+ Reusable files -> fmap (files ++) <$> descend ancestors path cs++selectRules :: AbsFile -> ImportDirs Abs -> IO (ReuseDecision (Maybe AbsFile))+selectRules csv dirs = checkedIO "could not select rules file" (toFilePath csv) $ do+ candidates <- rulesFileCandidates csv dirs+ selected <- firstConfirmedFile (map toFilePath candidates)+ Reusable <$> traverse parseAbsFile selected++isSeparatedValues :: AbsFile -> Bool+isSeparatedValues path = map toLower (F.takeExtension (toFilePath path)) `elem` [".csv",".tsv",".ssv"]++checkArguments :: [T.Text] -> ReuseDecision ()+checkArguments args = case filter ("@" `T.isPrefixOf`) args of+ a:_ -> Fallback (FallbackReason ("indirect hledger argument " <> a) "")+ [] -> pure ()++discoverConfig :: Release -> Maybe HledgerConf -> IO (ReuseDecision (Maybe (AbsFile,Digest)))+discoverConfig release configured = checkedIO "could not discover hledger config" "hledger.conf" $ do+ let given = case configured of+ Just (GivenConf p) -> Just p+ _ -> Nothing+ case checkArguments (maybe [] (pure . T.pack) given) of+ Fallback r -> pure (Fallback r)+ Reusable () -> do+ selected <- lookupHledgerConf given+ case hledgerConfPath <$> selected of+ Nothing -> pure (pure Nothing)+ Just path | "~" `isPrefixOf` path || ".." `elem` F.splitDirectories path ->+ pure (atPath "uncertain config path" path "")+ Just path -> do+ absolute <- D.makeAbsolute path+ case parseAbsFile absolute of+ Nothing -> pure (atPath "uncertain config path" path "")+ Just absPath -> fmap Just <$> readDependency (checkConfig release) absPath++discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile,Digest)))+discoverDirectives file = checkedIO "could not discover directives" (toFilePath file) $ do+ exists <- firstConfirmedFile [toFilePath file]+ case exists of+ Nothing -> pure (pure Nothing)+ Just _ -> fmap Just <$> readDependency (checkDirectives V1996) file++readDependency :: (AbsFile -> B.ByteString -> ReuseDecision ()) -> AbsFile -> IO (ReuseDecision (AbsFile,Digest))+readDependency check path = checkedIO "could not read dependency" (toFilePath path) $ do+ bytes <- B.readFile (toFilePath path)+ pure ((path,digestBytes bytes) <$ check path bytes)++checkConfig :: Release -> AbsFile -> B.ByteString -> ReuseDecision ()+checkConfig release path bytes = do+ ls <- physicalLines path bytes+ () <$ foldM line "general" (zip [1 :: Int ..] ls)+ where+ -- Only canonical built-ins other than print have an inert section. Addons,+ -- aliases, abbreviations and names introduced in another release can have+ -- different scope and must be passed through to the ordinary converter.+ inactive = T.words $ "accounts activity add aregister balance balancesheet balancesheetequity cashflow check close codes commodities descriptions diff files help import incomestatement notes payees prices register repl rewrite roi run setup stats tags " <>+ case release of+ V1524 -> "commands demo test"+ V1996 -> "get holdings transactions"+ OtherSupported -> ""+ line section (n,raw)+ | T.null body || "#" `T.isPrefixOf` body = pure section+ | release /= V1996, "[" `T.isPrefixOf` body, T.stripStart raw /= raw = unsupported n body+ | Just start <- T.stripPrefix "[" body = case T.breakOn "]" start of+ (s,end) | not (T.null end), not ("#" `T.isInfixOf` s), let name = T.strip s,+ name `elem` ("general":"print":inactive) -> option n name (T.drop 1 end)+ _ -> unsupported n body+ | otherwise = option n section body+ where body = T.strip raw+ option n section arg+ | section `elem` inactive = pure section+ | T.null setting = pure section+ | section == "print" && release /= V1524 && setting == "--layout=hledger1" = pure section+ | otherwise = unsupported n setting+ where setting = T.strip (T.takeWhile (/= '#') arg)+ unsupported n value = atPath "unsupported hledger config" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)++checkDirectives :: Release -> AbsFile -> B.ByteString -> ReuseDecision ()+checkDirectives _ path bytes = do+ ls <- physicalLines path (dropUtf8Bom bytes)+ case [(n,T.strip l) | (n,l) <- zip [1 :: Int ..] ls, not (allowed l)] of+ [] -> pure ()+ (n,value):_ -> atPath "unsupported journal directive" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)+ where+ allowed raw+ | T.null stripped || T.isPrefixOf ";" stripped || T.isPrefixOf "#" stripped || T.isPrefixOf "*" stripped = True+ | Just (c,_) <- T.uncons raw, isSpace c = False+ | otherwise = case T.words (T.takeWhile (/= ';') raw) of+ "account":xs -> not (null xs) && all (T.all accountChar) xs+ "commodity":xs -> amount xs+ "P":date:symbol:xs -> fullDate date && commodity symbol && amount xs+ _ -> False+ where stripped = T.strip raw+ accountChar c = isAlphaNum c || c `elem` (":_-/.'’" :: String)+ commodity s = not (T.null s) && T.all (\c -> isAlpha c || generalCategory c == CurrencySymbol) s+ amount [a,b] = (number a && commodity b) || (commodity a && number b)+ amount [a] =+ let (symbol,rest) = T.span (\c -> isAlpha c || generalCategory c == CurrencySymbol) a+ (value,suffix) = T.span (\c -> isDigit c || c `elem` (".,+-" :: String)) a+ in (commodity symbol && number rest) || (number value && commodity suffix)+ amount _ = False+ number s = case T.uncons (T.dropWhile (\c -> c == '+' || c == '-') s) of+ Just (c,rest) -> digit c && T.all (\x -> digit x || x == ',' || x == '.') rest+ _ -> False+ digit c = isAscii c && isDigit c+ fullDate s = case T.split (\c -> c == '-' || c == '/') s of+ [y,m,d] -> map T.length [y,m,d] == [4,2,2] && all (T.all digit) [y,m,d]+ _ -> False++data Converter = Converter+ { converterRelease :: Release+ , converterExecutable :: AbsFile+ , converterStamp :: FileStamp+ , converterKey :: [T.Text]+ } deriving (Eq, Show)++newConverterProbe :: RT.RuntimeOptions -> IO (IO (ReuseDecision Converter))+newConverterProbe opts = converterProbeWith (RT.hledgerExecutable opts) probeHledgerVersion++-- The expensive executable hash and version probe are shared by workers only+-- while the resolved executable and its handle-derived status still agree.+converterProbeWith :: AbsFile -> (AbsFile -> IO (ExitCode,T.Text)) -> IO (IO (ReuseDecision Converter))+converterProbeWith executable getVersion = do+ context <- checkedIO "could not identify the conversion context" original conversionContext+ cache <- newMVar Nothing+ pure $ case context of+ Fallback r -> pure (Fallback r)+ Reusable environment -> checkedIO "could not identify hledger executable" original $ do+ resolved <- D.canonicalizePath original >>= parseAbsFile+ let target = toFilePath resolved+ wrapped <- launcher original target+ if wrapped+ then pure (atPath "hledger launcher" original "select the real executable with --hledger-path")+ else modifyMVar cache $ \old -> do+ stamp <- statFile resolved+ case old of+ Just (p,s,c) | p == resolved && s == stamp -> pure (old, c)+ _ -> do+ observed <- observeFile resolved+ (status,version) <- getVersion resolved+ after <- statFile resolved+ live <- D.canonicalizePath original+ let key = [T.pack original,T.pack target,digestHex (observationDigest observed),version] ++ environment+ result | after /= stamp || fileStamp observed /= stamp || not (F.equalFilePath live target) =+ atPath "hledger executable changed" original ""+ | status /= ExitSuccess = atPath "could not identify hledger version" original ""+ | otherwise = case supportedRelease version of+ Just release -> Reusable (Converter release resolved stamp key)+ Nothing -> Fallback (FallbackReason ("unsupported hledger version " <> version) (T.pack original))+ pure (Just (resolved,stamp,result),result)+ where original = toFilePath executable++conversionContext :: IO (ReuseDecision [T.Text])+conversionContext = do+ offset <- timeZoneMinutes <$> getCurrentTimeZone+ locale <- show <$> getLocaleEncoding+ fs <- show <$> getFileSystemEncoding+ foreignEnc <- show <$> getForeignEncoding+#ifdef mingw32_HOST_OS+ pages <- sequence [show <$> getConsoleCP, show <$> getConsoleOutputCP, show <$> getACP]+#else+ let pages = []+#endif+ -- Rules and directives are inspected from their exact UTF-8 snapshot bytes.+ -- A different reader encoding could interpret a byte sequence as another path.+ pure $ if map toLower locale `elem` ["utf-8","utf8","cp65001"]+ then Reusable (map T.pack ([Sys.os,Sys.arch,show offset,locale,fs,foreignEnc] ++ pages))+ else Fallback (FallbackReason ("unsupported conversion encoding " <> T.pack locale) "")++supportedRelease :: T.Text -> Maybe Release+supportedRelease text = do+ let number = T.takeWhile (/= '-') (hledgerVersionNumber text)+ component part = case TR.decimal part of+ Right (n, rest) | T.null rest -> Just (n :: Integer)+ _ -> Nothing+ version <- traverse component (T.splitOn "." number)+ case version of+ [1,52,4] -> Just V1524+ [1,99,6] -> Just V1996+ _ | version >= [1,52,4] -> Just OtherSupported+ | otherwise -> Nothing++launcher :: FilePath -> FilePath -> IO Bool+launcher original target+ | any (\p -> map toLower (F.takeExtension p) `elem` [".cmd",".bat"]) [original,target] = pure True+ | otherwise = do+ shim <- firstConfirmedFile [F.replaceExtension p "shim" | p <- [original,target], map toLower (F.takeExtension p) == ".exe"]+ case shim of+ Just _ -> pure True+ Nothing -> do+ header <- withBinaryFile target ReadMode (flip B.hGet 4)+ pure $ not (B.take 2 header == "MZ" || header `elem` nativeMagic)+ where+ -- ELF, Mach-O (both byte orders) and universal Mach-O (32- and 64-bit).+ nativeMagic = map B.pack [[127,69,76,70],[0xfe,0xed,0xfa,0xce],[0xce,0xfa,0xed,0xfe],+ [0xfe,0xed,0xfa,0xcf],[0xcf,0xfa,0xed,0xfe],[0xca,0xfe,0xba,0xbe],[0xbe,0xba,0xfe,0xca],+ [0xca,0xfe,0xba,0xbf],[0xbf,0xba,0xfe,0xca]]
+ src/Hledger/Flow/Import/Fingerprint.hs view
@@ -0,0 +1,136 @@+{-# LANGUAGE BangPatterns #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE GeneralizedNewtypeDeriving #-}++module Hledger.Flow.Import.Fingerprint+ ( Digest (..), FileObservation (..), FileStamp (..),+ copyDigest, digestBytes, digestFile, observeFile, statFile+ ) where++import qualified Crypto.Hash.SHA256 as SHA+import Data.Aeson (FromJSON, ToJSON)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Builder as Builder+import qualified Data.ByteString.Lazy as BL+import Data.Text (Text)+import Data.Text.Encoding (decodeLatin1)+import Hledger.Flow.PathHelpers (AbsFile)+import Path (toFilePath)+import System.IO (Handle, IOMode (ReadMode, WriteMode), withBinaryFile)+#ifdef mingw32_HOST_OS+import qualified System.Win32.File as Win+import System.Win32.Time (FILETIME (..))+import System.Win32.Types (withHandleToHANDLE)+#else+import Control.Exception (bracket, bracketOnError)+import System.IO (hClose)+import qualified System.Posix.Files as Posix+import qualified System.Posix.IO as Posix+#endif+#ifdef darwin_HOST_OS+import Foreign.C.Error (throwErrnoPathIfMinus1_)+import Foreign.C.Types (CInt(..))+import Foreign.Marshal.Alloc (allocaBytes)+import Foreign.Ptr (Ptr, castPtr)+import Data.Word (Word8)+import System.Posix.Types (Fd(..))++foreign import ccall safe "hf_sha256_fd"+ sha256Fd :: Fd -> Ptr Word8 -> IO CInt+#endif++-- | Only the content digest is persisted across invocations.+newtype Digest = Digest {digestHex :: Text}+ deriving (Eq, Ord, Show, FromJSON, ToJSON)++data FileStamp = FileStamp+ { stampIdentity :: !(Integer, Integer),+ stampSize :: !Integer,+ stampModified :: !Rational,+ stampChanged :: !(Maybe Rational)+ }+ deriving (Eq, Show)++data FileObservation = FileObservation+ { fileStamp :: !FileStamp,+ observationDigest :: !Digest+ }+ deriving (Eq, Show)++hexDigest :: BS.ByteString -> Digest+hexDigest = Digest . decodeLatin1 . BL.toStrict . Builder.toLazyByteString+ . BS.foldr (mappend . Builder.word8HexFixed) mempty++digestBytes :: BS.ByteString -> Digest+digestBytes = hexDigest . SHA.hash++digestFile :: AbsFile -> IO Digest+digestFile path = withBinaryFile (toFilePath path) ReadMode (digestHandle (const (pure ())))++copyDigest :: AbsFile -> AbsFile -> IO Digest+copyDigest source destination =+ withBinaryFile (toFilePath source) ReadMode $ \input ->+ withBinaryFile (toFilePath destination) WriteMode $ \output ->+ digestHandle (BS.hPut output) input++digestHandle :: (BS.ByteString -> IO ()) -> Handle -> IO Digest+digestHandle consume handle = loop SHA.init+ where+ loop !context = do+ chunk <- BS.hGetSome handle (64 * 1024)+ if BS.null chunk+ then pure $! hexDigest (SHA.finalize context)+ else consume chunk >> loop (SHA.update context chunk)++-- | Reject an observation if the open file changed while it was being hashed.+observeFile :: AbsFile -> IO FileObservation+observeFile path = withOpenFile path $ \readDigest readStamp -> do+ before <- readStamp+ digest <- readDigest+ after <- readStamp+ if before == after+ then pure (FileObservation after digest)+ else ioError (userError ("File changed while hashing " ++ toFilePath path))++statFile :: AbsFile -> IO FileStamp+statFile path = withOpenFile path $ \_ readStamp -> readStamp++-- Both actions operate on one open file even if its path is replaced.+withOpenFile :: AbsFile -> (IO Digest -> IO FileStamp -> IO a) -> IO a+#ifdef mingw32_HOST_OS+withOpenFile path action =+ withBinaryFile (toFilePath path) ReadMode $ \handle -> action (digestHandle (const (pure ())) handle) $+ withHandleToHANDLE handle $ \native -> do+ info <- Win.getFileInformationByHandle native+ let FILETIME ticks = Win.bhfiLastWriteTime info+ pure FileStamp+ { stampIdentity = (toInteger (Win.bhfiVolumeSerialNumber info), toInteger (Win.bhfiFileIndex info)),+ stampSize = toInteger (Win.bhfiSize info),+ stampModified = fromIntegral ticks / 10000000,+ stampChanged = Nothing+ }+#else+withOpenFile path action = bracket acquire (hClose . snd) $ \opened@(fd, _) ->+ action (readDigest opened) $ do+ info <- Posix.getFdStatus fd+ pure FileStamp+ { stampIdentity = (toInteger (Posix.deviceID info), toInteger (Posix.fileID info)),+ stampSize = toInteger (Posix.fileSize info),+ stampModified = toRational (Posix.modificationTimeHiRes info),+ stampChanged = Just (toRational (Posix.statusChangeTimeHiRes info))+ }+ where+#ifdef darwin_HOST_OS+ readDigest (fd, _) = allocaBytes 32 $ \out -> do+ throwErrnoPathIfMinus1_ "hash open file" (toFilePath path) (sha256Fd fd out)+ hexDigest <$> BS.packCStringLen (castPtr out, 32)+#else+ readDigest (_, handle) = digestHandle (const (pure ())) handle+#endif+ -- fdToHandle transfers ownership. Keep the borrowed fd for fstat until hClose.+ acquire = bracketOnError+ (Posix.openFd (toFilePath path) Posix.ReadOnly Posix.defaultFileFlags {Posix.cloexec = True})+ Posix.closeFd+ (\fd -> do handle <- Posix.fdToHandle fd+ pure (fd, handle))+#endif
src/Hledger/Flow/Import/IncludeFiles.hs view
@@ -154,12 +154,18 @@ let priceFiles = map ((((FilePath.takeDirectory file </> "..") </> "prices") </> dirprefix) </>) prices let extraFiles = suffixFiles ++ suffixDirFiles ++ priceFiles filtered <- filterPaths doesFileExist extraFiles- let relative = relativeToBase' (Path.toFilePath base)+ let relative = T.pack . withoutDotDot . relativeToBase' (Path.toFilePath base)+ list = T.intercalate ", " . map relative+ missing = filter (`notElem` filtered) extraFiles logMsg =- "Looking for possible extra include files for '" <> T.pack (relative file)- <> "' among these " <> T.pack (show (length extraFiles)) <> " options: "- <> T.pack (show (map (T.pack . relative) extraFiles))- <> ". Found " <> T.pack (show (length filtered)) <> ": "- <> T.pack (show (map (T.pack . relative) filtered))+ "Extra files for " <> relative file <> ": "+ <> T.intercalate "; " (["found " <> list filtered | not (null filtered)] ++ ["not found: " <> list missing | not (null missing)]) logMessage logMsg return $ Map.fromList [(file, filtered)]++-- | @import/john/../prices@ → @import/prices@, for display.+withoutDotDot :: FilePath -> FilePath+withoutDotDot = FilePath.joinPath . reverse . foldl step [] . FilePath.splitDirectories+ where+ step (dir : dirs) ".." | dir /= ".." = dirs+ step dirs part = part : dirs
+ src/Hledger/Flow/Import/Incremental.hs view
@@ -0,0 +1,237 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE TupleSections #-}++module Hledger.Flow.Import.Incremental+ ( IncrementalRun, withIncrementalRun, importStatement ) where++import Control.Concurrent.STM (TChan)+import Control.Exception (finally)+import Control.Monad (unless, when)+import Data.IORef+import qualified Data.Map.Strict as Map+import Data.Maybe (maybeToList)+import qualified Data.Set as Set+import qualified Data.Text as T+import Hledger.Flow.BaseDir (relativeToBase)+import Hledger.Flow.Common (directivesFile, writeStagedText)+import Hledger.Flow.DocHelpers+import Hledger.Flow.Cache (initializeCache)+import Hledger.Flow.Import.Cache+import Hledger.Flow.Import.Conversion+import Hledger.Flow.Import.Dependencies+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.Inventory (generatedOutput)+import Hledger.Flow.Import.Snapshot+import Hledger.Flow.Import.Types+import Hledger.Flow.Logging+import Hledger.Flow.PathHelpers (AbsFile, withStagedOutput)+import Hledger.Flow.RuntimeOptions+import Hledger.Flow.Types (LogMessage)+import qualified Path as P+import qualified Path.IO as P+import qualified System.Directory as D+import System.IO.Error (tryIOError)+import qualified System.Info as Sys++-- Keep one path only when every occurrence of a reason names that file.+data Notices = Notices !(Set.Set AbsFile) !(Map.Map T.Text (Maybe T.Text))+data IncrementalRun = IncrementalRun SnapshotRun (IO (ReuseDecision Converter)) (IORef Notices)++withIncrementalRun :: RuntimeOptions -> TChan LogMessage -> Int -> (IncrementalRun -> IO a) -> IO a+withIncrementalRun opts ch count action = withSnapshotRun (baseDir opts) $ \snapshots -> do+ reasons <- newIORef (Notices Set.empty Map.empty)+ probe <- newConverterProbe opts+ when (count > 0) $ do+ initialized <- tryIOError $ initializeCache (baseDir opts)+ either (channelErrLn ch . ("Could not initialize incremental cache: " <>) . T.pack . show) pure initialized+ finally (action (IncrementalRun snapshots probe reasons)) $ do+ Notices affected warnings <- readIORef reasons+ unless (Map.null warnings) $ do+ let scope | Set.size affected == count = "all selected statements"+ | otherwise = T.pack (show (Set.size affected)) <> " of " <> T.pack (show count) <> " selected statements"+ describe (why,path) = why <> maybe "" (": " <>) path+ channelErrLn ch $ "Incremental reuse unavailable for " <> scope <> ": "+ <> T.intercalate ", " (map describe (Map.toAscList warnings)) <> ". See " <> docURL IncrementalImportFallback++data HookInputs = HookInputs AbsFile FileObservation AbsFile FileObservation+ deriving (Eq, Show)++observeHook :: AbsFile -> AbsFile -> IO (ReuseDecision HookInputs)+observeHook script source = checkedIO "could not check script inputs" (P.toFilePath script) $ do+ runnable <- if Sys.os == "mingw32" then pure True else D.executable <$> D.getPermissions (P.toFilePath script)+ if runnable+ then Reusable <$> (HookInputs script <$> observeFile script <*> pure source <*> observeFile source)+ else pure $ Fallback (OrdinaryInputError "script is not executable" (pathText script))++hookInputsKey :: ImportDirs P.Abs -> FilePath -> T.Text -> HookInputs -> StageInputs+hookInputsKey dirs cwd stage (HookInputs script s input i) =+ [("stage",stage),("working directory",T.pack cwd),("hook",pathText script),+ ("hook digest",digestHex (observationDigest s)),("input",pathText input),+ ("input digest",digestHex (observationDigest i))] ++ map ("argument",) (hookDirArgs dirs)++pathText :: AbsFile -> T.Text+pathText = T.pack . P.toFilePath++data JournalInputs = JournalInputs+ { engine :: Converter+ , copiedInputs :: SnapshotInput+ , recordedInputs :: StageInputs+ , copiedDigests :: Map.Map AbsFile Digest+ , liveConstruct :: Maybe HookInputs+ } deriving (Eq, Show)++importStatement :: IncrementalRun -> RuntimeOptions -> TChan LogMessage -> ImportDirs P.Abs -> AbsFile -> IO (AbsFile,Bool,Bool)+importStatement (IncrementalRun snapshots probe reasons) opts ch dirs source = do+ cwd <- D.getCurrentDirectory+ record <- recordPath (baseDir opts) source+ old <- readRecord record+ intermediate <- generatedOutput [P.reldir|2-preprocessed|] ".csv" source+ let hook name = findHook opts ch name (accountDir dirs P.</> (if name == "preprocess" then [P.relfile|preprocess|] else [P.relfile|construct|]))+ report OrdinaryInputError{} = pure ()+ report problem = do+ let reason = reasonScope problem+ path = case problem of+ FileReason _ file _ -> Just (T.pack (relativeToBase opts file))+ _ -> Nothing+ detail = reasonDetail problem+ common a b = if a == b then a else Nothing+ atomicModifyIORef' reasons $ \(Notices files summaries) ->+ (Notices (Set.insert source files) (Map.insertWith common reason path summaries), ())+ logVerbose opts ch $ T.intercalate " " (filter (not . T.null) ([reason] ++ maybeToList path ++ [detail]))+ reuseStage stage inputs target prior = do+ miss <- stageMiss inputs target prior+ case miss of+ Nothing -> pure True+ Just why -> logVerbose opts ch (stage <> ": " <> why) >> pure False+ changed = FileReason "dependency changed during conversion" (P.toFilePath source) ""+ skipScripts = importPolicy opts == SkipUnchangedScripts+ pre <- hook "preprocess"+ (csv,preRecord,preRan) <- case pre of+ Nothing -> do+ input <- savedInput opts ch source intermediate+ pure (input,Nothing,False)+ Just script -> do+ before <- observeHook script source+ let key = hookInputsKey dirs cwd "preprocess" <$> before+ prior = old >>= preprocessRecord+ hit <- case key of+ Reusable k | skipScripts -> reuseStage "Preprocess" k intermediate prior+ _ -> do+ unless skipScripts $ logVerbose opts ch "Running preprocess script in run-scripts mode."+ pure False+ if hit then do+ logVerbose opts ch "Skipped unchanged preprocess script."+ pure (intermediate,prior,False)+ else do+ previous <- digestIfPresent intermediate+ digest <- installOutput True intermediate (preprocess opts ch script dirs source intermediate)+ when (Just digest == previous) $ logVerbose opts ch "Preprocessing ran; the CSV is unchanged."+ after <- observeHook script source+ selected <- hook "preprocess"+ usable <- case key of+ Fallback reason -> when skipScripts (report reason) >> pure Nothing+ Reusable k | selected == pre && after == before -> pure (Just (StageRecord k digest))+ | otherwise -> when skipScripts (report changed) >> pure Nothing+ pure (intermediate,usable,True)+ output <- generatedOutput [P.reldir|3-journal|] ".journal" csv+ let discover = do+ candidate <- hook "construct"+ context <- probe+ case context of+ Fallback r -> pure (Fallback r)+ Reusable converter -> checkedIO "could not verify journal inputs" (P.toFilePath csv) $ do+ config <- discoverConfig (converterRelease converter) (hledgerConf opts)+ case config of+ Fallback r -> pure (Fallback r)+ Reusable conf -> do+ let common = [("source",pathText source),("effective CSV",pathText csv),("preprocess",maybe "" pathText pre)]+ ++ map ("converter",) (converterKey converter)+ configArgs = pinnedConfig ("<config>" <$ conf)+ mk inputs files args extra construct =+ JournalInputs converter inputs (common ++ map ("hledger argument",) args ++ extra ++ [(pathText p,digestHex d) | (p,d) <- files]) (Map.fromList files) construct+ case candidate of+ Just script -> do+ observation <- observeHook script csv+ pure $ (\h -> mk (ConstructInput (fst <$> conf)) (maybeToList conf)+ (constructArguments configArgs "<journal>") (hookInputsKey dirs cwd "construct" h) (Just h)) <$> observation+ Nothing | not (isSeparatedValues csv) -> pure $ atPath "unsupported input reader" (P.toFilePath csv) ""+ Nothing -> do+ rules <- selectRules csv dirs+ case rules of+ Fallback r -> pure (Fallback r)+ Reusable Nothing -> pure $ Fallback (OrdinaryInputError "missing rules file" (pathText csv))+ Reusable (Just rf) -> do+ tree <- discoverRules (baseDir opts) (converterRelease converter) rf+ directives <- discoverDirectives (directivesFile opts)+ digest <- digestFile csv+ pure $ do+ rulesTree <- tree+ ds <- directives+ let files = (csv,digest) : rulesTree ++ maybeToList ds ++ maybeToList conf+ pure $ mk (RulesInput csv rf (fst <$> ds) (fst <$> conf)) files+ (rulesArguments configArgs "<input>" "<rules>" ("<directives>" <$ ds)) [("rules",pathText rf)] Nothing+ ordinary = do+ script <- hook "construct"+ case script of+ Nothing -> hledgerImport opts ch dirs csv output+ Just s -> withStagedOutput output (customConstruct opts ch s dirs csv output)+ prior = old >>= journalRecord+ fallback reason = do+ report reason+ case reason of+ CyclicRules _ file -> ioError $ userError ("Cyclic CSV rules include at " ++ T.unpack file)+ _ -> ordinary >> pure (Nothing,True)+ rawDecision <- discover+ let agrees inputs = case preRecord of+ Nothing -> True+ Just entry -> Just (stageOutput entry) == case liveConstruct inputs of+ Nothing -> Map.lookup csv (copiedDigests inputs)+ Just (HookInputs _ _ _ observed) -> Just (observationDigest observed)+ decision = rawDecision >>= \inputs -> if agrees inputs then pure inputs else Fallback changed+ (journal,generated) <- case decision of+ Fallback r -> fallback r+ Reusable inputs -> do+ hit <- if skipScripts || liveConstruct inputs == Nothing+ then reuseStage "Journal" (recordedInputs inputs) output prior+ else logVerbose opts ch "Running construct script in run-scripts mode." >> pure False+ if hit then do+ when (liveConstruct inputs /= Nothing) $ logVerbose opts ch "Skipped unchanged construct script."+ logVerbose opts ch "Reused the verified journal."+ pure (prior,False)+ else do+ result <- withSnapshot snapshots (converterRelease (engine inputs)) (copiedInputs inputs) $ \snapshot -> do+ let pin conf = pinnedConfig (pathText <$> conf)+ translate = translateDiagnostic snapshot+ executable = converterExecutable (engine inputs)+ outDigest <- case (copiedInputs inputs,snapshotInputs snapshot,liveConstruct inputs) of+ (RulesInput src rf ds _,RulesInput src' rf' ds' conf,_) -> do+ contents <- convertRules opts ch executable (RulesPaths src rf ds) (RulesPaths src' rf' ds') (pin conf) translate+ target <- D.canonicalizePath (P.toFilePath output) >>= P.parseAbsFile+ installOutput False target (writeStagedText target contents)+ (ConstructInput _,ConstructInput conf,Just (HookInputs s _ _ _)) ->+ installOutput False output (constructWith opts ch executable (pin conf) translate s dirs csv output)+ _ -> ioError (userError "Inconsistent incremental conversion inputs")+ again <- discover+ preNow <- hook "preprocess"+ effective <- case preNow of+ Nothing -> P.doesFileExist intermediate >>= \exists -> pure (if exists then intermediate else source)+ Just _ -> pure intermediate+ let stable = again == Reusable inputs && snapshotManifest snapshot == copiedDigests inputs && preNow == pre && effective == csv+ if stable then pure $ Just (StageRecord (recordedInputs inputs) outDigest)+ else when (skipScripts || liveConstruct inputs == Nothing) (report changed) >> pure Nothing+ case result of+ Fallback r -> fallback r+ Reusable entry -> pure (entry,True)+ -- A concurrent writer can replace the intermediate after the hook returned.+ -- A later run must verify its bytes before trusting the preprocessing entry.+ currentPre <- case preRecord of+ Just entry -> do+ actual <- digestIfPresent intermediate+ pure $ if actual == Just (stageOutput entry) then Just entry else Nothing+ Nothing -> pure Nothing+ let next = StatementRecord currentPre journal+ when (Just next /= old && (old /= Nothing || next /= StatementRecord Nothing Nothing)) $ do+ saved <- tryIOError $ saveRecord record next+ either (channelErrLn ch . ("Could not save incremental import record: " <>) . T.pack . show) pure saved+ pure (output,generated,preRan || generated)
+ src/Hledger/Flow/Import/PrivateDirectory.hs view
@@ -0,0 +1,53 @@+{-# LANGUAGE CPP #-}++module Hledger.Flow.Import.PrivateDirectory (createPrivateDirectory) where++import Control.Monad (unless)+import Hledger.Flow.PathHelpers (AbsDir)+import Path (toFilePath)+import System.FilePath (dropTrailingPathSeparator)+#ifdef mingw32_HOST_OS+import Foreign.C.String (CWString, withCWString)+import System.Win32.Types (DWORD, failWith)++foreign import ccall safe "hf_create_private_directory"+ createDirectoryWithDACL :: CWString -> IO DWORD+#else+import Control.Exception (bracket, bracketOnError)+import Data.Bits ((.&.))+import qualified System.Posix.Directory as Posix+import qualified System.Posix.Files as Posix+import qualified System.Posix.IO as Posix+import qualified System.Posix.User as Posix+#ifdef darwin_HOST_OS+import Foreign.C.Error (throwErrnoPathIfMinus1_)+import Foreign.C.Types (CInt (..))+import System.Posix.Types (Fd (..))++foreign import ccall unsafe "hf_verify_private_acl"+ verifyPrivateACL :: Fd -> IO CInt+#endif+#endif++-- | Create a fresh directory and verify privacy before any copies are made.+createPrivateDirectory :: AbsDir -> IO ()+#ifdef mingw32_HOST_OS+createPrivateDirectory path =+ withCWString (dropTrailingPathSeparator (toFilePath path)) $ \name -> do+ err <- createDirectoryWithDACL name+ unless (err == 0) (failWith ("Create private directory " ++ toFilePath path) err)+#else+createPrivateDirectory path =+ bracketOnError (Posix.createDirectory name 0o700) (const (Posix.removeDirectory name)) $ \() ->+ bracket (Posix.openFd name Posix.ReadOnly flags) Posix.closeFd $ \fd -> do+ status <- Posix.getFdStatus fd+ owner <- Posix.getEffectiveUserID+ unless (Posix.fileOwner status == owner && Posix.fileMode status .&. 0o777 == 0o700) $+ ioError (userError ("Cannot verify owner-only permissions on " ++ name))+#ifdef darwin_HOST_OS+ throwErrnoPathIfMinus1_ "Verify private directory ACL" name (verifyPrivateACL fd)+#endif+ where+ name = dropTrailingPathSeparator (toFilePath path)+ flags = Posix.defaultFileFlags {Posix.nofollow = True, Posix.directory = True, Posix.cloexec = True}+#endif
+ src/Hledger/Flow/Import/Snapshot.hs view
@@ -0,0 +1,329 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}++module Hledger.Flow.Import.Snapshot+ ( SnapshotRun, SnapshotInput (..), PreparedSnapshot, snapshotInputs, snapshotManifest,+ withSnapshotRun, withSnapshotRunUsing, withSnapshot,+ translateDiagnostic, windowsSnapshotPathFits+ ) where++import Control.Concurrent (threadDelay)+import Control.Concurrent.MVar+import Control.Exception hiding (handle)+import Control.Monad (filterM, forM_, unless, void, when)+import qualified Data.ByteString as BS+import Data.Char (isAlphaNum, isAsciiLower, isDigit)+import Data.IORef+import qualified Data.Map.Strict as Map+import qualified Data.Set as Set+import Data.Text (Text)+import qualified Data.Text as T+import Data.Word (Word32)+import GHC.IO.Handle.Lock (LockMode (ExclusiveLock), hLock, hTryLock)+import Hledger.Flow.Import.Dependencies+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.PrivateDirectory+import Hledger.Flow.PathHelpers (AbsDir, AbsFile)+import Numeric (showHex)+import Path+import qualified Path.IO as P+import qualified System.Directory as D+import qualified System.FilePath as FP+import System.IO+#ifndef mingw32_HOST_OS+import qualified System.Posix.IO as Posix+#endif+import Text.Read (readMaybe)+import System.IO.Error++data SnapshotInput+ = RulesInput AbsFile AbsFile (Maybe AbsFile) (Maybe AbsFile) -- data, rules, directives, config+ | ConstructInput (Maybe AbsFile)+ deriving (Eq, Show)++data PreparedSnapshot = PreparedSnapshot+ { snapshotInputs :: SnapshotInput,+ snapshotManifest :: Map.Map AbsFile Digest,+ originalPaths :: Map.Map AbsFile AbsFile+ }++data SnapshotRun = SnapshotRun+ { ledger :: AbsDir,+ registration :: MVar (Maybe (AbsDir, Handle)),+ nextTree :: IORef Word32,+ makePrivate :: AbsDir -> IO ()+ }++withSnapshotRun :: AbsDir -> (SnapshotRun -> IO a) -> IO a+withSnapshotRun = withSnapshotRunUsing createPrivateDirectory++withSnapshotRunUsing :: (AbsDir -> IO ()) -> AbsDir -> (SnapshotRun -> IO a) -> IO a+withSnapshotRunUsing create base use = do+ run <- SnapshotRun base <$> newMVar Nothing <*> newIORef 0 <*> pure create+ let area = snapshotArea run+ -- Failure to claim an abandoned tree leaves it for a later invocation.+ void $ tryIOError $ do+ exists <- P.doesDirExist area+ when exists $ withRegistration area $ reap area+ use run `finally` (readMVar (registration run) >>= mapM_ (releaseRun area))++snapshotArea :: SnapshotRun -> AbsDir+snapshotArea run = ledger run </> [reldir|.hledger-flow/cache/snapshots|]++-- GHC prevents two writable opens of the same file within one process. Wait+-- for that case too (concurrent imports in one process). The OS lock also+-- serializes independent processes, including while an initializer is paused.+withRegistration :: AbsDir -> IO a -> IO a+withRegistration area action = bracket open hClose $ \handle -> hLock handle ExclusiveLock >> action+ where+ file = toFilePath (area </> [relfile|registration.lock|])+ open = catchIOError (openLockFile file) $ \e ->+ if isAlreadyInUseError e then threadDelay 1000 >> open else ioError e++-- A surviving hook must not retain its killed parent's lifetime lock.+openLockFile :: FilePath -> IO Handle+#ifdef mingw32_HOST_OS+openLockFile file = openBinaryFile file ReadWriteMode+#else+openLockFile file = bracketOnError (Posix.openFd file Posix.ReadWrite flags) Posix.closeFd Posix.fdToHandle+ where flags = Posix.defaultFileFlags {Posix.creat = Just 0o666, Posix.cloexec = True}+#endif++shortName :: Char -> Word32 -> String+shortName prefix n = prefix : replicate (8 - length digits) '0' ++ digits+ where digits = showHex n ""++lockFile :: AbsDir -> FilePath+lockFile dir = toFilePath (dir </> [relfile|run.lock|])++acquireTree :: SnapshotRun -> IO AbsDir+acquireTree run = do+ dir <- modifyMVarMasked (registration run) $ \current -> case current of+ Just registered -> pure (current, fst registered)+ Nothing -> do+ registered <- catchIOError register $ \e -> reject "private staging unavailable" (T.pack (displayException e))+ pure (Just registered, fst registered)+ n <- atomicModifyIORef' (nextTree run) $ \i -> (i + 1, i)+ tree <- (dir </>) <$> parseRelDir (shortName 't' n)+ P.createDir tree+ pure tree+ where+ area = snapshotArea run+ register = do+ P.createDirIfMissing True area+ withRegistration area (createRun 0)+ createRun n = do+ dir <- (area </>) <$> parseRelDir (shortName 'r' n)+ result <- tryIOError (makePrivate run dir)+ case result of+ Left e | isAlreadyExistsError e && n < maxBound -> createRun (n + 1)+ Left e -> ioError e+ Right () -> flip onException (P.removeDirRecur dir) $+ bracketOnError (openLockFile (lockFile dir)) hClose $ \handle -> do+ hLock handle ExclusiveLock+ pure (dir, handle)++releaseRun :: AbsDir -> (AbsDir, Handle) -> IO ()+releaseRun area (dir, handle) = withRegistration area $+ (clearRun dir `finally` hClose handle) >> removeEmptyRun dir++-- Keep the ownership marker until every byte of statement data is gone. A+-- process killed during reclamation leaves a lockable tree for the next run.+clearRun :: AbsDir -> IO ()+clearRun dir = do+ entries <- D.listDirectory (toFilePath dir)+ forM_ (filter (/= "run.lock") entries) $ \entry ->+ D.removePathForcibly (toFilePath dir FP.</> entry)++removeEmptyRun :: AbsDir -> IO ()+removeEmptyRun dir = D.removeFile (lockFile dir) >> P.removeDir dir++reap :: AbsDir -> IO ()+reap area = do+ names <- D.listDirectory (toFilePath area)+ forM_ (filter isRunName names) $ \name -> void $ tryIOError $ do+ dir <- (area </>) <$> parseRelDir name+ let path = FP.dropTrailingPathSeparator (toFilePath dir)+ linked <- D.pathIsSymbolicLink path+ unless linked $ do+ let marker = lockFile dir+ exists <- D.doesFileExist marker+ when exists $ do+ link <- D.pathIsSymbolicLink marker+ unless link $ do+ claimed <- bracket (openLockFile marker) hClose $ \handle -> do+ mine <- hTryLock handle ExclusiveLock+ when mine (clearRun dir)+ pure mine+ when claimed (removeEmptyRun dir)+ where+ isRunName ('r':digits) = length digits == 8 && all hex digits+ isRunName _ = False+ hex c = c >= '0' && c <= '9' || isAsciiLower c && c <= 'f'++newtype PreparationFailure = PreparationFailure FallbackReason deriving Show+instance Exception PreparationFailure++reject :: Text -> Text -> IO a+reject reason detail = throwIO (PreparationFailure (FallbackReason reason detail))++fromDecision :: ReuseDecision a -> IO a+fromDecision (Reusable x) = pure x+fromDecision (Fallback reason) = throwIO (PreparationFailure reason)++prepare :: IO a -> IO (ReuseDecision a)+prepare io = (Reusable <$> io) `catches`+ [ Handler $ \(PreparationFailure reason) -> pure (Fallback reason),+ Handler $ \(e :: IOException) -> pure (Fallback (FallbackReason "snapshot creation failed" (T.pack (displayException e))))+ ]++withSnapshot :: SnapshotRun -> Release -> SnapshotInput -> (PreparedSnapshot -> IO a) -> IO (ReuseDecision a)+withSnapshot run version input convert = mask $ \restore -> do+ allocation <- prepare (acquireTree run)+ case allocation of+ Fallback reason -> pure (Fallback reason)+ Reusable tree -> flip finally (P.removeDirRecur tree) $ restore $ do+ result <- prepare (copyInputs (ledger run) tree version input)+ case result of+ Fallback reason -> pure (Fallback reason)+ Reusable snapshot -> Reusable <$> convert snapshot++windowsSnapshotPathFits :: FilePath -> Bool+windowsSnapshotPathFits = (< 260) . sum . map units+ where units c = if fromEnum c > 0xffff then 2 :: Int else 1++copyInputs :: AbsDir -> AbsDir -> Release -> SnapshotInput -> IO PreparedSnapshot+copyInputs base tree version input = do+ manifest <- newIORef Map.empty+ paths <- newIORef Map.empty+ completed <- newIORef Set.empty+ let mirror file = case stripProperPrefix base file of+ Nothing -> reject "snapshot input outside ledger" (T.pack (toFilePath file))+ Just rel -> pure (tree </> [reldir|l|] </> rel)+ destination source target = do+#ifdef mingw32_HOST_OS+ unless (windowsSnapshotPathFits (toFilePath target)) $+ reject "overlong Windows snapshot path" (T.pack (toFilePath target))+#endif+ P.createDirIfMissing True (parent target)+ -- Source and cache directories can have different case or Unicode+ -- equivalence rules. Never overwrite a copy made for another spelling.+ previous <- Map.lookup target <$> readIORef paths+ collision <- case previous of+ Just original -> pure (toFilePath original /= toFilePath source)+ Nothing -> D.doesPathExist (toFilePath target)+ when collision $ reject "snapshot filenames collide" (T.pack (toFilePath source))+ record source target digest = do+ previous <- Map.lookup source <$> readIORef manifest+ when (maybe False (/= digest) previous) $+ reject "dependency changed during snapshot" (T.pack (toFilePath source))+ modifyIORef' manifest (Map.insert source digest)+ modifyIORef' paths (Map.insert target source)+ copyBytes source target = do+ destination source target+ bytes <- BS.readFile (toFilePath source)+ BS.writeFile (toFilePath target) bytes+ record source target (digestBytes bytes)+ pure bytes+ validated check source target = do+ bytes <- copyBytes source target+ () <- fromDecision (check version source bytes)+ pure target+ copyConfig source = validated checkConfig source (tree </> [reldir|c|] </> filename source)+ copyRule ancestors source = do+ target <- mirror source+ identity <- D.canonicalizePath (toFilePath source)+ when (Set.member identity ancestors || Set.size ancestors >= 256) $ reject "cyclic or excessively deep rules include" (T.pack (toFilePath source))+ done <- Set.member (toFilePath source) <$> readIORef completed+ unless done $ do+ bytes <- copyBytes source target+ let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes+ includes <- fromDecision (scan version source bytes)+ forM_ includes $ \spelling -> do+ child <- resolveRuleInclude base source spelling >>= fromDecision+ void $ copyRule (Set.insert identity ancestors) child+ modifyIORef' completed (Set.insert (toFilePath source))+ pure target+ copied <- case input of+ ConstructInput conf -> ConstructInput <$> traverse copyConfig conf+ RulesInput csv rules directives conf -> do+ dataCopy <- mirror csv+ destination csv dataCopy+ copyDigest csv dataCopy >>= record csv dataCopy+ rulesCopy <- copyRule Set.empty rules+ directivesCopy <- traverse (\source -> mirror source >>= validated checkDirectives source) directives+ RulesInput dataCopy rulesCopy directivesCopy <$> traverse copyConfig conf+ PreparedSnapshot copied <$> readIORef manifest <*> readIORef paths++-- hledger emits locations as "FILE:line:column:" (also "FILE:line:" and+-- "FILE: explanation"). Restrict replacement to its location fields.+-- Source excerpts following a Megaparsec gutter are user data.+translateDiagnostic :: PreparedSnapshot -> Text -> IO Text+translateDiagnostic snap diagnostic = T.intercalate "\n" <$> mapM translate (T.splitOn "\n" diagnostic)+ where+ translate line = do+ let (prefix, body) = splitHeader line+ candidates value = [ (T.take i value, T.drop i value)+ | (i, ':') <- zip [0..] (T.unpack value)+ , let next = T.take 1 (T.drop (i+1) value)+ , T.null next || next == " " || T.all isDigit next ]+ if FP.isAbsolute (T.unpack body)+ then firstKnown prefix line (candidates body)+ else case T.stripPrefix "\"" body of+ Just quoted -> quotedPath prefix line (candidates quoted)+ Nothing -> translateRulePosition line+ -- Some config errors quote the entire location with Haskell string escaping.+ quotedPath _ line [] = pure line+ quotedPath prefix line ((token,suffix):rest) =+ case readMaybe (T.unpack ("\"" <> token <> "\"")) of+ Just name | FP.isAbsolute name -> do+ mapped <- lookupPath (T.pack name)+ case mapped of+ Just p -> pure (prefix <> T.dropEnd 1 (T.pack (show (toFilePath p))) <> suffix)+ Nothing -> quotedPath prefix line rest+ _ -> quotedPath prefix line rest+ -- 1.99.6 appends a second location to a field-assignment diagnostic.+ -- Keep arbitrary template/record text ahead of the final location intact.+ translateRulePosition line+ | any (`T.isPrefixOf` line) ["the ", "rule ", " (overrides: "] || fieldLabel =+ let (before, body) = T.breakOnEnd " (" line+ (fileColon, lineEnd) = T.breakOnEnd ":" body+ (digits, close) = T.span isDigit lineEnd+ in if T.null before || T.null fileColon || T.null digits || close `notElem` [")", "))"]+ then pure line+ else firstKnown before line [(T.dropEnd 1 fileColon, ":" <> lineEnd)]+ | otherwise = pure line+ where+ fieldLabel = case T.stripPrefix " " line of+ Just field -> let (name, rest) = T.span isAlphaNum field+ in not (T.null name) && ":" `T.isPrefixOf` rest+ Nothing -> False+ splitHeader line =+ let (spaces, rest) = T.span (== ' ') line+ (program, message) = splitPrefix ["hledger: ", "hledger.exe: "] rest+ (label, body) = splitPrefix ["Warning: ", "Error: ", "error: ", "in "] message+ in (spaces <> program <> label, body)+ splitPrefix prefixes value = case [(p,s) | p <- prefixes, Just s <- [T.stripPrefix p value]] of+ (parts:_) -> parts+ [] -> ("",value)+ firstKnown _ original [] = pure original+ firstKnown prefix original ((token, suffix):more) = do+ mapped <- lookupPath token+ case mapped of+ Just path -> pure (prefix <> T.pack (toFilePath path) <> suffix)+ Nothing -> firstKnown prefix original more+ lookupPath token = case parseAbsFile (T.unpack token) >>= (\p -> Map.lookup p (originalPaths snap)) of+ Just path -> pure (Just path)+ Nothing -> catchIOError (byIdentity (T.unpack token)) (const (pure Nothing))+ byIdentity name = do+ actual <- D.canonicalizePath name+ -- canonicalizePath resolves symlinked ancestors and native spellings.+ -- File IDs also identify a Windows short path when its spelling differs.+ source <- parseAbsFile actual+ targetId <- stampIdentity <$> statFile source+ matches <- filterM (\p -> (== targetId) . stampIdentity <$> statFile p) (Map.keys (originalPaths snap))+ pure $ case matches of+ [p] -> Map.lookup p (originalPaths snap)+ _ -> Nothing
src/Hledger/Flow/Logging.hs view
@@ -8,14 +8,34 @@ import Control.Monad (when) import qualified Data.Text as T import qualified Data.Text.IO as T-import Data.Time.LocalTime (getZonedTime) import qualified GHC.IO.Handle.FD as H-import System.IO (hFlush)+import System.Console.ANSI (Color (..), ColorIntensity (..), ConsoleIntensity (..), ConsoleLayer (..), SGR (..), hSupportsANSIColor, setSGRCode)+import System.Environment (lookupEnv)+import System.IO (Handle, hFlush) import Hledger.Flow.Types -dummyLogger :: TChan LogMessage -> T.Text -> IO ()-dummyLogger _ _ = return ()+-- | Colour only on a terminal, and only when NO_COLOR is unset or empty.+useColor :: Handle -> IO Bool+useColor h = do+ noColor <- lookupEnv "NO_COLOR"+ supported <- hSupportsANSIColor h+ return (supported && maybe True null noColor) +paint :: Bool -> [SGR] -> T.Text -> T.Text+paint False _ txt = txt+paint True sgr txt = T.pack (setSGRCode sgr) <> txt <> resetCode++resetCode :: T.Text+resetCode = T.pack (setSGRCode [Reset])++green, red, yellow, cyan, bold, dim :: Bool -> T.Text -> T.Text+green c = paint c [SetColor Foreground Vivid Green]+red c = paint c [SetColor Foreground Vivid Red]+yellow c = paint c [SetColor Foreground Vivid Yellow]+cyan c = paint c [SetColor Foreground Vivid Cyan]+bold c = paint c [SetConsoleIntensity BoldIntensity]+dim c = paint c [SetConsoleIntensity FaintIntensity]+ channelOut :: TChan LogMessage -> T.Text -> IO () channelOut ch txt = atomically $ writeTChan ch $ StdOut txt @@ -28,30 +48,50 @@ channelErrLn :: TChan LogMessage -> T.Text -> IO () channelErrLn ch txt = channelErr ch (txt <> "\n") +-- | Mark the first line as hledger-flow's own, so it stands apart from what other programs print.+-- Callers indent any further lines.+verboseLine :: T.Text -> IO T.Text+verboseLine msg = do+ color <- useColor H.stderr+ return (dim color "hledger-flow:" <> " " <> msg <> "\n")+ logToChannel :: TChan LogMessage -> T.Text -> IO ()-logToChannel ch msg = do- ts <- timestampPrefix msg- channelErrLn ch ts+logToChannel ch msg = verboseLine msg >>= channelErr ch -timestampPrefix :: T.Text -> IO T.Text-timestampPrefix txt = do- t <- getZonedTime- return $ T.pack (show t) <> "\thledger-flow " <> txt+-- | In verbose mode, show one item's lines together under a heading. With 'streaming' they+-- appear as they're logged; otherwise they're held back until the item is done, so items+-- that run in parallel don't interleave.+withLogBlock :: (HasVerbosity o) => o -> Bool -> TChan LogMessage -> T.Text -> (TChan LogMessage -> IO a) -> IO a+withLogBlock opts streaming ch heading action+ | not (verbose opts) = action ch+ | otherwise = do+ color <- useColor H.stderr+ line <- verboseLine (bold color heading)+ if streaming+ then channelErr ch ("\n" <> line) >> action ch+ else do+ buffer <- newTChanIO+ result <- tryJust synchronous (action buffer)+ atomically $ writeTChan ch (StdErr ("\n" <> line)) >> drain buffer+ either throwIO return result+ where+ drain buffer = tryReadTChan buffer >>= maybe (return ()) (\msg -> writeTChan ch msg >> drain buffer) +-- | Errors from the action itself, not cancellation from outside.+synchronous :: SomeException -> Maybe SomeException+synchronous err = case fromException err :: Maybe SomeAsyncException of+ Just _ -> Nothing+ Nothing -> Just err+ withConsoleLogging :: (TChan LogMessage -> IO a) -> IO a withConsoleLogging action = do ch <- newTChanIO withAsync (consoleChannelLoop ch) $ \logger -> do+ -- Drain queued diagnostics on failure, but allow Ctrl-C to cancel a blocked writer. result <- tryJust synchronous (action ch) terminateChannelLoop ch wait logger either throwIO return result- where- -- Drain queued diagnostics on failure, but allow Ctrl-C to cancel a blocked writer.- synchronous :: SomeException -> Maybe SomeException- synchronous err = case fromException err :: Maybe SomeAsyncException of- Just _ -> Nothing- Nothing -> Just err consoleChannelLoop :: TChan LogMessage -> IO () consoleChannelLoop ch = do
src/Hledger/Flow/ReleaseNotice.hs view
@@ -1,14 +1,18 @@+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-} module Hledger.Flow.ReleaseNotice- ( ReleaseResult (..), withReleaseNotice, startCheck,- finishCommand, checkRelease, renderUpdate+ ( ReleaseResult (..), withReleaseNotice, withCachedReleaseNotice,+ prepareCheck, refreshIfDue, cachedRelease,+ finishCommand, renderUpdate ) where -import Control.Concurrent.Async (async, poll)-import Control.Exception (catch, evaluate, throwIO, try)-import Control.Monad (guard, join)-import Data.Aeson (decode, decodeFileStrict, withObject, (.:), (.:?), (.!=))+import Control.Concurrent.Async (async, waitCatch)+import Control.Exception (catch, throwIO, try)+import Control.Monad (guard, join, void, when)+import Data.Aeson (FromJSON, ToJSON, decode, decodeFileStrict, encode, withObject, (.:), (.:?), (.!=)) import Data.Aeson.Types (parseMaybe) import qualified Data.ByteString.Lazy as BL import Data.Char (isDigit)@@ -16,7 +20,13 @@ import qualified Data.Text as T import qualified Data.Text.Encoding as T import qualified Data.Text.IO as T+import Data.Time (NominalDiffTime, UTCTime, diffUTCTime, getCurrentTime) import Data.Version (Version, parseVersion, showVersion)+import GHC.Generics (Generic)+import GHC.IO.Handle.Lock (LockMode (ExclusiveLock), hTryLock)+import Hledger.Flow.Cache (cacheDir, initializeCache)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, withStagedOutput)+import qualified Path as P import Hledger.Flow.Status.Render (useColor, yellow) import Network.HTTP.Client (HttpException, checkResponse, httpLbs, parseRequest, requestHeaders, responseBody, throwErrorStatusCodes) import Network.HTTP.Client.TLS (getGlobalManager)@@ -25,7 +35,7 @@ import System.Environment (getExecutablePath, lookupEnv) import System.Exit (ExitCode) import System.FilePath ((</>), takeDirectory, takeFileName)-import System.IO (hIsTerminalDevice, stderr, stdout)+import System.IO (IOMode (AppendMode), hIsTerminalDevice, stderr, stdout, withBinaryFile) import System.IO.Error (catchIOError) import System.Timeout (timeout) import Text.ParserCombinators.ReadP (eof, readP_to_S)@@ -33,8 +43,63 @@ data ReleaseResult = UpToDate | UpdateAvailable !Version !T.Text | CheckUnavailable deriving (Eq, Show) -withReleaseNotice :: IO () -> IO ()-withReleaseNotice command = do+-- The cached version is compared with the binary reading it on every run.+data Observation = Observation+ { latestVersion :: !Version+ , checkedAt :: !UTCTime+ , formulaConfirmed :: !Bool+ } deriving (Generic, FromJSON, ToJSON)++data ReleaseCache = ReleaseCache+ { lastSuccess :: !(Maybe Observation)+ , lastFailure :: !(Maybe UTCTime)+ } deriving (Generic, FromJSON, ToJSON)++type Fetch = String -> IO (Maybe BL.ByteString)++cacheFile :: AbsDir -> AbsFile+cacheFile base = cacheDir base P.</> [P.relfile|release-check.json|]++readCache :: AbsDir -> IO ReleaseCache+readCache base = do+ record <- decodeFileStrict (P.toFilePath (cacheFile base)) `catchIOError` const (pure Nothing)+ pure (fromMaybe (ReleaseCache Nothing Nothing) record)++saveCache :: AbsDir -> ReleaseCache -> IO ()+saveCache base record = withStagedOutput (cacheFile base) $ \stage ->+ BL.writeFile (P.toFilePath stage) (encode record)++refreshDue :: UTCTime -> ReleaseCache -> Bool+refreshDue now record =+ not (maybe False (recent (24 * 60 * 60) . checkedAt) (lastSuccess record)+ || maybe False (recent (15 * 60)) (lastFailure record))+ where+ recent :: NominalDiffTime -> UTCTime -> Bool+ recent limit stamp = let age = diffUTCTime now stamp in age >= 0 && age < limit++cachedRelease :: AbsDir -> IO FilePath -> IO ReleaseResult+cachedRelease base executable = readCache base >>= releaseResult executable++releaseResult :: IO FilePath -> ReleaseCache -> IO ReleaseResult+releaseResult executable record = case lastSuccess record of+ Nothing -> pure CheckUnavailable+ Just observation+ | latestVersion observation <= version -> pure UpToDate+ | otherwise -> do+ owned <- if formulaConfirmed observation then brewOwned executable else pure False+ let advice = if owned+ then "Upgrade with: brew update && brew upgrade ferntrail/tap/hledger-flow"+ else "https://github.com/apauley/hledger-flow/releases/latest"+ pure (UpdateAvailable (latestVersion observation) advice)++withReleaseNotice :: AbsDir -> IO () -> IO ()+withReleaseNotice base = withNotice (prepareCheck base getCurrentTime fetch getExecutablePath)++withCachedReleaseNotice :: AbsDir -> IO () -> IO ()+withCachedReleaseNotice base = withNotice (pure (cachedRelease base getExecutablePath))++withNotice :: IO (IO ReleaseResult) -> IO () -> IO ()+withNotice prepare command = do finish <- setup `catchIOError` const (pure (pure ())) finishCommand finish command where@@ -43,21 +108,42 @@ outTTY <- hIsTerminalDevice stdout errTTY <- hIsTerminalDevice stderr if disabled /= Just "1" && outTTY && errTTY- then startCheck (checkRelease fetch getExecutablePath)+ then do+ result <- prepare+ pure (result >>= display) else pure (pure ())+ display (UpdateAvailable latest advice) = do+ color <- useColor stderr+ T.hPutStrLn stderr ("\n" <> renderUpdate color latest advice)+ display _ = pure () --- The CLI owns the worker. Let process exit handle any pending network cleanup.-startCheck :: IO ReleaseResult -> IO (IO ())-startCheck check = do- worker <- async (check >>= evaluate)- pure $ do- result <- poll worker- case result of- Just (Right (UpdateAvailable latest advice)) -> do- color <- useColor stderr- T.hPutStrLn stderr ("\n" <> renderUpdate color latest advice)- _ -> pure ()+prepareCheck :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> IO (IO ReleaseResult)+prepareCheck base clock get executable = do+ record <- readCache base+ now <- clock+ if refreshDue now record then do+ initializeCache base+ worker <- async (refreshIfDue base clock get executable)+ pure $ do+ case lastSuccess record of+ Nothing -> void (timeout 250000 (waitCatch worker))+ Just _ -> pure ()+ cachedRelease base executable+ else pure (releaseResult executable record) +-- Called after initializeCache. Keep the lock file in place so all processes+-- contend on the same file even as the JSON record is atomically replaced.+refreshIfDue :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> IO ()+refreshIfDue base clock get executable = do+ let lock = cacheDir base P.</> [P.relfile|release-check.lock|]+ withBinaryFile (P.toFilePath lock) AppendMode $ \handle -> do+ acquired <- hTryLock handle ExclusiveLock+ when acquired $ do+ -- A different process may have published since our first read.+ current <- readCache base+ started <- clock+ when (refreshDue started current) (refreshCache base clock get executable current)+ renderUpdate :: Bool -> Version -> T.Text -> T.Text renderUpdate color latest advice = " " <> yellow color ("↑ " <> availableMessage latest) <> "\n " <> advice@@ -90,30 +176,30 @@ ignoreHttpError :: HttpException -> IO (Maybe BL.ByteString) ignoreHttpError _ = pure Nothing -checkRelease :: (String -> IO (Maybe BL.ByteString)) -> IO FilePath -> IO ReleaseResult-checkRelease get executable = do+refreshCache :: AbsDir -> IO UTCTime -> Fetch -> IO FilePath -> ReleaseCache -> IO ()+refreshCache base clock get executable previous = do result <- timeout 3000000 (get "https://api.github.com/repos/apauley/hledger-flow/releases/latest")+ now <- clock case join result >>= releaseVersion of- Just latest | latest > version -> do- confirmed <- timeout 1000000 (homebrewOffers latest `catchIOError` const (pure False))- let advice = if confirmed == Just True- then "Upgrade with: brew update && brew upgrade ferntrail/tap/hledger-flow"- else "https://github.com/apauley/hledger-flow/releases/latest"- pure (UpdateAvailable latest advice)- Just _ -> pure UpToDate- Nothing -> pure CheckUnavailable- where- homebrewOffers latest = do- path <- executable >>= canonicalizePath- owned <- ownsHomebrewBinary path- if owned then do- formula <- get "https://raw.githubusercontent.com/FernTrail/homebrew-tap/main/Formula/hledger-flow.rb"- let tagLine = "tag = \"v" <> renderVersion latest <> "\""- pure $ fromMaybe False $ do- body <- formula- contents <- either (const Nothing) Just (T.decodeUtf8' (BL.toStrict body))- pure $ tagLine `elem` map T.strip (T.lines contents)- else pure False+ Nothing -> saveCache base previous { lastFailure = Just now }+ Just latest -> do+ let save confirmed = saveCache base (ReleaseCache (Just (Observation latest now confirmed)) Nothing)+ -- A short command can exit during the optional formula lookup.+ save False+ when (latest > version) $ do+ owned <- brewOwned executable+ when owned $ do+ response <- timeout 1000000 (get "https://raw.githubusercontent.com/FernTrail/homebrew-tap/main/Formula/hledger-flow.rb")+ let published = do+ body <- join response+ contents <- either (const Nothing) Just (T.decodeUtf8' (BL.toStrict body))+ let tagLine = "tag = \"v" <> renderVersion latest <> "\""+ pure (tagLine `elem` map T.strip (T.lines contents))+ when (published == Just True) (save True)++brewOwned :: IO FilePath -> IO Bool+brewOwned executable =+ (executable >>= canonicalizePath >>= ownsHomebrewBinary) `catchIOError` const (pure False) releaseVersion :: BL.ByteString -> Maybe Version releaseVersion bytes = do
src/Hledger/Flow/Reports.hs view
@@ -47,7 +47,7 @@ skippedText = if skipped == 0 then "" else " Skipped " <> plural skipped "empty report" <> "." channelOutLn ch $ "Generated " <> plural (length (rights reports)) "report" <> " in reports/." <> skippedText channelOutLn ch "Each report starts with the hledger command that made it. Copy that line to tailor the report."- logVerbose opts ch ("Reports took " <> T.pack (show diff))+ logVerbose opts ch ("Reports finished in " <> showDuration diff) generateReports' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> IO [Either AbsFile AbsFile] generateReports' opts ch = do@@ -115,31 +115,33 @@ generateReport opts ch journals year baseOutDir [relfile|transfer-balance.txt|] reportArgs (\txt -> (length $ T.lines txt) > 4) generateReport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [FilePath] -> Integer -> AbsDir -> RelFile -> [T.Text] -> (T.Text -> Bool) -> IO (Either AbsFile AbsFile)-generateReport opts ch journals year reportsDir fileName args successCheck = do+generateReport opts logCh journals year reportsDir fileName args successCheck = do yearDir <- parseRelDir (show year) let outputFile = reportsDir </> yearDir </> fileName- PathIO.createDirIfMissing True (parent outputFile)- let relativeJournals = map (relativeToBase opts) journals- let relativeOutputFile = relativeToBase opts (toFilePath outputFile)- let confArgs = hledgerConfArgs opts- info <- hledgerInfo opts- let fileArgs = concatMap (\file -> ["--file", hledgerFileArgument info file])- let periodArgs = ["--period", T.pack (show year)]- let reportArgs = confArgs ++ fileArgs journals ++ periodArgs ++ args- let reportDisplayArgs = confArgs ++ fileArgs relativeJournals ++ periodArgs ++ args- let hledger = T.pack (toFilePath (FlowTypes.hlPath info))- let cmdLabel = "hledger " <> showCmdArgs reportDisplayArgs- ((_, stdOut, _), _) <- timeAndExitOnErr opts ch cmdLabel dummyLogger channelErr (hledger, reportArgs, Nothing)- if (successCheck stdOut)- then do- T.writeFile (toFilePath outputFile) (cmdLabel <> "\n\n" <> stdOut)- logVerbose opts ch $ "Wrote " <> T.pack relativeOutputFile- return $ Right outputFile- else do- logVerbose opts ch $ "Skipped empty report " <> T.pack relativeOutputFile- exists <- PathIO.doesFileExist outputFile- when exists (PathIO.removeFile outputFile)- return $ Left outputFile+ withLogBlock opts (sequential opts) logCh ("Report " <> T.pack (relativeToBase opts (toFilePath outputFile))) $ \ch -> do+ PathIO.createDirIfMissing True (parent outputFile)+ let relativeJournals = map (relativeToBase opts) journals+ let relativeOutputFile = relativeToBase opts (toFilePath outputFile)+ let confArgs = hledgerConfArgs opts+ info <- hledgerInfo opts+ let fileArgs = concatMap (\file -> ["--file", hledgerFileArgument info file])+ let periodArgs = ["--period", T.pack (show year)]+ let reportArgs = confArgs ++ fileArgs journals ++ periodArgs ++ args+ let reportDisplayArgs = confArgs ++ fileArgs relativeJournals ++ periodArgs ++ args+ let hledger = T.pack (toFilePath (FlowTypes.hlPath info))+ let cmdLabel = "hledger " <> showCmdArgs reportDisplayArgs+ let external = External "hledger" ("write " <> T.pack relativeOutputFile) [hledger : reportArgs] []+ stdOut <- runExternal opts ch external False (hledger, reportArgs, Nothing)+ if (successCheck stdOut)+ then do+ T.writeFile (toFilePath outputFile) (cmdLabel <> "\n\n" <> stdOut)+ logVerbose opts ch $ "Wrote " <> T.pack relativeOutputFile+ return $ Right outputFile+ else do+ logVerbose opts ch $ "Skipped empty report " <> T.pack relativeOutputFile+ exists <- PathIO.doesFileExist outputFile+ when exists (PathIO.removeFile outputFile)+ return $ Left outputFile ownerParameters :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [RelDir] -> IO [ReportParams] ownerParameters opts ch owners = do
src/Hledger/Flow/RuntimeOptions.hs view
@@ -6,11 +6,17 @@ import Hledger.Flow.Types (BaseDir, HledgerConf, HledgerInfo (..), RunDir) import qualified Hledger.Flow.Types as Types +data ImportPolicy = Full | TimestampOnly | RunScripts | SkipUnchangedScripts+ deriving (Eq, Show)++onlyNewFiles :: RuntimeOptions -> Bool+onlyNewFiles = (== TimestampOnly) . importPolicy+ data RuntimeOptions = RuntimeOptions { baseDir :: BaseDir, importRunDir :: RunDir, importStartYear :: Maybe Integer,- onlyNewFiles :: Bool,+ importPolicy :: ImportPolicy, hfVersion :: T.Text, hledgerExecutable :: AbsFile, hledgerVersion :: IO T.Text,
src/Hledger/Flow/Status/Render.hs view
@@ -29,27 +29,10 @@ import Hledger.Flow.Status import Hledger.Flow.Types (HledgerConf, HledgerInfo (..)) import Path (toFilePath)-import System.Console.ANSI (Color (..), ColorIntensity (..), ConsoleIntensity (..), ConsoleLayer (..), SGR (..), hSupportsANSIColor, setSGRCode)-import System.Environment (lookupEnv)+import Hledger.Flow.Logging (bold, green, red, useColor, yellow) import qualified System.FilePath as FilePath-import System.IO (Handle, stdout)---- | Colour only on a terminal, and only when NO_COLOR is unset or empty.-useColor :: Handle -> IO Bool-useColor h = do- noColor <- lookupEnv "NO_COLOR"- supported <- hSupportsANSIColor h- return (supported && maybe True null noColor)--paint :: Bool -> [SGR] -> T.Text -> T.Text-paint False _ txt = txt-paint True sgr txt = T.pack (setSGRCode sgr) <> txt <> T.pack (setSGRCode [Reset])+import System.IO (stdout) -green, red, yellow, bold :: Bool -> T.Text -> T.Text-green c = paint c [SetColor Foreground Vivid Green]-red c = paint c [SetColor Foreground Vivid Red]-yellow c = paint c [SetColor Foreground Vivid Yellow]-bold c = paint c [SetConsoleIntensity BoldIntensity] baseText :: LedgerStatus -> FilePath baseText = FilePath.dropTrailingPathSeparator . toFilePath . lsBaseDir
src/Hledger/Flow/Types.hs view
@@ -4,7 +4,6 @@ import qualified Data.Text as T import Hledger.Flow.PathHelpers-import Data.Time.Clock (NominalDiffTime) import System.Exit (ExitCode) import System.IO (Handle) @@ -15,8 +14,6 @@ data LogMessage = StdOut T.Text | StdErr T.Text | Terminate deriving (Show) type FullOutput = (ExitCode, T.Text, T.Text)--type FullTimedOutput = (FullOutput, NominalDiffTime) type ProcInput = (T.Text, [T.Text], Maybe Handle)
test/Common/Integration.hs view
@@ -9,7 +9,6 @@ import Control.Exception (bracket, bracket_, finally) import Control.Monad (forM_, unless) import qualified Data.List as List (intercalate, isInfixOf, isPrefixOf)-import qualified Data.Text as T import qualified Data.Text.IO as T import Data.Time.Clock (UTCTime(..), addUTCTime, secondsToDiffTime) import Data.Time.Calendar (fromGregorian)@@ -102,19 +101,16 @@ file = base Path.</> [Path.relfile|statement.csv|] ch <- newTChanIO let check expected message = do- exists <- verboseTestFile opts ch file+ exists <- verboseTestFile opts ch "statement" file assertEqual "file existence" expected exists entry <- atomically (tryReadTChan ch) case entry of- Just (StdErr txt) -> do- let (timestamp, body) = T.breakOn "\t" txt- assertBool "verbose log starts with a timestamp" (not (T.null timestamp))- assertEqual "verbose log uses the relative path" ("\thledger-flow " <> message) body+ Just (StdErr txt) -> assertEqual "verbose log names hledger-flow and uses the relative path" ("hledger-flow: " <> message) txt _ -> assertFailure "expected a verbose message on stderr"- check False "Looked for but did not find 'statement.csv'\n"+ check False "No statement at statement.csv\n" T.writeFile (Path.toFilePath file) ""- check True "Found 'statement.csv'\n"- _ <- verboseTestFile (defaultOpts base) ch file+ check True "Found statement: statement.csv\n"+ _ <- verboseTestFile (defaultOpts base) ch "statement" file quietEntry <- atomically (tryReadTChan ch) case quietEntry of Nothing -> return ()@@ -215,15 +211,18 @@ -- Paths come out resolved (macOS /private/var, symlinked homes), so match both spellings of HOME. testHomeDirs :: Test-testHomeDirs = TestLabel "home folder spellings" $ TestCase $ unless (Sys.os == "mingw32") $ withTestDir $ \base -> do+testHomeDirs = TestLabel "home folder spellings" $ TestCase $ withTestDir $ \base -> do let root = Path.toFilePath base- Directory.createDirectory (root </> "real")- Directory.createDirectoryLink (root </> "real") (root </> "link")- real <- Directory.canonicalizePath (root </> "real")- forM_ ([root </> "link", root </> "link/", real ++ "/"] :: [FilePath]) $ \home -> withEnv "HOME" home $ do- homes <- homeDirs- assertEqual ("resolved path with HOME=" ++ home) ("~" </> "fin") (tildePath homes (real </> "fin"))- assertEqual ("HOME's spelling with HOME=" ++ home) ("~" </> "fin") (tildePath homes (home </> "fin"))+ if Sys.os == "mingw32"+ then withEnv "USERPROFILE" root $ homeDirs >>= assertEqual "keep native profile paths absolute" []+ else do+ Directory.createDirectory (root </> "real")+ Directory.createDirectoryLink (root </> "real") (root </> "link")+ real <- Directory.canonicalizePath (root </> "real")+ forM_ ([root </> "link", root </> "link/", real ++ "/"] :: [FilePath]) $ \home -> withEnv "HOME" home $ do+ homes <- homeDirs+ assertEqual ("resolved path with HOME=" ++ home) ("~" </> "fin") (tildePath homes (real </> "fin"))+ assertEqual ("HOME's spelling with HOME=" ++ home) ("~" </> "fin") (tildePath homes (home </> "fin")) testWriteGeneratedText :: Test testWriteGeneratedText = TestLabel "write generated text" $ TestCase $ withTestDir $ \base -> do
test/Common/Unit.hs view
@@ -94,7 +94,16 @@ "trailing backslash is special unless it separates paths" ~: homeShellPath Posix homes "/home/me/a\\" ~?= if FilePath.isPathSeparator '\\' then Just (quoted ["a"]) else Nothing, "powershell specials fall back" ~: map (\c -> homeShellPath PowerShell homes ("/home/me/a" ++ [c])) "$`\"\x201c\x201d\x201e" ~?= replicate 6 Nothing,- "posix-only specials are fine in powershell" ~: homeShellPath PowerShell homes "/home/me/a!" ~?= Just (quoted ["a!"])+ "posix-only specials are fine in powershell" ~: homeShellPath PowerShell homes "/home/me/a!" ~?= Just (quoted ["a!"]),+ "posix pipeline" ~: pasteCommand Posix "/home/me/fin/" homes [["/srv/construct", "a b", "-"], ["/srv/hledger", "print"]]+ ~?= "/srv/construct 'a b' - | /srv/hledger print",+ "powershell runs a quoted program with &" ~: pasteCommand PowerShell "/home/me/fin/" homes [["/home/me/bin/pre.ps1", "Lee's"]]+ ~?= "& " <> quoted ["bin", "pre.ps1"] <> " 'Lee''s'",+ "powershell leaves a bare program alone" ~: pasteCommand PowerShell "/home/me/fin/" homes [["hledger", "print"]] ~?= "hledger print",+ "paths below the base start with ./" ~: pasteCommand Posix "/home/me/fin/" homes [["/home/me/bin/hledger", "--file", "/home/me/fin/import/my a.csv", "/home/me/fin"]]+ ~?= displayedProgram <> " --file '" <> native [".", "import", "my a.csv"] <> "' .",+ "powershell paths below the base" ~: pasteCommand PowerShell "/home/me/fin/" homes [["/home/me/fin/my hooks/pre.ps1", "/home/me/fin/a b.csv"]]+ ~?= "& '" <> native [".", "my hooks", "pre.ps1"] <> "' '" <> native [".", "a b.csv"] <> "'" ] testRelativeToBase :: Test@@ -175,6 +184,6 @@ TestList [ "long line" ~: trimToolOutput (T.replicate 400 "x") ~?= T.replicate 300 "x" <> "…", "many lines" ~: T.lines (trimToolOutput (T.unlines (map (T.pack . show) [1 .. 25 :: Int])))- ~?= map (T.pack . show) [1 .. 20 :: Int] ++ ["… (5 more lines)"],+ ~?= map (T.pack . show) [1 .. 20 :: Int] ++ ["… (5 more lines; run with --verbose to see them all)"], "short output unchanged" ~: trimToolOutput "a\nb\n" ~?= "a\nb\n" ]
test/Environment/Unit.hs view
@@ -19,6 +19,9 @@ testImportLines = TestCase $ do let out = renderSettings id (importSettings [] defaultHlInfo opts {importStartYear = Just 2024, batchSize = 20}) line l = filter ((l <> " ") `T.isPrefixOf`) out+ let files policy = [(sValue row,sFlags row) | row <- importSettings [] defaultHlInfo opts {importPolicy = policy}, sLabel row == "Files"]+ files RunScripts @?= [("incremental (run-scripts)", "--incremental")]+ files SkipUnchangedScripts @?= [("incremental (skip-unchanged-scripts)", "--incremental")] assertEqual "one line per setting" 8 (length out) assertBool "full hledger path and version" (any ((T.pack (toFilePath [relfile|path/to/hledger|]) <> " (1.2.3)") `T.isInfixOf`) (line "hledger")) assertBool "flag beside its value" (any ("2024 onwards" `T.isInfixOf`) (line "Start year") && any ("--start-year" `T.isSuffixOf`) (line "Start year"))
+ test/Incremental/Cache.hs view
@@ -0,0 +1,56 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Incremental.Cache (tests) where++import Control.Exception (IOException, try)+import Data.Maybe (isJust)+import qualified Data.Text.IO as T+import Hledger.Flow.Cache (initializeCache)+import Hledger.Flow.Import.Cache+import Hledger.Flow.Import.Fingerprint+import qualified Path as P+import qualified Path.IO as P+import Test.HUnit++tests :: Test+tests = TestLabel "incremental records" $ TestCase $ P.withSystemTempDir "records" $ \base -> do+ let source = base P.</> [P.relfile|import/alice/bank/card/1-in/2026/march.csv|]+ other = base P.</> [P.relfile|import/alice/bank/card/1-in/2026/march.tsv|]+ out = base P.</> [P.relfile|march.journal|]+ ignored = base P.</> [P.relfile|.hledger-flow/cache/.gitignore|]+ bytes = "2026-03-01 coffee\n"+ record <- recordPath base source+ another <- recordPath base other+ assertBool "the source extension distinguishes records" (another /= record)+ assertEqual "missing record" Nothing =<< readRecord record+ initializeCache base+ assertEqual "self-ignoring cache" "*\n" =<< T.readFile (P.toFilePath ignored)+ T.writeFile (P.toFilePath ignored) "mine"+ initializeCache base+ assertEqual "preserve custom ignore" "mine" =<< T.readFile (P.toFilePath ignored)+ digest <- installOutput False out $ \staged -> T.writeFile (P.toFilePath staged) bytes+ assertEqual "published digest is the staged bytes" digest =<< digestFile out+ let stage = StageRecord [("source", "march.csv")] digest+ entry = StatementRecord Nothing (Just stage)+ saveRecord record entry+ assertEqual "read published record" (Just entry) =<< readRecord record+ assertEqual "verified output" Nothing =<< stageMiss (stageInputs stage) out (Just stage)+ before <- P.getModificationTime out+ _ <- installOutput True out $ \staged -> T.writeFile (P.toFilePath staged) bytes+ assertEqual "preserve identical intermediate" before =<< P.getModificationTime out+ _ <- try (installOutput False out $ \_ -> ioError (userError "conversion failed")) :: IO (Either IOException Digest)+ assertEqual "failed conversion leaves installed output" digest =<< digestFile out+ T.writeFile (P.toFilePath out) "another importer"+ assertBool "intervening writer invalidates record" . isJust =<< stageMiss (stageInputs stage) out (Just stage)+ digestB <- installOutput False out $ \staged -> T.writeFile (P.toFilePath staged) "output from writer B"+ let stageB = StageRecord [("source", "B")] digestB+ saveRecord record (StatementRecord Nothing (Just stageB))+ -- Writer A, paused after installing A, saves its record after B installs B.+ saveRecord record entry+ late <- readRecord record+ assertBool "late publication must miss on B's output" . isJust =<< stageMiss (stageInputs stage) out (late >>= journalRecord)+ T.writeFile (P.toFilePath record) "[1,{\"preprocessRecord\":null,\"journalRecord\":null}]"+ assertEqual "discard records predating the snapshot collision check" Nothing =<< readRecord record+ T.writeFile (P.toFilePath record) "{}"+ assertEqual "invalid record" Nothing =<< readRecord record
+ test/Incremental/Dependencies.hs view
@@ -0,0 +1,185 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Incremental.Dependencies (tests) where++import Control.Monad (forM_)+import Control.Exception (bracket)+import GHC.IO.Encoding (getLocaleEncoding, setLocaleEncoding, utf8, latin1)+import qualified Data.ByteString as B+import Data.IORef+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import Hledger.Flow.Import.Dependencies+import Hledger.Flow.Import.Fingerprint (digestBytes)+import Hledger.Flow.Import.ImportHelpers (extractImportDirs)+import Hledger.Flow.Types (HledgerConf(..))+import Path+import qualified Path.IO as P+import qualified System.Directory as D+import System.Exit (ExitCode(..))+import Data.Time (addUTCTime)+import Test.HUnit hiding (path)+import TestHelpers (withTestDir)++fallback :: Show a => ReuseDecision a -> Assertion+fallback (Fallback _) = pure ()+fallback r = assertFailure ("expected fallback, got " ++ show r)++tests :: Test+tests = TestList+ [ "include expansion and selection" ~: withTestDir (\base -> do+ let root = base </> [relfile|main.rules|]+ mid = base </> [relfile|middle.rules|]+ leaf = base </> [relfile|empty.rules|]+ mainBytes = "include middle.rules\r\ninclude empty.rules\n"+ midBytes = "include empty.rules"+ write p = B.writeFile (toFilePath p)+ write root mainBytes+ write mid midBytes+ write leaf ""+ forM_ [V1524,V1996,OtherSupported] $ \release -> do+ tree <- discoverRules base release root+ tree @?= Reusable [(root,digestBytes mainBytes),(mid,digestBytes midBytes),(leaf,digestBytes ""),(leaf,digestBytes "")]+ ruleIncludes V1524 root "include a \r\n include b\ninclude\tc\n" @?= Reusable ["a "]+ ruleIncludes V1996 root "include a \r\n include b\ninclude\tc\n" @?= Reusable ["a"]+ fallback (nestedRuleIncludes V1524 mid "include final.rules\r\n")+ nestedRuleIncludes V1996 mid "include final.rules\r\n" @?= Reusable ["final.rules"]+ forM_ ["include filename.rules ", " include file.rules", "include\tfile.rules", "Include file.rules"] $+ fallback . ruleIncludes OtherSupported root+ fallback (nestedRuleIncludes OtherSupported mid "include final.rules\r\n")+ fallback (nestedRuleIncludes OtherSupported mid (B.pack [0xef,0xbb,0xbf] <> "include final.rules"))+ ruleIncludes OtherSupported root "include shopping categories.rules\r\n" @?= Reusable ["shopping categories.rules"]+ write root "include middle.rules\r\n"+ write mid "include main.rules\n"+ native <- discoverRules base V1524 root+ case native of+ Fallback CyclicRules{} -> assertFailure "top-level and native reads have different include targets"+ Fallback _ -> pure ()+ _ -> assertFailure "expected ordinary conversion for native line endings"+ write root mainBytes+ write mid "include missing.rules"+ discoverRules base V1996 root >>= fallback+ write mid "include main.rules"+ discoverRules base V1996 root >>= fallback+ write mid (T.encodeUtf8 ("include " <> T.pack (toFilePath leaf)))+ discoverRules base V1996 root >>= fallback+#ifndef mingw32_HOST_OS+ D.removeFile (toFilePath mid)+ D.createFileLink (toFilePath leaf) (toFilePath mid)+ tree <- discoverRules base V1996 root+ tree @?= Reusable [(root,digestBytes mainBytes),(mid,digestBytes ""),(leaf,digestBytes "")]+ D.removeFile (toFilePath mid)+ D.createFileLink (toFilePath root) (toFilePath mid)+ write root "include middle.rules"+ cycleResult <- discoverRules base V1524 root+ case cycleResult of+ Fallback CyclicRules{} -> pure ()+ _ -> assertFailure "a nested symlink resolving the same relative include repeats the cycle"+#endif+ let imports = base </> [reldir|import|]+ csv = imports </> [relfile|alice/bank/acct/1-in/2026/01_rfo-special.csv|]+ bank = imports </> [relfile|bank.rules|]+ override = imports </> [relfile|alice/bank/acct/rfo-special.rules|]+ dirs <- either (fail . T.unpack) pure (extractImportDirs imports csv)+ P.createDirIfMissing True (parent csv)+ write bank ""+ selectRules csv dirs >>= (@?= Reusable (Just bank))+ write override ""+ D.getModificationTime (toFilePath bank) >>= D.setModificationTime (toFilePath override) . addUTCTime (-1000)+ selectRules csv dirs >>= (@?= Reusable (Just override))+ D.removeFile (toFilePath override)+ selectRules csv dirs >>= (@?= Reusable (Just bank)))+ , "config syntax and whole-file digest" ~: withTestDir (\base -> do+ let file = base </> [relfile|hledger.conf|]+ ok release s = checkConfig release file s @?= Reusable ()+ no release s = fallback (checkConfig release file s)+ forM_ [V1524,V1996,OtherSupported] $ \release -> do+ ok release "# comment\n[balance] --period=thismonth\n[register]\n@file\n[general]\n[print]\n"+ forM_ ["--period=thismonth", "[general] --debug", "[print]\n@indirect", "[txns]", "[bal]", "[b]", "[alias]", "[missing", "[general", "[balance]\n[print] --period=thismonth"] (no release)+ ok V1996 "[print] --layout=hledger1 # layout\n"+ ok OtherSupported "[print] --layout=hledger1 # layout\n"+ no OtherSupported "[print]\n [balance]\n--empty"+ no OtherSupported "[holdings]"+ no OtherSupported "[demo]"+ no V1524 "[print]\n--layout=hledger1"+ no V1996 "[general]\n--layout=hledger1"+ ok V1996 "[holdings]"+ no V1524 "[print]\n [balance]\n--empty"+ no V1524 "[holdings]"+ ok V1524 "[demo]"+ no V1996 "[demo]"+ forM_ ["[balance]\n--period=thismonth", "[balance]\n--period=thisyear"] $ \bytes -> do+ B.writeFile (toFilePath file) bytes+ discoverConfig V1996 (Just (GivenConf (toFilePath file))) >>= (@?= Reusable (Just (file,digestBytes bytes)))+ fallback (checkArguments ["print","@file"])+ discoverConfig V1996 (Just (GivenConf (toFilePath base))) >>= fallback+#ifndef mingw32_HOST_OS+ D.removeFile (toFilePath file)+ D.createFileLink "hledger.conf" (toFilePath file)+ D.withCurrentDirectory (toFilePath base) (discoverConfig V1996 Nothing >>= fallback)+#endif+ )+ , "directives eligibility" ~: withTestDir (\base -> do+ let file = base </> [relfile|directives.journal|]+ check = checkDirectives V1996 file . T.encodeUtf8+ forM_ ["account assets:bank:current ; type: C","commodity 1,000.00000000 BTC ; lots:","commodity €1,000.00","P 2000-01-01 EUR €1","P 2000/01/01 BTC 12.30 EUR","; tags\n# prices\n* prices\n\n"] $ \s -> check s @?= Reusable ()+ forM_ ["account assets:any\n note", "comment\ncommodity €1\nend comment","include *.journal","P today EUR €1","P 2000-01-01 EUR €1 trailing","commodity 1,234 ABC extra", "2025/01/01 A"] (fallback . check))+ , "converter identity" ~: bracket getLocaleEncoding setLocaleEncoding (\_ -> withTestDir (\base -> do+ setLocaleEncoding utf8+ let real = base </> [relfile|actual|]+ second = base </> [relfile|other|]+ native = B.pack [127,69,76,70]+ B.writeFile (toFilePath real) (native <> "build A")+ B.writeFile (toFilePath second) native+#ifndef mingw32_HOST_OS+ let link = base </> [relfile|hledger|]+ D.createFileLink (toFilePath real) (toFilePath link)+ let executable = link+#else+ let executable = real+#endif+ probes <- newIORef (0 :: Int)+ let version _ = modifyIORef' probes (+1) >> pure (ExitSuccess,"hledger 1.99.6-gc3a608c2b-20261101, mac-aarch64")+ probe <- converterProbeWith executable version+ first <- probe+ case first of+ Reusable c -> converterRelease c @?= V1996+ Fallback why -> assertFailure (show why)+ probe >>= (@?= first)+ readIORef probes >>= (@?= 1)+ let replacement = toFilePath (base </> [relfile|replacement|])+ previousTime <- D.getModificationTime (toFilePath real)+ B.writeFile replacement (native <> "build B")+ D.setModificationTime replacement previousTime+ D.renameFile replacement (toFilePath real)+ -- Compare the key persisted between imports, which excludes file stamps.+ -- Both executables have the same path, version, length and mtime.+ let checkReplacement result = case (first, result) of+ (Reusable old, Reusable new) ->+ assertBool "replacement changes saved converter key" (converterKey old /= converterKey new)+ _ -> assertFailure ("expected two eligible builds: " ++ show (first, result))+ probe >>= checkReplacement+ freshProbe <- converterProbeWith executable version+ freshProbe >>= checkReplacement+#ifndef mingw32_HOST_OS+ D.removeFile (toFilePath link)+ D.createFileLink (toFilePath second) (toFilePath link)+ retargeted <- probe+ assertBool "retargeted symlink" (retargeted /= first)+#endif+ forM_ ["h.cmd","h.BAT","scoop.exe","wrapped","unknown"] $ \name -> do+ name' <- parseRelFile name+ let path = base </> name'+ B.writeFile (toFilePath path) (if name == "wrapped" then "#!/bin/sh\n" else if name == "unknown" then "hledger" else native)+ if name == "scoop.exe" then B.writeFile (toFilePath (base </> [relfile|scoop.shim|])) "path = hidden" else pure ()+ converterProbeWith path version >>= (>>= fallback)+ let identify v = converterProbeWith real (const (pure (ExitSuccess,"hledger " <> v))) >>= id+ forM_ [("1.52.4",V1524),("1.99.6",V1996),("1.52.40",OtherSupported),+ ("1.99-g01a4b4866-20261113, mac-aarch64",OtherSupported),("1.100",OtherSupported),("2.0",OtherSupported)] $ \(v, expected) ->+ (fmap converterRelease <$> identify v) >>= (@?= Reusable expected)+ forM_ ["1.52.3","1.52","1.9.40","1.99.x","1.99.","1..99","unknown"] $ \v -> identify v >>= fallback+ setLocaleEncoding latin1+ converterProbeWith real version >>= (>>= fallback)))+ ]
+ test/Incremental/Platform.hs view
@@ -0,0 +1,157 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Incremental.Platform (tests) where++import Control.Monad (forM_)+import qualified Data.ByteString as BS+import Data.Time (addUTCTime)+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.PrivateDirectory+import Path+import qualified Path.IO as PathIO+import qualified System.Directory as Dir+import System.IO.Error (tryIOError)+import Test.HUnit hiding (Path)+import TestHelpers (withTestDir)+#ifdef mingw32_HOST_OS+import System.Process (callProcess)+import System.Environment (setEnv, unsetEnv)+import Control.Exception (bracket_)+#else+import Control.Exception (bracket)+import Data.Bits ((.&.))+import qualified System.Posix.Files as Posix+#endif+#ifdef darwin_HOST_OS+import Data.List (isInfixOf)+import qualified System.Posix.Directory as Posix+import System.Process (callProcess, readProcess)+#endif++tests :: Test+tests = TestLabel "Incremental platform primitives" $ TestList+ [ TestLabel "SHA-256 over exact file bytes" $ TestCase $ withTestDir $ \root -> do+ let file = root </> [relfile|input|]+ forM_ [(0, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"),+ (1000000, "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0")] $+ \(len, expected) -> do+ BS.writeFile (toFilePath file) (BS.replicate len 97)+ assertEqual "in-memory digest" expected (digestHex (digestBytes (BS.replicate len 97)))+ digest <- digestFile file+ assertEqual "fixed SHA-256 vector" expected (digestHex digest)+ let copy = root </> [relfile|copied|]+ writeFile (toFilePath copy) "old contents"+ copied <- copyDigest file copy+ assertEqual "hash of copied bytes" digest copied+ bytes <- BS.readFile (toFilePath copy)+ assertEqual "copied all bytes" (BS.replicate len 97) bytes+ observation <- observeFile file+ stamp <- statFile file+ assertEqual "same file metadata" stamp (fileStamp observation)+ assertEqual "observed bytes" digest (observationDigest observation)+ assertEqual "observed size" (fromIntegral len) (stampSize (fileStamp observation))+ , TestLabel "same-byte rewrite changes transient observation" $ TestCase $ withTestDir $ \root -> do+ let file = root </> [relfile|input|]+ name = toFilePath file+ writeFile name "same bytes"+ first <- observeFile file+ originalTime <- Dir.getModificationTime name+ writeFile name "same bytes"+ Dir.setModificationTime name (addUTCTime 10 originalTime)+ second <- observeFile file+ assertBool "observed rewrite" (first /= second)+ assertEqual "unchanged content key" (observationDigest first) (observationDigest second)+ , TestLabel "file identity distinguishes copies" $ TestCase $ withTestDir $ \root -> do+ let file = root </> [relfile|input|]+ copy = root </> [relfile|copy|]+ writeFile (toFilePath file) "shared bytes"+ Dir.copyFile (toFilePath file) (toFilePath copy)+ a <- observeFile file+ b <- observeFile copy+ assertBool "different files" (stampIdentity (fileStamp a) /= stampIdentity (fileStamp b))+ , TestLabel "private staging under a shared directory" $ TestCase $ withTestDir $ \root -> do+ let run = root </> [reldir|run|]+#ifdef mingw32_HOST_OS+ withAclPaths root run $ do+ powershell $ unlines+ [ "$acl = [IO.Directory]::GetAccessControl($env:HF_ACL_PARENT)"+ , "$everyone = [System.Security.Principal.SecurityIdentifier]'S-1-1-0'"+ , "$acl.AddAccessRule([System.Security.AccessControl.FileSystemAccessRule]::new($everyone,'ReadAndExecute','ContainerInherit,ObjectInherit','None','Allow'))"+ , "[IO.Directory]::SetAccessControl($env:HF_ACL_PARENT, $acl)"+ ]+ createPrivateDirectory run+ let nested = run </> [reldir|nested|]+ copied = nested </> [relfile|statement|]+ source = root </> [relfile|source|]+ PathIO.createDir nested+ writeFile (toFilePath source) "private"+ _ <- copyDigest source copied+ powershell $ unlines+ [ "$sid = [System.Security.Principal.WindowsIdentity]::GetCurrent().User.Value"+ , "$dir = [IO.Directory]::GetAccessControl($env:HF_ACL_RUN)"+ , "if (-not $dir.AreAccessRulesProtected) { throw 'inheriting cache parent DACL' }"+ , "$file = [IO.File]::GetAccessControl([IO.Path]::Combine($env:HF_ACL_RUN, 'nested/statement'))"+ , "foreach ($acl in @($dir, $file)) {"+ , " $rules = $acl.GetAccessRules($true,$true,[System.Security.Principal.SecurityIdentifier])"+ , " if ($rules.Count -ne 1 -or $rules[0].IdentityReference.Value -ne $sid -or $rules[0].AccessControlType -ne 'Allow' -or $rules[0].FileSystemRights -ne 'FullControl') { throw 'expected only current user with full access' }"+ , " if ($acl -ne $dir -and -not $rules[0].IsInherited) { throw 'file lacks inherited access' }"+ , "}"+ ]+#else+ Posix.setFileMode (toFilePath root) 0o777+ createPrivateDirectory run+ mode <- Posix.fileMode <$> Posix.getFileStatus (toFilePath run)+ assertEqual "owner-only directory" 0o700 (mode .&. 0o777)+#endif+#ifndef mingw32_HOST_OS+ , TestLabel "unusable creation mode rejects the empty directory" $ TestCase $ withTestDir $ \root -> do+ let run = root </> [reldir|run|]+ result <- bracket (Posix.setFileCreationMask 0o177) Posix.setFileCreationMask $+ \_ -> tryIOError (createPrivateDirectory run)+ assertRejectedDirectory run result+#endif+#ifdef darwin_HOST_OS+ , TestLabel "inherited macOS ACL rejects staging" $ TestCase $ withTestDir $ \root -> do+ let probe = root </> [reldir|probe|]+ run = root </> [reldir|run|]+ callProcess "/bin/chmod" ["+a", "everyone allow list,search,directory_inherit", toFilePath root]+ Posix.createDirectory (toFilePath probe) 0o700+ acl <- readProcess "/bin/ls" ["-lde", toFilePath probe] ""+ assertBool "the filesystem inherited the test grant" ("inherited allow" `isInfixOf` acl)+ result <- tryIOError (createPrivateDirectory run)+ assertRejectedDirectory run result+#endif+ , TestLabel "occupied path stops staging before copying" $ TestCase $ withTestDir $ \root -> do+ let run = root </> [reldir|occupied|]+ PathIO.createDir run+ result <- tryIOError $ do+ createPrivateDirectory run+ writeFile (toFilePath (run </> [relfile|statement|])) "private"+ case result of+ Left _ -> return ()+ Right () -> assertFailure "used a pre-existing directory"+ entries <- Dir.listDirectory (toFilePath run)+ assertEqual "no copies made" [] entries+ ]++#ifndef mingw32_HOST_OS+assertRejectedDirectory :: Path Abs Dir -> Either IOError () -> Assertion+assertRejectedDirectory run result = do+ case result of+ Left _ -> pure ()+ Right () -> assertFailure "accepted unverified staging permissions"+ exists <- PathIO.doesDirExist run+ assertBool "removed the rejected empty directory" (not exists)+#endif++#ifdef mingw32_HOST_OS+withAclPaths :: Path Abs Dir -> Path Abs Dir -> IO a -> IO a+withAclPaths root run =+ bracket_ (setEnv "HF_ACL_PARENT" (toFilePath root) >> setEnv "HF_ACL_RUN" (toFilePath run))+ (unsetEnv "HF_ACL_PARENT" >> unsetEnv "HF_ACL_RUN")++powershell :: String -> IO ()+powershell command = callProcess "powershell.exe" ["-NoProfile", "-NonInteractive", "-Command", "$ErrorActionPreference = 'Stop'; " ++ command]+#endif
+ test/Incremental/Snapshot.hs view
@@ -0,0 +1,313 @@+{-# LANGUAGE CApiFFI #-}+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Incremental.Snapshot (tests, runWorker) where++import Control.Concurrent+import Control.Concurrent.STM (newTChanIO)+import Control.Exception (bracket, try, IOException, throwIO)+import Control.Monad (unless)+import GHC.Conc (threadStatus, ThreadStatus(..))+import GHC.IO.Encoding (getLocaleEncoding, setLocaleEncoding, utf8)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Char8 as B8+import qualified Data.Map.Strict as Map+import qualified Data.Text as T+import Hledger.Flow.Import.Cache+import Hledger.Flow.Import.Dependencies+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.Import.ImportHelpers (extractImportDirs)+import Hledger.Flow.Import.Incremental+import Hledger.Flow.Import.PrivateDirectory+import Hledger.Flow.Import.Snapshot+import Hledger.Flow.RuntimeOptions+import Path+import qualified Path.IO as P+import qualified System.Directory as D+#ifdef mingw32_HOST_OS+import Data.Word (Word32)+import Data.Text.Encoding (encodeUtf8)+import Foreign.C.String (CWString, peekCWString, withCWString)+import Foreign.Marshal.Array (allocaArray)+#endif+import qualified System.FilePath as FP+import System.Environment (getExecutablePath)+import System.IO (hFlush, hGetLine, stdout)+import System.Process+import System.Timeout (timeout)+import Test.HUnit+import TestHelpers (defaultOpts, withTestDir)++tests :: Test+tests = TestLabel "Private input snapshots" $ TestList+ [ TestLabel "no snapshot area for a hit" $ TestCase $ withTestDir $ \base -> do+ withSnapshotRun base $ \_ -> pure ()+ P.doesDirExist (base </> [reldir|.hledger-flow/cache/snapshots|]) >>= assertBool "created lazily" . not+ , TestLabel "copy selected bytes and translate locations" $ TestCase $ withTestDir $ \base -> do+ let statement = base </> [relfile|account/2026/input.csv|]+ rules = base </> [relfile|account/bank.rules|]+ category = base </> [relfile|shared/coffee.rules|]+ directives = base </> [relfile|directives.journal|]+ a = "account2 expenses:cafe\n"+ P.createDirIfMissing True (parent statement)+ P.createDir (parent category)+ BS.writeFile (toFilePath statement) "2026-01-01,Coffee,3\n"+ BS.writeFile (toFilePath rules) "include ../shared/coffee.rules\n"+ BS.writeFile (toFilePath category) a+ BS.writeFile (toFilePath directives) "account expenses:cafe\n"+ result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (RulesInput statement rules (Just directives) Nothing) $ \snapshot -> do+ RulesInput dataCopy rulesCopy (Just directivesCopy) Nothing <- pure (snapshotInputs snapshot)+ let deps = snapshotManifest snapshot+ assertEqual "manifest includes nested rule" (Just (digestBytes a)) (Map.lookup category deps)+ assertEqual "all four files" 4 (Map.size deps)+ assertEqual "CSV basename" (filename statement) (filename dataCopy)+ canonicalCopy <- D.canonicalizePath (toFilePath rulesCopy)+ let location = ":1:2:\n"+ excerpt = "1 | " <> T.pack (toFilePath rulesCopy) <> "\n"+ assertEqual "maps location, preserves excerpt" ("hledger: Error: " <> T.pack (toFilePath rules) <> location <> excerpt)+ =<< translateDiagnostic snapshot ("hledger: Error: " <> T.pack canonicalCopy <> location <> excerpt)+ let leafCopy = parent (parent rulesCopy) </> [relfile|shared/coffee.rules|]+ pure [dataCopy, rulesCopy, leafCopy, directivesCopy]+ case result of+ Reusable files -> mapM_ (\p -> P.doesFileExist p >>= assertBool "copy released" . not) files+ Fallback reason -> assertFailure (show reason)+ , TestLabel "publish only when converted snapshot matches live inputs" $ TestCase snapshotPublication+ , TestLabel "construct pins an external config" $ TestCase $ withTestDir $ \base -> withTestDir $ \home -> do+ let config = home </> [relfile|hledger.conf|]+ BS.writeFile (toFilePath config) "[balance]\n--period=thismonth\n"+ result <- withSnapshotRun base $ \run -> withSnapshot run V1996 (ConstructInput (Just config)) $ \snapshot -> do+ ConstructInput (Just copied) <- pure (snapshotInputs snapshot)+ assertBool "separate private copy" (config /= copied)+ assertEqual "config path" (T.pack (toFilePath config) <> ":6:3:")+ =<< translateDiagnostic snapshot (T.pack (toFilePath copied) <> ":6:3:")+ let windowsError p = "hledger.exe: Error: " <> T.pack (toFilePath p) <> ":6:3:"+ assertEqual "Windows hledger diagnostic" (windowsError config)+ =<< translateDiagnostic snapshot (windowsError copied)+ BS.readFile (toFilePath copied) >>= assertEqual "whole-file content" "[balance]\n--period=thismonth\n"+ case result of+ Reusable () -> pure ()+ Fallback reason -> assertFailure (show reason)+ , TestLabel "conversion errors propagate and clean up" $ TestCase $ withTestDir $ \base -> do+ result <- try $ withSnapshotRun base $ \run ->+ withSnapshot run V1524 (ConstructInput Nothing) $ \_ -> ioError (userError "converter failed")+ case result :: Either IOException (ReuseDecision ()) of+ Left _ -> pure ()+ Right _ -> assertFailure "swallowed converter error"+ names <- D.listDirectory (toFilePath (base </> [reldir|.hledger-flow/cache/snapshots|]))+ assertBool "only registration lock survives" (length names <= 1)+#ifndef mingw32_HOST_OS+ , TestLabel "canonical snapshot below symlink maps to original spelling" $ TestCase $ withTestDir $ \root -> do+ let base = root </> [reldir|ledger alias|]+ real = root </> [reldir|ledger|]+ config = base </> [relfile|config with spaces|]+ P.createDir real+ D.createDirectoryLink (toFilePath real) (FP.dropTrailingPathSeparator (toFilePath base))+ BS.writeFile (toFilePath config) "# ok\n"+ result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (ConstructInput (Just config)) $ \snap -> do+ ConstructInput (Just copied) <- pure (snapshotInputs snap)+ canonical <- D.canonicalizePath (toFilePath copied)+ assertBool "release canonicalized the ancestor" (canonical /= toFilePath copied)+ let ruleLine fileName = "the date rule is: %1 (" <> T.pack fileName <> ":7)"+ let err name = "hledger: Error: " <> T.dropEnd 1 (T.pack (show name)) <> ":2:2:"+ warning name = "Warning: " <> T.pack name <> ": replaced delimiter"+ assertEqual "quoted config diagnostic" (err (toFilePath config)) =<< translateDiagnostic snap (err canonical)+ assertEqual "CSV warning" (warning (toFilePath config)) =<< translateDiagnostic snap (warning canonical)+ assertEqual "diagnostic location" (ruleLine (toFilePath config))+ =<< translateDiagnostic snap (ruleLine canonical)+ assertEqual "converted" (Reusable ()) result+#else+ , TestLabel "260 UTF-16 units in data, include and pinned config" $ TestCase $ withTestDir $ \base -> do+ let csv = base </> [relfile|in.csv|]+ rules = base </> [relfile|in.rules|]+ fixed copied = toFilePath base FP.</> ".hledger-flow" FP.</> "cache" FP.</> "snapshots"+ FP.</> "r00000000" FP.</> "t00000000" FP.</> copied+ runFor part lengthWanted = do+ let folder = if part == "config" then "c" else "l"+ suffix = if part == "CSV" then "😀.csv" else "😀.rules"+ units = sum . map (\c -> if fromEnum c > 65535 then 2 else 1)+ padding = lengthWanted - units (fixed (folder FP.</> suffix))+ assertBool "fixture has room for the filename" (padding >= 0)+ file <- (base </>) <$> parseRelFile (replicate padding 'x' ++ suffix)+ BS.writeFile (toFilePath file) ""+ BS.writeFile (toFilePath csv) "2026-01-01,coffee,3"+ if part == "include"+ then BS.writeFile (toFilePath rules) (encodeUtf8 ("include " <> T.pack (toFilePath (filename file)) <> "\n"))+ else writeFile (toFilePath rules) ""+ let input = case part of+ "CSV" -> RulesInput file rules Nothing Nothing+ "include" -> RulesInput csv rules Nothing Nothing+ _ -> ConstructInput (Just file)+ result <- withSnapshotRun base $ \run -> withSnapshot run V1996 input (const (pure ()))+ case (lengthWanted, result) of+ (259, Reusable ()) -> pure ()+ (260, Fallback reason) -> assertEqual part "overlong Windows snapshot path" (reasonScope reason)+ _ -> assertFailure (part ++ " at " ++ show lengthWanted ++ ": " ++ show result)+ mapM_ (\part -> mapM_ (runFor part) [259,260]) ["CSV","include","config"]+ , TestLabel "short filename diagnostic when available" $ TestCase $ withTestDir $ \base -> do+ let config = base </> [relfile|config containing spaces|]+ BS.writeFile (toFilePath config) "# config\n"+ result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (ConstructInput (Just config)) $ \snap -> do+ ConstructInput (Just copied) <- pure (snapshotInputs snap)+ withCWString (toFilePath copied) $ \long -> allocaArray 260 $ \buffer -> do+ n <- getShortPath long buffer 260+ if n == 0 || n >= 260 then pure () else do+ short <- peekCWString buffer+ assertEqual "resolved file identity" (T.pack (toFilePath config) <> ":4:2:")+ =<< translateDiagnostic snap (T.pack short <> ":4:2:")+ assertEqual "converted" (Reusable ()) result+#endif+ , TestLabel "permission failure makes no copies" $ TestCase $ withTestDir $ \base -> do+ let deny _ = ioError (userError "permission denied")+ result <- withSnapshotRunUsing deny base $ \run ->+ withSnapshot run V1524 (ConstructInput Nothing) $ \_ -> assertFailure "ran a private conversion"+ case result of+ Fallback r -> assertBool "private staging reason" ("private staging unavailable" `T.isPrefixOf` reasonScope r)+ Reusable _ -> assertFailure "admitted unprotected staging"+ D.listDirectory (toFilePath (base </> [reldir|.hledger-flow/cache/snapshots|])) >>= assertEqual "no financial data" ["registration.lock"]+ , TestLabel "registration protects an initializing directory" $ TestCase $ withTestDir $ \base -> do+ created <- newEmptyMVar+ proceed <- newEmptyMVar+ initializer <- newEmptyMVar+ cleaner <- newEmptyMVar+ let suspended dir = createPrivateDirectory dir >> putMVar created () >> takeMVar proceed+ _ <- forkFinally (withSnapshotRunUsing suspended base $ \run ->+ withSnapshot run V1524 (ConstructInput Nothing) (const (pure ()))) (putMVar initializer)+ takeMVar created+ tid <- forkFinally (withSnapshotRun base (const (pure ()))) (putMVar cleaner)+ let blocked = do+ status <- threadStatus tid+ case status of+ ThreadBlocked _ -> pure True+ ThreadFinished -> pure False+ ThreadDied -> pure False+ ThreadRunning -> yield >> blocked+ waiting <- timeout 5000000 blocked+ putMVar proceed ()+ first <- takeMVar initializer >>= either throwIO pure+ takeMVar cleaner >>= either throwIO pure+ assertEqual "cleanup waited for registration" (Just True) waiting+ assertEqual "initializer consumed its own tree" (Reusable ()) first+ , TestLabel "reap killed worker, preserve live and unknown directories" $ TestCase $ withTestDir $ \base -> do+ let config = base </> [relfile|hledger.conf|]+ BS.writeFile (toFilePath config) "# private config\n"+ exe <- getExecutablePath+ let child = (proc exe ["--snapshot-test-worker", toFilePath base]) {std_in = CreatePipe, std_out = CreatePipe}+ result <- withSnapshotRun base $ \active -> withSnapshot active V1524 (ConstructInput (Just config)) $ \snap -> do+ ConstructInput (Just live) <- pure (snapshotInputs snap)+ let unknown = base </> [reldir|.hledger-flow/cache/snapshots/r01234567|]+ keep = unknown </> [relfile|unrelated|]+ P.createDir unknown+ writeFile (toFilePath keep) "preserve"+ withCreateProcess child $ \_ out _ pid -> do+ Just stdoutPipe <- pure out+ copiedName <- hGetLine stdoutPipe+ abandoned <- parseAbsFile copiedName+ terminateProcess pid+ _ <- waitForProcess pid+ P.doesFileExist abandoned >>= assertBool "private data survived unhandled termination"+ withSnapshotRun base (const (pure ()))+ P.doesFileExist abandoned >>= assertBool "abandoned copy reclaimed" . not+ P.doesFileExist keep >>= assertBool "unknown directory preserved"+ BS.readFile (toFilePath live) >>= assertEqual "other run untouched" "# private config\n"+ assertEqual "live run completes" (Reusable ()) result+ , TestLabel "different include spellings cannot overwrite a snapshot file" $ TestCase $ withTestDir $ \base -> do+ let csv = base </> [relfile|input.csv|]+ rules = base </> [relfile|bank.rules|]+ upper = [relfile|A.rules|]+ lower = [relfile|a.rules|]+ BS.writeFile (toFilePath csv) "2026-01-01,Coffee,3\n"+ BS.writeFile (toFilePath rules) "include A.rules\ninclude a.rules\n"+ BS.writeFile (toFilePath (base </> upper)) "account1 assets:bank\n"+ sameFile <- P.doesFileExist (base </> lower)+ BS.writeFile (toFilePath (base </> lower)) "account2 expenses:cafe\n"+ result <- withSnapshotRun base $ \run -> withSnapshot run V1996 (RulesInput csv rules Nothing Nothing) $ \snap -> do+ RulesInput _ staged _ _ <- pure (snapshotInputs snap)+ mapM (BS.readFile . toFilePath . (parent staged </>)) [upper,lower]+ case result of+ Fallback reason -> assertEqual "fallback reason" "snapshot filenames collide" (reasonScope reason)+ Reusable contents -> do+ assertBool "aliased destinations must fall back" (not sameFile)+ assertEqual "distinct copies preserve each set of rules" ["account1 assets:bank\n","account2 expenses:cafe\n"] contents+ , TestLabel "Windows uses UTF-16 units without the NUL" $ TestCase $ do+ assertBool "259" (windowsSnapshotPathFits (replicate 257 'a' ++ "😀"))+ assertBool "260" (not (windowsSnapshotPathFits (replicate 258 'a' ++ "😀")))+ ]++-- Run the real importer with this native test executable as its converter.+-- Its print command changes the live include while consuming the supplied rules.+snapshotPublication :: IO ()+snapshotPublication = withTestDir $ \base -> P.withCurrentDir base $+ bracket getLocaleEncoding setLocaleEncoding $ \_ -> do+ setLocaleEncoding utf8+ exe <- getExecutablePath >>= parseAbsFile+ let opts = (defaultOpts base) {hledgerExecutable = exe, importPolicy = RunScripts}+ root = base </> [reldir|import|]+ source = root </> [relfile|o/b/a/1-in/2026/in.csv|]+ leaf = root </> [relfile|shared/category.rules|]+ a = "account2 expenses:cafe\n"+ P.createDirIfMissing True (parent source)+ P.createDir (parent leaf)+ BS.writeFile (toFilePath source) "2026-01-01,Coffee,3\n"+ BS.writeFile (toFilePath (root </> [relfile|b.rules|])) "include shared/category.rules\n"+ BS.writeFile (toFilePath leaf) a+ BS.writeFile "hledger.conf" ""+ dirs <- either (assertFailure . T.unpack) pure (extractImportDirs root source)+ ch <- newTChanIO+ let run = withIncrementalRun opts ch 1 $ \ctx -> importStatement ctx opts ch dirs source+ journal = "2026-01-01 Coffee\n expenses:cafe 3\n assets:bank\n"+ record <- recordPath base source+ BS.writeFile "snapshot-converter-test" "restore"+ (out,_,_) <- run+ readFile (toFilePath out) >>= assertEqual "journal built from copied rule A" journal+ entry <- readRecord record >>= maybe (assertFailure "missing journal record") pure . (>>= journalRecord)+ lookup (T.pack (toFilePath leaf)) (stageInputs entry) @?= Just (digestHex (digestBytes a))+ digestFile out >>= assertEqual "record identifies installed journal bytes" (stageOutput entry)+ -- Force a rebuild with an existing record. Leaving B in place must also+ -- discard the old A record, even though the snapshot produced A again.+ P.removeFile out+ BS.writeFile "snapshot-converter-test" "keep"+ _ <- run+ readFile (toFilePath out) >>= assertEqual "kept journal still built from A" journal+ BS.readFile (toFilePath leaf) >>= assertEqual "live include remains B" "account2 expenses:travel\n"+ readRecord record >>= assertEqual "invalidated journal record" Nothing . (>>= journalRecord)++-- The real test executable owns the lifetime lock. Its parent terminates it+-- abruptly, so no in-process finalizer can remove the abandoned tree.+runWorker :: [String] -> IO Bool+runWorker ["--snapshot-test-worker", baseName] = do+ base <- parseAbsDir baseName+ let config = base </> [relfile|hledger.conf|]+ result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (ConstructInput (Just config)) $ \snapshot -> do+ ConstructInput (Just copied) <- pure (snapshotInputs snapshot)+ putStrLn (toFilePath copied)+ hFlush stdout+ _ <- getLine+ pure ()+ case result of+ Fallback reason -> ioError (userError (show reason))+ Reusable () -> pure True+runWorker args = do+ fixture <- D.doesFileExist "snapshot-converter-test"+ if fixture then fixtureConverter args >> pure True else pure False++fixtureConverter :: [String] -> IO ()+fixtureConverter ["--version"] = putStrLn "hledger 1.99.6"+fixtureConverter ["--conf",_,"print","--explicit","--file",_,"--rules-file",rules] = do+ D.renameDirectory "import/shared" "import/held"+ D.createDirectory "import/shared"+ BS.writeFile "import/shared/category.rules" "account2 expenses:travel\n"+ ["include",relative] <- words . B8.unpack <$> BS.readFile rules+ ["account2",account] <- words . B8.unpack <$> BS.readFile (FP.takeDirectory rules FP.</> relative)+ keep <- (== "keep") <$> BS.readFile "snapshot-converter-test"+ unless keep $ do+ D.removeDirectoryRecursive "import/shared"+ D.renameDirectory "import/held" "import/shared"+ putStr $ "2026-01-01 Coffee\n " ++ account ++ " 3\n assets:bank\n"+fixtureConverter args = ioError (userError ("Unexpected converter arguments: " ++ show args))++#ifdef mingw32_HOST_OS+foreign import capi unsafe "windows.h GetShortPathNameW"+ getShortPath :: CWString -> CWString -> Word32 -> IO Word32+#endif
test/ReleaseNotice/Unit.hs view
@@ -4,21 +4,39 @@ import Control.Concurrent.MVar import Control.Exception (AsyncException (UserInterrupt), throwIO, try)-import Control.Monad (forM_)+import Control.Monad (forM_, join) import Data.Aeson (encode, object, (.=))+import qualified Data.ByteString.Lazy as BL import Data.IORef import qualified Data.Text as T+import Data.Time (UTCTime, addUTCTime, getCurrentTime) import Data.Version (makeVersion, showVersion) import Paths_hledger_flow (version)-import Hledger.Flow.ReleaseNotice (ReleaseResult (..), checkRelease, finishCommand, renderUpdate, startCheck)+import Hledger.Flow.Cache (initializeCache)+import Hledger.Flow.PathHelpers (AbsDir)+import Hledger.Flow.ReleaseNotice import Path (toFilePath)-import System.Directory (createDirectoryIfMissing)+import Path.IO (withSystemTempDir)+import System.Directory (createDirectoryIfMissing, doesDirectoryExist, getModificationTime, setModificationTime) import System.Exit (ExitCode (ExitFailure)) import System.FilePath ((</>)) import System.Timeout (timeout) import Test.HUnit import TestHelpers (withTestDir) +-- Exercise a complete, deterministic refresh without the CLI's exit deadline.+checkRelease :: (String -> IO (Maybe BL.ByteString)) -> IO FilePath -> IO ReleaseResult+checkRelease get executable = withSystemTempDir "release-check" $ \base -> do+ initializeCache base+ refreshIfDue base getCurrentTime get executable+ cachedRelease base executable++recordFile :: AbsDir -> FilePath+recordFile base = toFilePath base </> ".hledger-flow/cache/release-check.json"++testTime :: UTCTime+testTime = read "2026-11-16 12:00:00 UTC"+ tests :: Test tests = TestLabel "release notice" $ TestList [ "invalid or missing responses cannot confirm release status" ~: TestCase $@@ -62,16 +80,64 @@ check releases Nothing check releases (Just " tag = \"v999.9.0\"\n") check "Upgrade with: brew update && brew upgrade ferntrail/tap/hledger-flow" published- , "command finishes with worker still pending" ~: TestCase $ do- started <- newEmptyMVar- release <- newEmptyMVar- finish <- startCheck (putMVar started () >> takeMVar release)- takeMVar started- success <- timeout 1000000 $ finishCommand finish (pure ())- failure <- timeout 1000000 $ try (finishCommand finish (throwIO (ExitFailure 37)))- Just () @=? success- Just (Left (ExitFailure 37)) @=? failure- putMVar release UpToDate+ , "cached formula advice follows the current executable" ~: TestCase $+ withTestDir $ \base -> do+ let cache = toFilePath base </> ".hledger-flow/cache"+ createDirectoryIfMissing True cache+ writeFile (recordFile base) "{\"lastSuccess\":{\"latestVersion\":\"999\",\"checkedAt\":\"2026-11-16T12:00:00Z\",\"formulaConfirmed\":true}}"+ notice <- cachedRelease base (pure $ toFilePath base </> "hledger-flow")+ UpdateAvailable (makeVersion [999]) "https://github.com/apauley/hledger-flow/releases/latest" @=? notice+ , "missing cache stays absent on read and corrupt JSON can be refreshed" ~: TestCase $+ withTestDir $ \base -> do+ let cache = toFilePath base </> ".hledger-flow/cache"+ readOnly = cachedRelease base (pure "hledger-flow")+ assertEqual "no local notice" CheckUnavailable =<< readOnly+ assertEqual "read-only lookup" False =<< doesDirectoryExist cache+ createDirectoryIfMissing True cache+ writeFile (recordFile base) "partial JSON"+ assertEqual "corrupt cache" CheckUnavailable =<< readOnly+ notice <- join $ prepareCheck base (pure testTime) (const $ pure $ Just "{\"tag_name\":\"v0\"}") (pure "hledger-flow")+ UpToDate @=? notice+ let ignore = cache </> ".gitignore"+ assertEqual "shared ignore file repaired" "*\n" =<< readFile ignore+ setModificationTime ignore testTime+ timestamp <- getModificationTime ignore+ initializeCache base+ assertEqual "incremental initializer preserves ignore mtime" timestamp =<< getModificationTime ignore+ , "refresh intervals and offline fallback" ~: TestCase $ withTestDir $ \base -> do+ calls <- newIORef (0 :: Int)+ let executable = pure (toFilePath base </> "hledger-flow")+ fetch body _ = modifyIORef' calls (+1) >> pure body+ newer = Just "{\"tag_name\":\"v999\"}"+ at seconds = pure (addUTCTime seconds testTime)+ refresh seconds body = refreshIfDue base (at seconds) (fetch body) executable+ notice = UpdateAvailable (makeVersion [999]) "https://github.com/apauley/hledger-flow/releases/latest"+ first <- join $ prepareCheck base (at 0) (fetch newer) executable+ notice @=? first+ assertEqual "fresh result" notice =<< join (prepareCheck base (at 86399) (fetch Nothing) executable)+ assertEqual "only initial request" 1 =<< readIORef calls+ refresh 86400 Nothing+ assertEqual "offline result" notice =<< cachedRelease base executable+ assertEqual "status reads and failure cooldown" notice =<< join (prepareCheck base (at 87299) (fetch newer) executable)+ assertEqual "one failed refresh" 2 =<< readIORef calls+ refresh 87300 (Just "{\"tag_name\":\"v0\"}")+ assertEqual "successful retry" UpToDate =<< cachedRelease base executable+ assertEqual "retry after 15 minutes" 3 =<< readIORef calls+ , "cold command finishes while another refresh is pending" ~: TestCase $+ withTestDir $ \base -> do+ started <- newEmptyMVar+ release <- newEmptyMVar+ calls <- newIORef (0 :: Int)+ let fetch _ = modifyIORef' calls (+1) >> putMVar started () >> takeMVar release+ start = prepareCheck base (pure testTime) fetch (pure "hledger-flow")+ finish <- start+ takeMVar started+ overlapping <- start+ assertEqual "contending check finishes" (Just CheckUnavailable) =<< timeout 1000000 overlapping+ assertEqual "cold wait is bounded" (Just CheckUnavailable) =<< timeout 1000000 finish+ assertEqual "one fetch" 1 =<< readIORef calls+ putMVar release (Just "{\"tag_name\":\"v0\"}")+ assertEqual "finished worker published" UpToDate =<< finish , "ready notice follows diagnostics and preserves the exit code" ~: TestCase $ do events <- newIORef ([] :: [String]) let record msg = modifyIORef' events (++ [msg])
test/RuntimeOptions/Unit.hs view
@@ -20,4 +20,4 @@ assertEqual "verbose alone" (True, False, False) (flags opts {verbose = True}) assertEqual "sequential alone" (False, True, False) (flags opts {sequential = True}) assertEqual "pretty reports alone" (False, False, True) (flags opts {prettyReports = True})- assertEqual "unrelated flags" (False, False, False) (flags opts {onlyNewFiles = True, showOptions = True})+ assertEqual "unrelated flags" (False, False, False) (flags opts {importPolicy = TimestampOnly, showOptions = True})
test/Spec.hs view
@@ -10,6 +10,10 @@ import qualified Common.Unit import qualified Environment.Unit import qualified ImportHelpers.Integration+import qualified Incremental.Cache+import qualified Incremental.Dependencies+import qualified Incremental.Platform+import qualified Incremental.Snapshot import qualified Init.Integration import qualified PathHelpers.Unit import qualified ReleaseNotice.Unit@@ -18,6 +22,8 @@ import qualified Status.Integration import qualified Status.Unit import Test.HUnit+import Control.Monad (unless, when)+import System.Environment (getArgs) import System.Exit (exitFailure) tests :: Test@@ -32,6 +38,10 @@ CSVImport.Integration.tests, Status.Unit.tests, Status.Integration.tests,+ Incremental.Cache.tests,+ Incremental.Dependencies.tests,+ Incremental.Platform.tests,+ Incremental.Snapshot.tests, Init.Integration.tests, ReleaseNotice.Unit.tests, RulesTemplate.Unit.tests,@@ -39,9 +49,9 @@ Environment.Unit.tests ] -main :: IO Counts+main :: IO () main = do- errCounts <- runTestTT tests- if (errors errCounts > 0 || failures errCounts > 0)- then exitFailure- else return errCounts+ child <- getArgs >>= Incremental.Snapshot.runWorker+ unless child $ do+ results <- runTestTT tests+ when (errors results > 0 || failures results > 0) exitFailure
test/TestHelpers.hs view
@@ -49,7 +49,7 @@ { baseDir = bd, importRunDir = [reldir|./|], importStartYear = Nothing,- onlyNewFiles = False,+ importPolicy = Full, hfVersion = versionInfo testSystemInfo, hledgerExecutable = FlowTypes.hlPath defaultHlInfo, hledgerVersion = return (FlowTypes.hlVersion defaultHlInfo),