packages feed

hledger-flow 0.20.1 → 0.21.0

raw patch · 40 files changed

+2860/−384 lines, 40 filesdep +ShellCheckPVP ok

version bump matches the API change (PVP)

Dependencies added: ShellCheck

API changes (from Hackage documentation)

- Hledger.Flow.Import.Dependencies: checkDirectives :: Release -> AbsFile -> ByteString -> ReuseDecision ()
- Hledger.Flow.Import.Dependencies: resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)
- Hledger.Flow.Status: instance GHC.Classes.Eq Hledger.Flow.Status.LedgerFile
- Hledger.Flow.Status: instance GHC.Internal.Show.Show Hledger.Flow.Status.LedgerFile
+ Hledger.Flow.Cache: initializeIgnoredDir :: AbsDir -> IO ()
+ Hledger.Flow.Cache: trashDir :: AbsDir -> AbsDir
+ Hledger.Flow.Common: createDirectoryCommand :: Shell -> [FilePath] -> AbsDir -> AbsDir -> Text
+ Hledger.Flow.Common: includePreamble :: Text
+ Hledger.Flow.Common: makeDirectory :: Shell -> Text -> Text
+ Hledger.Flow.Common: moveFileCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> RelFile -> RelFile -> Text
+ Hledger.Flow.DocHelpers: Balances :: DocLink
+ Hledger.Flow.DocHelpers: ManualJournals :: DocLink
+ Hledger.Flow.DocHelpers: PriceFiles :: DocLink
+ Hledger.Flow.DocHelpers: ReviewTransfers :: DocLink
+ Hledger.Flow.DocHelpers: ReviewUnknownTransactions :: DocLink
+ Hledger.Flow.DocHelpers: StatementYearFolders :: DocLink
+ Hledger.Flow.HledgerConfig: BalanceConfig :: ConfigUse
+ Hledger.Flow.HledgerConfig: ConfigLine :: Text -> Int -> Text -> [Text] -> Bool -> ConfigLine
+ Hledger.Flow.HledgerConfig: ImportConfig :: ConfigUse
+ Hledger.Flow.HledgerConfig: ReportConfig :: ConfigUse
+ Hledger.Flow.HledgerConfig: [configArguments] :: ConfigLine -> [Text]
+ Hledger.Flow.HledgerConfig: [configLineNumber] :: ConfigLine -> Int
+ Hledger.Flow.HledgerConfig: [configSection] :: ConfigLine -> Text
+ Hledger.Flow.HledgerConfig: [configSource] :: ConfigLine -> Text
+ Hledger.Flow.HledgerConfig: [configStartsSection] :: ConfigLine -> Bool
+ Hledger.Flow.HledgerConfig: configConflicts :: FilePath -> [ConfigLine] -> [ConfigConflict]
+ Hledger.Flow.HledgerConfig: data ConfigConflict
+ Hledger.Flow.HledgerConfig: data ConfigLine
+ Hledger.Flow.HledgerConfig: data ConfigUse
+ Hledger.Flow.HledgerConfig: findConfigConflict :: HledgerInfo -> ConfigUse -> Maybe HledgerConf -> IO (Maybe ConfigConflict)
+ Hledger.Flow.HledgerConfig: findConfigConflicts :: HledgerInfo -> Maybe HledgerConf -> IO [ConfigConflict]
+ Hledger.Flow.HledgerConfig: firstConfigConflict :: ConfigUse -> [ConfigConflict] -> Maybe ConfigConflict
+ Hledger.Flow.HledgerConfig: instance GHC.Classes.Eq Hledger.Flow.HledgerConfig.ConfigConflict
+ Hledger.Flow.HledgerConfig: instance GHC.Classes.Eq Hledger.Flow.HledgerConfig.ConfigLine
+ Hledger.Flow.HledgerConfig: instance GHC.Classes.Eq Hledger.Flow.HledgerConfig.ConfigUse
+ Hledger.Flow.HledgerConfig: instance GHC.Internal.Show.Show Hledger.Flow.HledgerConfig.ConfigConflict
+ Hledger.Flow.HledgerConfig: instance GHC.Internal.Show.Show Hledger.Flow.HledgerConfig.ConfigLine
+ Hledger.Flow.HledgerConfig: instance GHC.Internal.Show.Show Hledger.Flow.HledgerConfig.ConfigUse
+ Hledger.Flow.HledgerConfig: parseConfigLines :: Bool -> [Text] -> [ConfigLine]
+ Hledger.Flow.HledgerConfig: renderConfigConflict :: ConfigConflict -> Text
+ Hledger.Flow.HledgerConfig: requireSafeConfig :: ConfigUse -> RuntimeOptions -> IO ()
+ Hledger.Flow.Import.Dependencies: ConfigEntry :: Text -> FilePath -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: CsvRules :: Release -> IncludeFile
+ Hledger.Flow.Import.Dependencies: IncludeNode :: AbsFile -> AbsFile -> [AbsDir] -> ByteString -> IncludeNode
+ Hledger.Flow.Import.Dependencies: JournalDirectives :: IncludeFile
+ Hledger.Flow.Import.Dependencies: [includeBytes] :: IncludeNode -> ByteString
+ Hledger.Flow.Import.Dependencies: [includeDirectories] :: IncludeNode -> [AbsDir]
+ Hledger.Flow.Import.Dependencies: [includePosition] :: IncludeNode -> AbsFile
+ Hledger.Flow.Import.Dependencies: [includeSource] :: IncludeNode -> AbsFile
+ Hledger.Flow.Import.Dependencies: data IncludeFile
+ Hledger.Flow.Import.Dependencies: data IncludeNode
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq Hledger.Flow.Import.Dependencies.IncludeFile
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show Hledger.Flow.Import.Dependencies.IncludeFile
+ Hledger.Flow.Import.Dependencies: journalIncludes :: AbsFile -> ByteString -> ReuseDecision [FilePath]
+ Hledger.Flow.Import.Dependencies: readIncludeTree :: AbsDir -> IncludeFile -> AbsFile -> IO (ReuseDecision [IncludeNode])
+ Hledger.Flow.Import.Dependencies: resolveInclude :: IncludeFile -> AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)
+ Hledger.Flow.Import.Dependencies: scanIncludes :: IncludeFile -> Bool -> AbsFile -> ByteString -> ReuseDecision [FilePath]
+ Hledger.Flow.Import.ShellSources: data SourceInput
+ Hledger.Flow.Import.ShellSources: hookSources :: SourceCache -> AbsFile -> FileObservation -> IO [SourceInput]
+ Hledger.Flow.Import.ShellSources: instance GHC.Classes.Eq Hledger.Flow.Import.ShellSources.SourceInput
+ Hledger.Flow.Import.ShellSources: instance GHC.Internal.Show.Show Hledger.Flow.Import.ShellSources.SourceInput
+ Hledger.Flow.Import.ShellSources: newSourceCache :: IO SourceCache
+ Hledger.Flow.Import.ShellSources: sourceKey :: SourceInput -> [(Text, Text)]
+ Hledger.Flow.Import.ShellSources: type SourceCache = IORef Map Digest [Step]
+ Hledger.Flow.Init.Example: directivesAreStarter :: BaseDir -> IO Bool
+ Hledger.Flow.Init.Example: exampleAccount :: RelDir -> RelDir
+ Hledger.Flow.Init.Example: exampleAccountPath :: RelDir -> Text
+ Hledger.Flow.Init.Example: exampleBaseFiles :: Integer -> [(RelFile, Text)]
+ Hledger.Flow.Init.Example: exampleImportFiles :: RelDir -> Integer -> [(RelFile, Text)]
+ Hledger.Flow.Init.Example: isSampleStatement :: AbsFile -> Bool
+ Hledger.Flow.Init.Example: journalIsCommentsOnly :: AbsFile -> IO Bool
+ Hledger.Flow.Init.Example: sampleAccountDir :: BaseDir -> RelDir -> AbsDir
+ Hledger.Flow.Init.Example: starterOwner :: IO RelDir
+ Hledger.Flow.Init.Example: untouchedExample :: BaseDir -> [AbsFile] -> IO (Maybe RelDir)
+ Hledger.Flow.InputSnapshot: data InputSnapshot
+ Hledger.Flow.InputSnapshot: inputTargets :: InputSnapshot -> [FilePath]
+ Hledger.Flow.InputSnapshot: inputsMatch :: InputSnapshot -> IO Bool
+ Hledger.Flow.InputSnapshot: inputsUnchanged :: InputSnapshot -> UTCTime -> IO Bool
+ Hledger.Flow.InputSnapshot: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: instance GHC.Classes.Eq Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance GHC.Classes.Eq Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: instance GHC.Internal.Generics.Generic Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance GHC.Internal.Generics.Generic Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: instance GHC.Internal.Show.Show Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance GHC.Internal.Show.Show Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: snapshotInputs :: [AbsFile] -> IO InputSnapshot
+ Hledger.Flow.LedgerSelection: CoversLedger :: Coverage
+ Hledger.Flow.LedgerSelection: LedgerFile :: FilePath -> Bool -> Coverage -> LedgerFile
+ Hledger.Flow.LedgerSelection: LedgerSelection :: Maybe LedgerFile -> Maybe AbsFile -> Maybe Text -> LedgerSelection
+ Hledger.Flow.LedgerSelection: UnreadableLedger :: Text -> Coverage
+ Hledger.Flow.LedgerSelection: UnrelatedLedger :: Coverage
+ Hledger.Flow.LedgerSelection: [aggregateJournal] :: LedgerSelection -> Maybe AbsFile
+ Hledger.Flow.LedgerSelection: [lfCoverage] :: LedgerFile -> Coverage
+ Hledger.Flow.LedgerSelection: [lfExists] :: LedgerFile -> Bool
+ Hledger.Flow.LedgerSelection: [lfValue] :: LedgerFile -> FilePath
+ Hledger.Flow.LedgerSelection: [selectedLedgerFile] :: LedgerSelection -> Maybe LedgerFile
+ Hledger.Flow.LedgerSelection: [selectionError] :: LedgerSelection -> Maybe Text
+ Hledger.Flow.LedgerSelection: coverageFor :: FilePath -> Maybe FilePath -> [FilePath] -> Coverage
+ Hledger.Flow.LedgerSelection: data Coverage
+ Hledger.Flow.LedgerSelection: data LedgerFile
+ Hledger.Flow.LedgerSelection: data LedgerSelection
+ Hledger.Flow.LedgerSelection: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.LedgerSelection.CachedInputs
+ Hledger.Flow.LedgerSelection: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.LedgerSelection.CachedInputs
+ Hledger.Flow.LedgerSelection: instance GHC.Classes.Eq Hledger.Flow.LedgerSelection.Coverage
+ Hledger.Flow.LedgerSelection: instance GHC.Classes.Eq Hledger.Flow.LedgerSelection.LedgerFile
+ Hledger.Flow.LedgerSelection: instance GHC.Internal.Generics.Generic Hledger.Flow.LedgerSelection.CachedInputs
+ Hledger.Flow.LedgerSelection: instance GHC.Internal.Show.Show Hledger.Flow.LedgerSelection.Coverage
+ Hledger.Flow.LedgerSelection: instance GHC.Internal.Show.Show Hledger.Flow.LedgerSelection.LedgerFile
+ Hledger.Flow.LedgerSelection: instance GHC.Internal.Show.Show Hledger.Flow.LedgerSelection.LedgerSelection
+ Hledger.Flow.LedgerSelection: journalInputs :: HledgerInfo -> [AbsFile] -> IO (Either Text [AbsFile])
+ Hledger.Flow.LedgerSelection: ledgerFileProblem :: [FilePath] -> LedgerFile -> Text
+ Hledger.Flow.LedgerSelection: selectLedger :: [FilePath] -> Maybe HledgerInfo -> BaseDir -> IO LedgerSelection
+ Hledger.Flow.PathHelpers: pathOccupied :: Path Abs t -> IO Bool
+ Hledger.Flow.PathHelpers: publishNewFile :: AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.PathHelpers: writeNewUtf8File :: AbsFile -> Text -> IO ()
+ Hledger.Flow.PathHelpers: writeTimestampMarker :: AbsFile -> IO UTCTime
+ Hledger.Flow.ReportReview: Review :: ReviewCategory -> ReviewCategory -> [ReviewItem] -> Review
+ Hledger.Flow.ReportReview: ReviewCategory :: Maybe Int -> [FilePath] -> ReviewCategory
+ Hledger.Flow.ReportReview: ReviewItem :: ReviewKind -> FilePath -> Integer -> Maybe Int -> ReviewItem
+ Hledger.Flow.ReportReview: TransferBalances :: ReviewKind
+ Hledger.Flow.ReportReview: UnknownTransactions :: ReviewKind
+ Hledger.Flow.ReportReview: [itemCount] :: ReviewItem -> Maybe Int
+ Hledger.Flow.ReportReview: [itemKind] :: ReviewItem -> ReviewKind
+ Hledger.Flow.ReportReview: [itemPath] :: ReviewItem -> FilePath
+ Hledger.Flow.ReportReview: [itemYear] :: ReviewItem -> Integer
+ Hledger.Flow.ReportReview: [reviewCount] :: ReviewCategory -> Maybe Int
+ Hledger.Flow.ReportReview: [reviewFiles] :: ReviewCategory -> [FilePath]
+ Hledger.Flow.ReportReview: [reviewItems] :: Review -> [ReviewItem]
+ Hledger.Flow.ReportReview: [reviewTransfers] :: Review -> ReviewCategory
+ Hledger.Flow.ReportReview: [reviewUnknowns] :: Review -> ReviewCategory
+ Hledger.Flow.ReportReview: captureReportInputs :: HledgerInfo -> UTCTime -> AbsFile -> Maybe HledgerConf -> [[AbsFile]] -> IO (Either Text ReportInputs)
+ Hledger.Flow.ReportReview: countTransferAccounts :: Text -> Maybe Int
+ Hledger.Flow.ReportReview: countUnknownTransactions :: Text -> Maybe Int
+ Hledger.Flow.ReportReview: data Review
+ Hledger.Flow.ReportReview: data ReviewCategory
+ Hledger.Flow.ReportReview: data ReviewFile
+ Hledger.Flow.ReportReview: data ReviewItem
+ Hledger.Flow.ReportReview: data ReviewKind
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReportReview.Manifest
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReportReview.ReportInputs
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReportReview.ReviewFile
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReportReview.Manifest
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReportReview.ReportInputs
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReportReview.ReviewFile
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: instance GHC.Classes.Eq Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: instance GHC.Internal.Generics.Generic Hledger.Flow.ReportReview.Manifest
+ Hledger.Flow.ReportReview: instance GHC.Internal.Generics.Generic Hledger.Flow.ReportReview.ReportInputs
+ Hledger.Flow.ReportReview: instance GHC.Internal.Generics.Generic Hledger.Flow.ReportReview.ReviewFile
+ Hledger.Flow.ReportReview: instance GHC.Internal.Generics.Generic Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: instance GHC.Internal.Show.Show Hledger.Flow.ReportReview.Review
+ Hledger.Flow.ReportReview: instance GHC.Internal.Show.Show Hledger.Flow.ReportReview.ReviewCategory
+ Hledger.Flow.ReportReview: instance GHC.Internal.Show.Show Hledger.Flow.ReportReview.ReviewItem
+ Hledger.Flow.ReportReview: instance GHC.Internal.Show.Show Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: invalidateReportReview :: BaseDir -> IO ()
+ Hledger.Flow.ReportReview: loadReportReview :: BaseDir -> AbsFile -> Maybe HledgerConf -> IO (Maybe Review)
+ Hledger.Flow.ReportReview: reviewFile :: BaseDir -> AbsFile -> Maybe ReviewFile
+ Hledger.Flow.ReportReview: reviewHasData :: ReviewKind -> Text -> Bool
+ Hledger.Flow.ReportReview: saveReportReview :: BaseDir -> ReportInputs -> [ReviewFile] -> IO ()
+ Hledger.Flow.ReportReview: startReportRun :: BaseDir -> IO UTCTime
+ Hledger.Flow.RulesTemplate: rulesStarterFor :: BaseDir -> Text -> IO (Either Text (AbsFile, Text))
+ Hledger.Flow.RulesTemplate: runRules :: Maybe HledgerConf -> BaseDir -> Text -> IO ()
+ Hledger.Flow.StatementScan: DelimitedStatement :: Char -> Maybe [Text] -> [[Text]] -> DelimitedStatement
+ Hledger.Flow.StatementScan: SingleYear :: Integer -> StatementYear
+ Hledger.Flow.StatementScan: SpanningYears :: [Integer] -> StatementYear
+ Hledger.Flow.StatementScan: UnknownYear :: StatementYear
+ Hledger.Flow.StatementScan: [statementHeader] :: DelimitedStatement -> Maybe [Text]
+ Hledger.Flow.StatementScan: [statementRows] :: DelimitedStatement -> [[Text]]
+ Hledger.Flow.StatementScan: [statementSeparator] :: DelimitedStatement -> Char
+ Hledger.Flow.StatementScan: data DateCell
+ Hledger.Flow.StatementScan: data DelimitedStatement
+ Hledger.Flow.StatementScan: data StatementYear
+ Hledger.Flow.StatementScan: dateCellYear :: DateCell -> Integer
+ Hledger.Flow.StatementScan: inferDateFormat :: [DateCell] -> Maybe Text
+ Hledger.Flow.StatementScan: inferStatementYear :: DelimitedStatement -> StatementYear
+ Hledger.Flow.StatementScan: instance GHC.Classes.Eq Hledger.Flow.StatementScan.DateCell
+ Hledger.Flow.StatementScan: instance GHC.Classes.Eq Hledger.Flow.StatementScan.DelimitedStatement
+ Hledger.Flow.StatementScan: instance GHC.Classes.Eq Hledger.Flow.StatementScan.StatementYear
+ Hledger.Flow.StatementScan: instance GHC.Internal.Show.Show Hledger.Flow.StatementScan.DateCell
+ Hledger.Flow.StatementScan: instance GHC.Internal.Show.Show Hledger.Flow.StatementScan.DelimitedStatement
+ Hledger.Flow.StatementScan: instance GHC.Internal.Show.Show Hledger.Flow.StatementScan.StatementYear
+ Hledger.Flow.StatementScan: isDelimitedFile :: FilePath -> Bool
+ Hledger.Flow.StatementScan: isSignedDecimal :: Text -> Bool
+ Hledger.Flow.StatementScan: looksLikeHeader :: [Text] -> Bool
+ Hledger.Flow.StatementScan: parseDateCell :: Text -> Maybe DateCell
+ Hledger.Flow.StatementScan: parseDelimitedStatement :: FilePath -> Text -> Either Text DelimitedStatement
+ Hledger.Flow.StatementScan: parseRuleSample :: FilePath -> Text -> Either Text DelimitedStatement
+ Hledger.Flow.StatementScan: readDelimitedStatement :: Maybe Int -> AbsFile -> IO (Either Text DelimitedStatement)
+ Hledger.Flow.StatementScan: readRuleSample :: AbsFile -> IO (Either Text DelimitedStatement)
+ Hledger.Flow.StatementScan: splitHeader :: Text -> (Char, [Text])
+ Hledger.Flow.Status: [asOtherRules] :: AccountStatus -> [FilePath]
+ Hledger.Flow.Status: [asRulesTargetOccupied] :: AccountStatus -> Bool
+ Hledger.Flow.Status: [lfCoverage] :: LedgerFile -> Coverage
+ Hledger.Flow.Status: [lsAggregateJournal] :: LedgerStatus -> Maybe AbsFile
+ Hledger.Flow.Status: [lsConfigConflicts] :: LedgerStatus -> [ConfigConflict]
+ Hledger.Flow.Status: [lsFirstStatementOwner] :: LedgerStatus -> RelDir
+ Hledger.Flow.Status: [lsHasManualWork] :: LedgerStatus -> Bool
+ Hledger.Flow.Status: [lsHledgerConf] :: LedgerStatus -> Maybe HledgerConf
+ Hledger.Flow.Status: [lsNeedsPrices] :: LedgerStatus -> Bool
+ Hledger.Flow.Status: [lsReview] :: LedgerStatus -> Maybe Review
+ Hledger.Flow.Status: [lsUntouchedExample] :: LedgerStatus -> Maybe RelDir
+ Hledger.Flow.Status: [mpDestinationExists] :: Misplaced -> Bool
+ Hledger.Flow.Status: [mpYear] :: Misplaced -> StatementYear
+ Hledger.Flow.Status: balanceHint :: Shell -> [FilePath] -> LedgerStatus -> Maybe Text
+ Hledger.Flow.Status: firstStatementStep :: Integer -> Shell -> [FilePath] -> AbsDir -> BaseDir -> RelDir -> NextStep
+ Hledger.Flow.Status: flowCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Maybe HledgerConf -> Text -> [Text] -> Text
+ Hledger.Flow.Status: ledgerFileNote :: Shell -> [FilePath] -> LedgerStatus -> Maybe Text
+ Hledger.Flow.Status: misplacedMoveCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Misplaced -> Maybe Text
+ Hledger.Flow.Status: placementDestination :: Misplaced -> Maybe RelFile
+ Hledger.Flow.Status: withConfigCheck :: ConfigUse -> LedgerStatus -> NextStep -> NextStep
+ Hledger.Flow.Status.Gather: gatherStatusWithConfig :: Maybe HledgerInfo -> Maybe HledgerConf -> BaseDir -> IO LedgerStatus
+ Hledger.Flow.Status.Gather: mainRulesCandidates :: AbsDir -> [AbsFile] -> IO [AbsFile]
- Hledger.Flow.Import.Dependencies: discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile, Digest)))
+ Hledger.Flow.Import.Dependencies: discoverDirectives :: AbsFile -> IO (ReuseDecision [(AbsFile, Digest)])
- Hledger.Flow.Import.ImportHelpers: withStagedOutput :: AbsFile -> (AbsFile -> IO ()) -> IO ()
+ Hledger.Flow.Import.ImportHelpers: withStagedOutput :: AbsFile -> (AbsFile -> IO a) -> IO a
- Hledger.Flow.Import.IncludeFiles: includesNeedUpdate :: BaseDir -> AbsDir -> [AbsFile] -> [AbsFile] -> IO Bool
+ Hledger.Flow.Import.IncludeFiles: includesNeedUpdate :: BaseDir -> AbsDir -> [AbsFile] -> [AbsFile] -> Bool -> IO Bool
- Hledger.Flow.Import.Inventory: moveToTrash :: AbsFile -> IO AbsFile
+ Hledger.Flow.Import.Inventory: moveToTrash :: AbsDir -> AbsFile -> IO AbsFile
- Hledger.Flow.Import.Snapshot: snapshotManifest :: PreparedSnapshot -> Map AbsFile Digest
+ Hledger.Flow.Import.Snapshot: snapshotManifest :: PreparedSnapshot -> [(AbsFile, Digest)]
- Hledger.Flow.Init: Created :: BaseDir -> Maybe BaseDir -> InitOutcome
+ Hledger.Flow.Init: Created :: BaseDir -> Maybe BaseDir -> RelDir -> [(RelFile, Bool)] -> InitOutcome
- Hledger.Flow.Init: runInit :: Maybe FilePath -> AbsDir -> AbsDir -> IO ()
+ Hledger.Flow.Init: runInit :: Maybe FilePath -> Maybe FilePath -> AbsDir -> AbsDir -> IO ()
- Hledger.Flow.PathHelpers: withStagedOutput :: AbsFile -> (AbsFile -> IO ()) -> IO ()
+ Hledger.Flow.PathHelpers: withStagedOutput :: AbsFile -> (AbsFile -> IO a) -> IO a
- Hledger.Flow.Status: AccountStatus :: Text -> Int -> [Integer] -> Maybe FilePath -> [FilePath] -> Int -> Bool -> Bool -> Int -> Int -> AccountStatus
+ Hledger.Flow.Status: AccountStatus :: Text -> Int -> [Integer] -> Maybe FilePath -> [FilePath] -> Int -> [FilePath] -> Bool -> Bool -> Bool -> Int -> Int -> AccountStatus
- Hledger.Flow.Status: LedgerFile :: FilePath -> Bool -> LedgerFile
+ Hledger.Flow.Status: LedgerFile :: FilePath -> Bool -> Coverage -> LedgerFile
- Hledger.Flow.Status: LedgerStatus :: BaseDir -> Maybe HledgerInfo -> Maybe LedgerFile -> [AccountStatus] -> [Misplaced] -> Int -> Bool -> Bool -> LedgerStatus
+ Hledger.Flow.Status: LedgerStatus :: BaseDir -> Maybe HledgerInfo -> Maybe LedgerFile -> Maybe AbsFile -> Maybe HledgerConf -> [ConfigConflict] -> [AccountStatus] -> [Misplaced] -> Int -> Bool -> Bool -> Bool -> Bool -> Maybe Review -> RelDir -> Maybe RelDir -> LedgerStatus
- Hledger.Flow.Status: Misplaced :: FilePath -> FilePath -> Misplaced
+ Hledger.Flow.Status: Misplaced :: FilePath -> FilePath -> StatementYear -> Bool -> Misplaced
- Hledger.Flow.Status.Render: printNext :: Maybe FilePath -> BaseDir -> IO ()
+ Hledger.Flow.Status.Render: printNext :: Maybe FilePath -> Maybe HledgerConf -> BaseDir -> IO ()
- Hledger.Flow.Status.Render: statusJSON :: SystemInfo -> Maybe HledgerConf -> LedgerStatus -> NextStep -> Value
+ Hledger.Flow.Status.Render: statusJSON :: SystemInfo -> Maybe HledgerConf -> AbsDir -> LedgerStatus -> NextStep -> Value

Files

ChangeLog.md view
@@ -31,6 +31,16 @@   existing destination belongs in release-procedure.md. --> +## 0.21.0++- `import` moves orphaned generated files to `.hledger-flow/trash/` at the ledger root instead of each account's `.trash` folder, keeping their path from the root. The trash has its own `.gitignore`. Recover what you need from existing account `.trash` folders, then delete them and any `.trash/` line in your `.gitignore`. See [generated files and recovery](https://hledger-flow.github.io/reference/input-files/#generated-files-and-recovery).+- `import` and `report` now stop before writing output if the hledger config they use supplies `-f`, `--file` or an argument file such as `@args.txt`. The error names the file and line. Remove the setting or pass a separate config with `--hledger-conf`. See [hledger config file](https://hledger-flow.github.io/reference/status/#hledger-config-file).+- If `LEDGER_FILE` points to an unrelated ledger, `report` now reads the local `all-years.journal` for aggregate reports. To keep using a custom journal, make it include `import/all-years.journal` and, if you have one, `directives.journal`. `status` shows which journal was selected. See [journal used for reports](https://hledger-flow.github.io/reference/status/#journal-used-for-reports).+- `init` creates a working example account under your username, with starter journals for manual entries, opening and closing balances, directives and prices. The [tutorial](https://hledger-flow.github.io/tutorial/) starts from it. `status` then prints a command for your shell that creates your first account's input directory, keeping real statements out of the example.+- Run `hledger-flow rules OWNER/BANK/ACCOUNT` to create a starter rules file in the account folder. `status --rules-template` also prints [starter rules for statements without a header](https://hledger-flow.github.io/reference/rules-files/#starter-rules-for-exports-without-a-header), with a sample row. Generated date formats accept days and months with or without leading zeroes.+- `status` guides more steps. It helps you [reuse existing account rules](https://hledger-flow.github.io/reference/rules-files/#reuse-existing-rules) when the main rules file needs a different name, and suggests [year folders](https://hledger-flow.github.io/reference/input-files/#year-folders) for misplaced delimited statements, with safe move commands when the destination is known. After a successful report, it points to [unknown transactions and transfers that need review](https://hledger-flow.github.io/reference/status/#review-unknown-transactions). When the imported journal uses more than one commodity and has no market price directives, it suggests adding [market prices](https://hledger-flow.github.io/reference/price-files/), with one file per currency pair.+- `import --incremental` reuses journals in more cases and notices more changes. It follows CSV rules that include shared files outside the ledger by relative path, literal journal includes in `directives.journal`, and, with `--incremental=skip-unchanged-scripts`, [sourced shell helpers](https://hledger-flow.github.io/reference/incremental-import/#sourced-shell-helpers). Editing one of these files rebuilds the statements that depend on it. More `hledger.conf` settings allow reuse. When a setting such as a date relative to today or a reference to an untracked file prevents it, the notice quotes the setting and its line. See [what gets checked](https://hledger-flow.github.io/reference/incremental-import/#what-gets-checked).+ ## 0.20.1  - Use `hledger-flow import --incremental` with hledger 1.52.4 or later to reuse journals when their inputs, including nested rules files, still match. Choose `--incremental=skip-unchanged-scripts` to also reuse the outputs of unchanged scripts. See the [incremental import reference](https://hledger-flow.github.io/reference/incremental-import/).@@ -38,7 +48,7 @@ - On macOS, incremental import checks the hledger executable faster and still detects a replacement build that reports the same version. - `hledger-flow import --verbose` and `hledger-flow report --verbose` are   easier to read. Each statement's lines appear together, files are labelled-  by what import looked for, and every hledger or hook command is shown ready+  by what import looked for, and every hledger or import script command is shown ready   to paste into a shell at the base directory, with paths there starting   with `./`. On a terminal, the output uses colour unless   `NO_COLOR` is set.@@ -157,9 +167,9 @@   Each binary archive now includes a [quick-start](https://github.com/apauley/hledger-flow/blob/master/QUICKSTART.md)   covering installation, download verification and a first import. - Windows imports now run `preprocess.ps1` and `construct.ps1` directly-  with PowerShell. Native hooks can use `preprocess.exe` and `construct.exe`.-  Existing extensionless hooks take precedence. See-  [Windows import hooks](https://github.com/apauley/hledger-flow/blob/master/docs/README.md#windows-import-hooks).+  with PowerShell. Native executables can be named `preprocess.exe` and `construct.exe`.+  Existing extensionless scripts take precedence. See+  [Windows import scripts](https://github.com/apauley/hledger-flow/blob/master/docs/README.md#windows-import-hooks). - Manual-only years and ledgers now get include files without dummy inputs,   including `_manual_` files at the import, owner, bank, and account levels.   Fixes [#103](https://github.com/apauley/hledger-flow/issues/103).@@ -180,9 +190,9 @@  ## 0.17.0 -- If you use a custom `preprocess` hook, write to the path supplied as its+- If you use a custom `preprocess` script, write to the path supplied as its   second argument. It now points into a temporary directory and keeps the-  final filename and extension. Failed hooks preserve the previous+  final filename and extension. Failed scripts preserve the previous   preprocessed CSV or constructed journal. - If you use the `Hledger.Flow.*` library, update calls that use Turtle   types to use `path` and `Text` types. The `TurtlePath` alias has been
app/Main.hs view
@@ -54,10 +54,11 @@ import Hledger.Flow.Status.Render ( printStatus ) import qualified Data.Text as Text import Path.IO ( getCurrentDir, resolveDir' )+import Hledger.Flow.HledgerConfig (ConfigUse (..), requireSafeConfig) import Hledger.Flow.Init ( runInit ) import Hledger.Flow.DocHelpers ( DocLink (..), docURL, siteURL ) import Hledger.Flow.Logging ( useColor )-import Hledger.Flow.RulesTemplate ( rulesTemplateFor )+import Hledger.Flow.RulesTemplate ( rulesTemplateFor, runRules ) import Data.Maybe (fromMaybe) import Hledger.Flow.Internals (availableCores, versionInfo, systemInfo) import Hledger.Flow.BaseDir ( determineBaseDir, findBaseDir )@@ -92,7 +93,7 @@  newtype InitParams = InitParams { initDir :: Maybe FilePath } deriving (Show) -data Command = Import ImportParams | Report ReportParams | Status StatusParams | Init InitParams deriving (Show)+data Command = Import ImportParams | Report ReportParams | Status StatusParams | Rules String (Maybe FilePath) | Init InitParams deriving (Show)  data MainParams = MainParams { verbosity :: Int                              , hledgerPathOpt :: Maybe FilePath@@ -151,15 +152,23 @@       T.putStrLn $ versionInfo sysInfo      Command mainParams' (Just (Import subParams)) -> do       (base, runDir) <- determineBaseDir (maybeImportBaseDir subParams)-      withReleaseNotice base $ (toRuntimeOptionsImport base runDir mainParams' subParams >>= importCSVs) `catch` reportFatal+      opts <- toRuntimeOptionsImport base runDir mainParams' subParams+      requireSafeConfig ImportConfig opts+      withReleaseNotice base $ importCSVs opts `catch` reportFatal     Command mainParams' (Just (Report subParams)) -> do       (base, _) <- determineBaseDir (maybeReportBaseDir subParams)-      withReleaseNotice base $ (toRuntimeOptionsReport base mainParams' subParams >>= generateReports) `catch` reportFatal+      opts <- toRuntimeOptionsReport base mainParams' subParams+      requireSafeConfig ReportConfig opts+      withReleaseNotice base $ generateReports opts `catch` reportFatal     Command mainParams' (Just (Status subParams)) -> runStatus mainParams' subParams+    Command mainParams' (Just (Rules account dir)) -> do+      base <- statusBaseDir dir+      conf <- findHledgerConf (hledgerConfOpt mainParams')+      runRules conf base (Text.pack account)     Command mainParams' (Just (Init subParams)) -> do       cwd <- getCurrentDir       dir <- maybe (return cwd) resolveDir' (initDir subParams)-      runInit (hledgerPathOpt mainParams') cwd dir+      runInit (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') cwd dir     Command mainParams' Nothing -> runBare mainParams'  -- | (name, one-line description) of each command, in display order.@@ -167,6 +176,7 @@ commandSummaries =   [ ("status", "Show what's in your ledger and what to do next"),     ("init", "Start a new ledger in a folder"),+    ("rules", "Create a starter rules file for an account"),     ("import", "Turn statements into journals"),     ("report", "Generate reports from your journals")   ]@@ -184,14 +194,18 @@  runStatus :: MainParams -> StatusParams -> IO () runStatus mainParams' params = do-  cwd <- getCurrentDir-  dir <- maybe (return cwd) (resolveExistingDir cwd) (statusDir params)-  base <- findBaseDir dir >>= maybe (throwIO (MissingBaseDir dir)) return+  base <- statusBaseDir (statusDir params)   case statusRulesTemplate params of     Just account -> rulesTemplateFor base (Text.pack account) >>= either (ioError . userError . Text.unpack) T.putStr     Nothing | statusJson params -> printStatus (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') True base             | otherwise -> printInteractiveStatus mainParams' base +statusBaseDir :: Maybe FilePath -> IO BaseDir+statusBaseDir target = do+  cwd <- getCurrentDir+  dir <- maybe (return cwd) (resolveExistingDir cwd) target+  findBaseDir dir >>= maybe (throwIO (MissingBaseDir dir)) return+ printInteractiveStatus :: MainParams -> BaseDir -> IO () printInteractiveStatus params base = withCachedReleaseNotice base $   printStatus (hledgerPathOpt params) (hledgerConfOpt params) False base@@ -264,6 +278,10 @@       ("hledger-flow status --rules-template me/mybank/checking", "Print a starter rules file for one account") ]     <> subcommand "init" GettingStarted (Init . InitParams <$> optional (argument str (metavar "DIR" <> help "The folder for the new ledger, created if missing. Default: the current folder.")))     [ ("hledger-flow init ~/finances", "Start a ledger in ~/finances") ]+    <> subcommand "rules" RulesFiles+      (Rules <$> argument str (metavar "ACCOUNT" <> help "Account as owner/bank/account.")+             <*> optional (argument str (metavar "DIR" <> help "Inspect the ledger containing this folder.")))+    [ ("hledger-flow rules me/mybank/checking", "Write a starter rules file for this account") ]     <> subcommand "import" InputFiles (Import <$> subcommandParserImport colour)     [ ("hledger-flow import", "Import the current folder"),       ("hledger-flow import import/me/mybank", "Import one bank's statements"),
hledger-flow.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack  name:           hledger-flow-version:        0.20.1+version:        0.21.0 synopsis:       An hledger workflow focusing on automated statement import and classification. description:    hledger-flow uses hledger's CSV rules to turn each bank statement into a                 separate journal. It combines those journals with your manual entries for@@ -38,6 +38,7 @@       Hledger.Flow.DateTime       Hledger.Flow.DocHelpers       Hledger.Flow.Environment+      Hledger.Flow.HledgerConfig       Hledger.Flow.Import.Cache       Hledger.Flow.Import.Conversion       Hledger.Flow.Import.CSVImport@@ -48,16 +49,22 @@       Hledger.Flow.Import.Incremental       Hledger.Flow.Import.Inventory       Hledger.Flow.Import.PrivateDirectory+      Hledger.Flow.Import.ShellSources       Hledger.Flow.Import.Snapshot       Hledger.Flow.Import.Types       Hledger.Flow.Init+      Hledger.Flow.Init.Example+      Hledger.Flow.InputSnapshot       Hledger.Flow.Internals+      Hledger.Flow.LedgerSelection       Hledger.Flow.Logging       Hledger.Flow.PathHelpers       Hledger.Flow.ReleaseNotice+      Hledger.Flow.ReportReview       Hledger.Flow.Reports       Hledger.Flow.RulesTemplate       Hledger.Flow.RuntimeOptions+      Hledger.Flow.StatementScan       Hledger.Flow.Status       Hledger.Flow.Status.Gather       Hledger.Flow.Status.Render@@ -68,7 +75,8 @@       src   ghc-options: -Wall   build-depends:-      aeson >=2.2.5.1 && <2.3+      ShellCheck ==0.10.*+    , aeson >=2.2.5.1 && <2.3     , ansi-terminal >=1.1.5 && <1.2     , async >=2.2.6 && <2.3     , base >=4.20 && <4.22@@ -136,14 +144,17 @@       CSVImport.Integration       CSVImport.Unit       Environment.Unit+      HledgerConfig.Unit       ImportHelpers.Integration       Incremental.Cache       Incremental.Dependencies       Incremental.Platform       Incremental.Snapshot       Init.Integration+      LedgerSelection.Unit       PathHelpers.Unit       ReleaseNotice.Unit+      ReportReview.Unit       RulesTemplate.Unit       RuntimeOptions.Unit       Status.Integration
site/go-links.json view
@@ -2,10 +2,17 @@   "docs": "/quickstart/",   "getting-started": "/quickstart/#import-your-first-statement",   "input-files": "/reference/input-files/",+  "statement-year-folders": "/reference/input-files/#year-folders",   "incremental-import-fallback": "/reference/incremental-import/#when-reuse-is-unavailable",   "rules-files": "/reference/rules-files/",+  "price-files": "/reference/price-files/",+  "manual-journals": "/reference/manual-journals/",+  "balances": "/reference/balances/",   "preprocess-script": "/reference/preprocess/",   "set-ledger-file": "/quickstart/#import-your-first-statement",   "status": "/reference/status/",-  "reports": "/tutorial/part-2/#reports"+  "reports": "/tutorial/part-3/#reports",+  "review-unknown-transactions": "/reference/status/#review-unknown-transactions",+  "review-transfers": "/reference/status/#review-transfers",+  "sample-ledger": "/guides/sample-ledger/" }
src/Hledger/Flow/Cache.hs view
@@ -1,7 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} -module Hledger.Flow.Cache (cacheDir, initializeCache) where+module Hledger.Flow.Cache (cacheDir, initializeCache, trashDir, initializeIgnoredDir) where  import Control.Monad (unless) import qualified Data.ByteString as BS@@ -14,14 +14,21 @@ cacheDir :: AbsDir -> AbsDir cacheDir base = base P.</> [P.reldir|.hledger-flow/cache|] +-- | Where import moves generated files whose statement is gone.+trashDir :: AbsDir -> AbsDir+trashDir base = base P.</> [P.reldir|.hledger-flow/trash|]+ initializeCache :: AbsDir -> IO ()-initializeCache base = do-  let cache = cacheDir base-      ignore = cache P.</> [P.relfile|.gitignore|]-      lock = cache P.</> [P.relfile|initialize.lock|]+initializeCache = initializeIgnoredDir . cacheDir++-- | Create a directory whose contents Git ignores.+initializeIgnoredDir :: AbsDir -> IO ()+initializeIgnoredDir dir = do+  let ignore = dir P.</> [P.relfile|.gitignore|]+      lock = dir P.</> [P.relfile|initialize.lock|]   exists <- P.doesFileExist ignore   unless exists $ do-    P.createDirIfMissing True cache+    P.createDirIfMissing True dir     -- Serialize first use across processes. Closing the handle releases the lock.     withBinaryFile (P.toFilePath lock) AppendMode $ \handle -> do       hLock handle ExclusiveLock
src/Hledger/Flow/Common.hs view
@@ -23,10 +23,10 @@ import Hledger.Flow.DateTime (timeAction) import Hledger.Flow.BaseDir (relativeToBase) import Hledger.Flow.Logging-import Hledger.Flow.PathHelpers (AbsFile, RelDir, literalGlobPath, withStagedOutput)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile, literalGlobPath, withStagedOutput) import qualified Hledger.Flow.RuntimeOptions as RT import Hledger.Flow.Types-import Path (reldir, relfile, toFilePath)+import Path (reldir, relfile, stripProperPrefix, toFilePath) import qualified Path import qualified Path.IO as Path import qualified System.Directory as Directory@@ -269,6 +269,42 @@   logFun errorMessage   exitWith (ExitFailure exitStatus) +-- | Move to a file path, with the same collision rules in either shell.+moveFileCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> RelFile -> RelFile -> T.Text+moveFileCommand shell homes cwd base source target = case shell of+  Posix ->+    makeDirectory Posix (path dir) <> " && [ ! -e " <> dest <> " ] && [ ! -L " <> dest+      <> " ] && mv -n " <> path src <> " " <> dest+  PowerShell ->+    makeDirectory PowerShell (literal dir) <> " -ErrorAction Stop | Out-Null; "+      <> "[System.IO.File]::Move(" <> nativePath src <> ", " <> nativePath file <> ")"+  where+    atRoot = cwd == base+    fromHere rel = if atRoot then toFilePath rel else toFilePath (base Path.</> rel)+    src = fromHere source+    file = fromHere target+    dir = FilePath.takeDirectory file+    literal = powerShellLiteral . T.pack+    -- .NET resolves paths against the process directory, not PowerShell's location.+    nativePath name+      | atRoot = "(Join-Path $PWD.ProviderPath " <> literal name <> ")"+      | otherwise = literal name+    path name = fromMaybe (shellArgument shell (T.pack name)) (homeShellPath shell homes name)+    dest = path file++createDirectoryCommand :: Shell -> [FilePath] -> AbsDir -> AbsDir -> T.Text+createDirectoryCommand shell homes cwd dir = makeDirectory shell argument+  where+    absolute = FilePath.dropTrailingPathSeparator (toFilePath dir)+    relative = FilePath.dropTrailingPathSeparator . toFilePath <$> stripProperPrefix cwd dir+    argument = case relative of+      Just path -> shellArgument shell (T.pack path)+      Nothing -> fromMaybe (shellArgument shell (T.pack absolute)) (homeShellPath shell homes absolute)++makeDirectory :: Shell -> T.Text -> T.Text+makeDirectory Posix argument = "mkdir -p " <> argument+makeDirectory PowerShell argument = "New-Item -ItemType Directory -Force -Path " <> argument+ -- | An external program hledger-flow runs, described for the people reading its output. data External = External   { -- | Who printed the output and who failed: @hledger@, or a hook script's path.@@ -549,12 +585,17 @@       return []     Right years -> return years +includePreamble :: T.Text+includePreamble = "### Generated by hledger-flow - DO NOT EDIT ###\n"+ includeYears' :: T.Text -> Either T.Text [Integer]-includeYears' txt = case partitionEithers (includeYears'' txt) of-  (errors, []) -> do-    let msg = "Unable to extract years from the following text:\n" <> txt <> "\nErrors:\n" <> T.intercalate "\n" (map T.pack errors)-    Left msg-  (_, years) -> Right years+includeYears' txt+  | T.stripEnd txt == T.stripEnd includePreamble = Right []+  | otherwise = case partitionEithers (includeYears'' txt) of+      (errors, []) -> do+        let msg = "Unable to extract years from the following text:\n" <> txt <> "\nErrors:\n" <> T.intercalate "\n" (map T.pack errors)+        Left msg+      (_, years) -> Right years  includeYears'' :: T.Text -> [Either String Integer] includeYears'' txt = map extractDigits (T.lines txt)
src/Hledger/Flow/DocHelpers.hs view
@@ -20,11 +20,17 @@   | GettingStarted   | IncrementalImportFallback   | InputFiles+  | StatementYearFolders   | RulesFiles+  | PriceFiles+  | ManualJournals+  | Balances   | PreprocessScript   | SetLedgerFile   | StatusCommand   | Reports+  | ReviewUnknownTransactions+  | ReviewTransfers   deriving (Bounded, Enum, Eq, Show)  docSlug :: DocLink -> Text@@ -32,11 +38,17 @@ docSlug GettingStarted = "getting-started" docSlug IncrementalImportFallback = "incremental-import-fallback" docSlug InputFiles = "input-files"+docSlug StatementYearFolders = "statement-year-folders" docSlug RulesFiles = "rules-files"+docSlug PriceFiles = "price-files"+docSlug ManualJournals = "manual-journals"+docSlug Balances = "balances" docSlug PreprocessScript = "preprocess-script" docSlug SetLedgerFile = "set-ledger-file" docSlug StatusCommand = "status" docSlug Reports = "reports"+docSlug ReviewUnknownTransactions = "review-unknown-transactions"+docSlug ReviewTransfers = "review-transfers"  siteURL :: Text siteURL = "https://hledger-flow.github.io/"
+ src/Hledger/Flow/HledgerConfig.hs view
@@ -0,0 +1,165 @@+{-# LANGUAGE OverloadedStrings #-}++module Hledger.Flow.HledgerConfig+  ( ConfigUse (..),+    ConfigConflict,+    ConfigLine (..),+    parseConfigLines,+    configConflicts,+    firstConfigConflict,+    findConfigConflict,+    findConfigConflicts,+    renderConfigConflict,+    requireSafeConfig,+  )+where++import qualified Data.ByteString as B+import Data.Char (isSpace)+import Data.List (find)+import Data.Maybe (maybeToList)+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import qualified Data.Text.Read as TR+import Hledger.Flow.Common (hledgerVersionNumber)+import qualified Hledger.Flow.RuntimeOptions as RT+import Hledger.Flow.Types (HledgerConf, HledgerInfo (..), hledgerConfPath)++data ConfigLine = ConfigLine+  { configSection :: T.Text,+    configLineNumber :: Int,+    configSource :: T.Text,+    configArguments :: [T.Text],+    configStartsSection :: Bool+  }+  deriving (Eq, Show)++parseConfigLines :: Bool -> [T.Text] -> [ConfigLine]+parseConfigLines allowIndented = parseConfigLinesWith allowIndented configWords++parseConfigLinesWith :: Bool -> (T.Text -> [T.Text]) -> [T.Text] -> [ConfigLine]+parseConfigLinesWith allowIndented splitWords = go "general" . zip [1 ..]+  where+    go _ [] = []+    go section ((n, raw) : more)+      | T.null body || "#" `T.isPrefixOf` body = go section more+      | allowIndented || raw == T.stripStart raw,+        Just start <- T.stripPrefix "[" body,+        (name, end) <- T.breakOn "]" start, not (T.null end) =+          let next = T.strip name+           in ConfigLine next n body (splitWords (T.drop 1 end)) True : go next more+      | otherwise = ConfigLine section n body (splitWords body) False : go section more+      where body = T.strip raw++data ConfigUse = ImportConfig | ReportConfig | BalanceConfig+  deriving (Eq, Show)++data ConfigConflict = ConfigConflict FilePath ConfigLine Bool+  deriving (Eq, Show)++configConflicts :: FilePath -> [ConfigLine] -> [ConfigConflict]+configConflicts file rows = concatMap fromSection sections+  where+    sections = ["general", "print", "balancesheet", "incomestatement", "balance", "accounts"]+    fromSection section = scan+      [ (row, token)+        | row <- reverse (lastDefinition (reverse (filter ((== section) . configSection) rows))),+          token <- configArguments row+      ]+    lastDefinition [] = []+    lastDefinition (row : more) =+      row : if configStartsSection row then [] else lastDefinition more+    scan [] = []+    scan ((row, token) : remaining)+      | token == "--" = []+      | token == "--file" || "--file=" `T.isPrefixOf` token =+          ConfigConflict file row True : scan (if token == "--file" then drop 1 remaining else remaining)+      | Just (option, value) <- shortValue token =+          [ConfigConflict file row True | option == 'f'] ++ scan (if T.null value then drop 1 remaining else remaining)+      | "@" `T.isPrefixOf` token = ConfigConflict file row False : scan remaining+      | token `elem` takesValue = scan (drop 1 remaining)+      | otherwise = scan remaining+    takesValue =+      [ "--alias", "--pivot", "--begin", "--end", "--period", "--depth",+        "--output-file", "--output-format", "--rules", "--rules-file",+        "--today", "--value", "--conf"+      ]+    -- hledger accepts bundles such as -If. The first short option that takes+    -- a value consumes the rest of the token, or the following token.+    shortValue token+      | "--" `T.isPrefixOf` token = Nothing+      | otherwise = T.stripPrefix "-" token >>= T.uncons . T.dropWhile (`notElem` ("fbepoOXcmw" :: String))++firstConfigConflict :: ConfigUse -> [ConfigConflict] -> Maybe ConfigConflict+firstConfigConflict use = find (\(ConfigConflict _ row _) -> configSection row `elem` relevant)+  where+    relevant = "general" : case use of+      ImportConfig -> ["print"]+      ReportConfig -> ["balancesheet", "incomestatement", "balance", "print", "accounts"]+      BalanceConfig -> ["balance"]++readConfigConflicts :: HledgerInfo -> HledgerConf -> IO [ConfigConflict]+readConfigConflicts info conf = do+  let file = hledgerConfPath conf+      indented = case TR.decimal (hledgerVersionNumber (hlVersion info)) of+        Right (major, rest)+          | Just suffix <- T.stripPrefix "." rest, Right (minor, _) <- TR.decimal suffix ->+              (major, minor) >= ((1, 99) :: (Int, Int))+        _ -> True+      splitWords = if indented then configWords else legacyConfigWords+  bytes <- B.readFile file+  case TE.decodeUtf8' bytes of+    Left err -> ioError (userError ("Cannot read hledger config " ++ file ++ ": " ++ show err))+    Right contents -> return (configConflicts file+      (parseConfigLinesWith indented splitWords (T.lines (T.dropWhile (== '\xFEFF') contents))))++findConfigConflict :: HledgerInfo -> ConfigUse -> Maybe HledgerConf -> IO (Maybe ConfigConflict)+findConfigConflict info use = fmap (firstConfigConflict use) . findConfigConflicts info++findConfigConflicts :: HledgerInfo -> Maybe HledgerConf -> IO [ConfigConflict]+findConfigConflicts info = maybe (pure []) (readConfigConflicts info)++renderConfigConflict :: ConfigConflict -> T.Text+renderConfigConflict (ConfigConflict file row addsJournal) =+  T.pack file <> " line " <> T.pack (show (configLineNumber row)) <> " [" <> configSection row <> "] "+    <> (if addsJournal then "adds a journal: " else "reads arguments from another file: ")+    <> configSource row <> "\n"+    <> (if addsJournal then "hledger-flow picks its own journals, so this would mix another journal into the ledger."+        else "hledger-flow cannot check which journals that argument file would add.")+    <> "\nRemove that line, or pass --hledger-conf with a config that doesn't have it."++requireSafeConfig :: ConfigUse -> RT.RuntimeOptions -> IO ()+requireSafeConfig use opts = case RT.hledgerConf opts of+  Nothing -> return ()+  conf -> do+    info <- RT.hledgerInfo opts+    conflict <- findConfigConflict info use conf+    maybe (return ()) (ioError . userError . T.unpack . renderConfigConflict) conflict++-- Config arguments can quote spaces or a #; neither quotes nor comments are+-- arguments. Leave malformed quoting for hledger to diagnose.+configWords :: T.Text -> [T.Text]+configWords = map T.pack . wordsOutside Nothing Nothing . T.unpack+  where+    wordsOutside _ word [] = map reverse (maybeToList word)+    wordsOutside quote word (c:cs)+      | Just c == quote = wordsOutside Nothing word cs+      | quote == Nothing, c `elem` ['\'', '"'] = wordsOutside (Just c) (Just (maybe "" id word)) cs+      | quote == Nothing, c == '#' = map reverse (maybeToList word)+      | quote == Nothing, isSpace c =+          map reverse (maybeToList word) ++ wordsOutside Nothing Nothing cs+      | otherwise = wordsOutside quote (Just (c : maybe "" id word)) cs++-- Older hledger config files start comments at any # and only interpret a+-- quote at the start of a word. Quotes within an unquoted word stay literal.+legacyConfigWords :: T.Text -> [T.Text]+legacyConfigWords = go . T.stripStart . T.takeWhile (/= '#')+  where+    go input = case T.uncons input of+      Nothing -> []+      Just (quote, contents) | quote == '\'' || quote == '"' ->+        let (word, end) = T.break (== quote) contents+         in word : case T.uncons (T.drop 1 end) of+              Just (next, _) | isSpace next -> go (T.stripStart (T.drop 1 end))+              _ -> []+      _ -> let (word, rest) = T.break isSpace input in word : go (T.stripStart rest)
src/Hledger/Flow/Import/CSVImport.hs view
@@ -94,7 +94,7 @@       cached = if or [refresh | (_, _, refresh) <- outcomes] then Nothing else Just inventory   ((refreshed, orphans, journalsOnDisk), journalFindTime) <- timeAction $ reconciliationInventory importRoot cached   forM_ (filter selected orphans) $ \file -> do-    target <- moveToTrash file+    target <- moveToTrash (baseDir opts) file     channelOutLn ch $ "Moved orphaned output " <> relative file <> " to " <> relative target   withLogBlock opts True ch "Include files" $ \_ -> do     (_, writeIncludeTime1) <- timeAction $ writeIncludesFromInventory opts ch importRoot refreshed journalsOnDisk
src/Hledger/Flow/Import/Dependencies.hs view
@@ -4,8 +4,9 @@  module Hledger.Flow.Import.Dependencies   ( Release (..), FallbackReason (..), ReuseDecision (..), Converter (..),-    ruleIncludes, nestedRuleIncludes, resolveRuleInclude, discoverRules, selectRules, isSeparatedValues,-    checkedIO, atPath, checkArguments, discoverConfig, discoverDirectives, checkConfig, checkDirectives,+    IncludeFile (..), IncludeNode (..), readIncludeTree, scanIncludes, resolveInclude,+    ruleIncludes, nestedRuleIncludes, discoverRules, selectRules, isSeparatedValues,+    checkedIO, atPath, checkArguments, discoverConfig, discoverDirectives, checkConfig, journalIncludes,     newConverterProbe, converterProbeWith   ) where @@ -14,7 +15,7 @@ import qualified Data.ByteString as B import Data.Char (isAlpha, isAlphaNum, isAscii, isControl, isDigit, isSpace, toLower, GeneralCategory(CurrencySymbol), generalCategory) import Data.List (isPrefixOf)-import Data.Maybe (mapMaybe)+import Data.Maybe (fromMaybe, mapMaybe, maybeToList) import qualified Data.Set as Set import qualified Data.Text as T import qualified Data.Text.Encoding as TE@@ -22,13 +23,14 @@ import Data.Time (getCurrentTimeZone, timeZoneMinutes) import GHC.IO.Encoding (getLocaleEncoding, getFileSystemEncoding, getForeignEncoding) import Hledger.Flow.Common (firstConfirmedFile, lookupHledgerConf, probeHledgerVersion, hledgerVersionNumber)+import Hledger.Flow.HledgerConfig (ConfigLine (..), parseConfigLines) import Hledger.Flow.Import.Fingerprint import Hledger.Flow.Import.ImportHelpers (rulesFileCandidates) import Hledger.Flow.Import.Types (ImportDirs) import Hledger.Flow.PathHelpers (AbsDir, AbsFile) import qualified Hledger.Flow.RuntimeOptions as RT import Hledger.Flow.Types (HledgerConf(..), hledgerConfPath)-import Path (Abs, parseAbsFile, toFilePath)+import Path (Abs, parent, parseAbsFile, parseRelDir, parseRelFile, toFilePath, (</>)) import qualified System.Directory as D import qualified System.FilePath as F import System.Exit (ExitCode(..))@@ -47,6 +49,8 @@ data FallbackReason   = FallbackReason {reasonScope :: T.Text, reasonDetail :: T.Text}   | FileReason {reasonScope :: T.Text, reasonFile :: FilePath, reasonDetail :: T.Text}+  -- The notice quotes the entry, so people see exactly what to change.+  | ConfigEntry {reasonScope :: T.Text, reasonFile :: FilePath, reasonDetail :: T.Text}   | OrdinaryInputError {reasonScope :: T.Text, reasonDetail :: T.Text}   | CyclicRules {reasonScope :: T.Text, reasonDetail :: T.Text}   deriving (Eq, Ord, Show)@@ -108,60 +112,130 @@ dropUtf8Bom :: B.ByteString -> B.ByteString dropUtf8Bom b = maybe b id (B.stripPrefix (B.pack [0xef,0xbb,0xbf]) b) --- Walk leading .. components relative to the including file's spelled path.--- Verify the result against the filesystem's resolution, which can differ when a--- symlink precedes .. . An internal symlink is mirrored under its original name.-resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)-resolveRuleInclude base parent child = checkedIO "uncertain rules include" raw $+data IncludeFile = CsvRules Release | JournalDirectives+  deriving (Eq, Show)++scanIncludes :: IncludeFile -> Bool -> AbsFile -> B.ByteString -> ReuseDecision [FilePath]+scanIncludes (CsvRules release) isRoot = if isRoot then ruleIncludes release else nestedRuleIncludes release+scanIncludes JournalDirectives _ = journalIncludes++fileKind :: IncludeFile -> T.Text+fileKind (CsvRules _) = "rules"+fileKind JournalDirectives = "journal"++resolveInclude :: IncludeFile -> AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)+resolveInclude kind base including child = checkedIO ("uncertain " <> label) raw $   if F.hasDrive child || not (F.isRelative child) || any F.isPathSeparator (take 1 child)-    then pure (atPath "absolute rules include" child (T.pack (toFilePath parent)))-    else do+    then pure (atPath ("absolute " <> label) child (T.pack (toFilePath including)))+    else case kind of+      CsvRules _ -> resolveRules+      JournalDirectives -> resolveJournal+  where+    -- Keep the filename as opened: a file symlink's nested includes are+    -- relative to its containing directory. A path without ".." is kept as+    -- spelled. Otherwise the OS resolves the parent, so a directory link+    -- before ".." leads where hledger's open would.+    resolveRules = case parseAbsFile raw of+      Just file -> pure (Reusable file)+      Nothing -> do+        exists <- D.doesFileExist raw+        if exists+          then do+            dir <- D.canonicalizePath (F.takeDirectory raw)+            pure (maybe outside Reusable (parseAbsFile (dir F.</> F.takeFileName raw)))+          else pure (atPath "could not read rules file" raw (T.pack (toFilePath including)))+    resolveJournal = do       let (ups, rest) = span (== "..") (filter (/= ".") (F.splitDirectories child))-          lexical = foldr (const F.takeDirectory) (F.takeDirectory (toFilePath parent)) ups F.</> F.joinPath rest+          lexical = foldr (const F.takeDirectory) directory ups F.</> F.joinPath rest       if ".." `elem` rest || null rest || not (inside (toFilePath base) lexical)         then pure outside         else do           actual <- D.canonicalizePath raw           mirror <- D.canonicalizePath lexical           root <- D.canonicalizePath (toFilePath base)-          if F.equalFilePath actual mirror && inside root actual+          -- hledger resolves journal includes from the canonical including file.+          -- The snapshot preserves its spelled directory and journal reader.+          source <- D.canonicalizePath (toFilePath including)+          dir <- D.canonicalizePath directory+          let compatible = F.equalFilePath dir (F.takeDirectory source) &&+                F.equalFilePath actual (root F.</> F.makeRelative (toFilePath base) lexical)+          if compatible && F.equalFilePath actual mirror && inside root actual             then pure (maybe outside Reusable (parseAbsFile lexical))             else pure outside-  where-    raw = F.takeDirectory (toFilePath parent) F.</> child-    outside = atPath "rules include escapes the ledger or cannot be mirrored" raw (T.pack (toFilePath parent))+    directory = F.takeDirectory (toFilePath including)+    raw = directory F.</> child+    label = fileKind kind <> " include"+    outside = atPath (label <> " escapes the ledger or cannot be mirrored") raw (T.pack (toFilePath including))     inside root path = F.splitDirectories (F.dropTrailingPathSeparator root) `isPrefixOf` F.splitDirectories path  discoverRules :: AbsDir -> Release -> AbsFile -> IO (ReuseDecision [(AbsFile,Digest)])-discoverRules base release = walk Set.empty+discoverRules base release = discoverTree base (CsvRules release)++discoverTree :: AbsDir -> IncludeFile -> AbsFile -> IO (ReuseDecision [(AbsFile,Digest)])+discoverTree base kind root = fmap (map dependency) <$> readIncludeTree base kind root+  where dependency node = (includeSource node, digestBytes (includeBytes node))++-- A rules file can be read through a directory link followed by "..". Its+-- source and its position in a link-free copy can then be different.+data IncludeNode = IncludeNode+  { includeSource :: AbsFile+  , includePosition :: AbsFile+  , includeDirectories :: [AbsDir]+  , includeBytes :: B.ByteString+  }++-- Walk the spelling that hledger will traverse in the private copy. Retain+-- directories visited before "..", even when no included file lives there.+rulesPosition :: AbsFile -> FilePath -> Maybe (AbsFile, [AbsDir])+rulesPosition including spelling = do+  name <- parseRelFile (F.takeFileName spelling)+  (dir, visited) <- foldM step (parent including, []) (F.splitDirectories (F.takeDirectory spelling))+  pure (dir </> name, visited)   where-    walk ancestors path = checkedIO "could not read rules file" (toFilePath path) $ do+    step acc "." = Just acc+    step (dir, visited) ".."+      | parent dir /= dir = Just (parent dir, dir : parent dir : visited)+      | otherwise = Nothing+    step (dir, visited) component = do+      child <- parseRelDir component+      pure (dir </> child, visited)++readIncludeTree :: AbsDir -> IncludeFile -> AbsFile -> IO (ReuseDecision [IncludeNode])+readIncludeTree base kind root = walk Set.empty root [] root+  where+    nativeRules = case kind of+      CsvRules release -> release /= V1996+      JournalDirectives -> False+    walk ancestors position dirs path = checkedIO ("could not read " <> fileKind kind <> " file") (toFilePath path) $ do       canonical <- D.canonicalizePath (toFilePath path)       directory <- D.canonicalizePath (F.takeDirectory (toFilePath path))       -- The same rules bytes can resolve includes in different directories.       -- In 1.52.4 the first read also decodes line endings differently.-      let node = (canonical, directory, release /= V1996 && Set.null ancestors)-      if Set.member node ancestors && release /= V1996+      let node = (canonical, directory, nativeRules && Set.null ancestors)+      if Set.member node ancestors && nativeRules         then pure $ Fallback (CyclicRules "cyclic rules include" (T.pack (toFilePath path)))         else if Set.member node ancestors || Set.size ancestors >= 256-          then pure (atPath "cyclic or excessively deep rules include" (toFilePath path) "")+          then pure (atPath ("cyclic or excessively deep " <> fileKind kind <> " include") (toFilePath path) "")           else do             bytes <- B.readFile (toFilePath path)-            let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes-            case scan release path bytes of+            case scanIncludes kind (Set.null ancestors) path bytes of               Fallback r -> pure (Fallback r)               Reusable children -> do-                nested <- descend (Set.insert node ancestors) path children-                pure (((path,digestBytes bytes):) <$> nested)-    descend _ _ [] = pure (pure [])-    descend ancestors path (c:cs) = do-      selected <- resolveRuleInclude base path c+                nested <- descend (Set.insert node ancestors) path position children+                pure ((IncludeNode path position dirs bytes :) <$> nested)+    descend _ _ _ [] = pure (pure [])+    descend ancestors path position (c:cs) = do+      selected <- resolveInclude kind base path c       tree <- case selected of         Fallback r -> pure (Fallback r)-        Reusable file -> walk ancestors file+        Reusable file -> case kind of+          JournalDirectives -> walk ancestors file [] file+          CsvRules _ -> case rulesPosition position c of+            Just (child, dirs) -> walk ancestors child dirs file+            Nothing -> pure (atPath "rules include cannot be mirrored" c (T.pack (toFilePath path)))       case tree of         Fallback r -> pure (Fallback r)-        Reusable files -> fmap (files ++) <$> descend ancestors path cs+        Reusable files -> fmap (files ++) <$> descend ancestors path position cs  selectRules :: AbsFile -> ImportDirs Abs -> IO (ReuseDecision (Maybe AbsFile)) selectRules csv dirs = checkedIO "could not select rules file" (toFilePath csv) $ do@@ -196,12 +270,12 @@             Nothing -> pure (atPath "uncertain config path" path "")             Just absPath -> fmap Just <$> readDependency (checkConfig release) absPath -discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile,Digest)))+discoverDirectives :: AbsFile -> IO (ReuseDecision [(AbsFile,Digest)]) discoverDirectives file = checkedIO "could not discover directives" (toFilePath file) $ do   exists <- firstConfirmedFile [toFilePath file]   case exists of-    Nothing -> pure (pure Nothing)-    Just _ -> fmap Just <$> readDependency (checkDirectives V1996) file+    Nothing -> pure (pure [])+    Just _ -> discoverTree (parent file) JournalDirectives file  readDependency :: (AbsFile -> B.ByteString -> ReuseDecision ()) -> AbsFile -> IO (ReuseDecision (AbsFile,Digest)) readDependency check path = checkedIO "could not read dependency" (toFilePath path) $ do@@ -210,41 +284,101 @@  checkConfig :: Release -> AbsFile -> B.ByteString -> ReuseDecision () checkConfig release path bytes = do-  ls <- physicalLines path bytes-  () <$ foldM line "general" (zip [1 :: Int ..] ls)+  ls <- physicalLines path (dropUtf8Bom bytes)+  let settings =+        [ (configLineNumber row, word)+          | row <- parseConfigLines (release == V1996) ls,+            configSection row `elem` ["general", "print"],+            word <- configArguments row+        ]+  case configDependency settings of+    Nothing -> pure ()+    Just n -> Fallback $ ConfigEntry "non-deterministic hledger config entry" (toFilePath path)+      ("line " <> T.pack (show n) <> ": " <> T.strip (ls !! (n - 1)))++-- A denylist, not a config validator: hledger validates unfamiliar syntax and+-- options on conversion. Add only known dependencies outside the cache inputs.+configDependency :: [(Int,T.Text)] -> Maybe Int+configDependency settings = go settings   where-    -- Only canonical built-ins other than print have an inert section. Addons,-    -- aliases, abbreviations and names introduced in another release can have-    -- different scope and must be passed through to the ordinary converter.-    inactive = T.words $ "accounts activity add aregister balance balancesheet balancesheetequity cashflow check close codes commodities descriptions diff files help import incomestatement notes payees prices register repl rewrite roi run setup stats tags " <>-      case release of-        V1524 -> "commands demo test"-        V1996 -> "get holdings transactions"-        OtherSupported -> ""-    line section (n,raw)-      | T.null body || "#" `T.isPrefixOf` body = pure section-      | release /= V1996, "[" `T.isPrefixOf` body, T.stripStart raw /= raw = unsupported n body-      | Just start <- T.stripPrefix "[" body = case T.breakOn "]" start of-          (s,end) | not (T.null end), not ("#" `T.isInfixOf` s), let name = T.strip s,-                    name `elem` ("general":"print":inactive) -> option n name (T.drop 1 end)-          _ -> unsupported n body-      | otherwise = option n section body-      where body = T.strip raw-    option n section arg-      | section `elem` inactive = pure section-      | T.null setting = pure section-      | section == "print" && release /= V1524 && setting == "--layout=hledger1" = pure section-      | otherwise = unsupported n setting-      where setting = T.strip (T.takeWhile (/= '#') arg)-    unsupported n value = atPath "unsupported hledger config" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)+    -- A fixed --today pins relative dates and valuation, but hledger ignores+    -- it for forecasting.+    pinned = or (zipWith fixedToday (map snd settings) (drop 1 (map snd settings) ++ [""]))+    fixedToday arg next = case T.stripPrefix "--today" arg of+      Just "" -> fixedDate next+      Just v | Just date <- T.stripPrefix "=" v -> fixedDate date+      _ -> False+    fixedDate date = not (T.null date || relativeDate date)+    go [] = Nothing+    go ((n,arg):rest)+      -- Untracked files, saved import state, extra output and temporary paths.+      | "@" `T.isPrefixOf` arg || flag ["--file", "--rules", "--rules-file", "--conf"] ["-f"] = found+      | flag ["--new", "--output-file", "--location", "--locations"] ["-o"] = found+      -- The rest depend on today's date.+      | flag ["--forecast"] [], T.null suffix || relativeDate value || openForecast value = found+      | pinned = go rest+      -- Without a report end date, hledger values amounts at today's prices.+      | flag ["--exchange"] ["-X", "-V"] = found+      | flag ["--value"] [], T.toLower (T.takeWhile (/= ',') value) `elem` ["end", "now"] = found+      | flag ["--begin", "--end", "--period", "--today"] ["-b", "-e", "-p"], relativeDate value = found+      | any relativeDate (queryDates arg) = found+      | otherwise = go rest+      where+        found = Just n+        (name,suffix) = T.breakOn "=" arg+        short = [s | s <- ["-f", "-o", "-X", "-b", "-e", "-p"], s `T.isPrefixOf` arg]+        flag longs shorts = name `elem` longs || any (`T.isPrefixOf` arg) shorts+        value | not (T.null suffix) = T.drop 1 suffix+              | s:_ <- short, T.length arg > T.length s = T.drop (T.length s) arg+              | (_,v):_ <- rest = v+              | otherwise = ""+    openForecast period = ".." `T.isSuffixOf` period+      || any (`T.isPrefixOf` period) ["from", "since"] && not ("to" `T.isInfixOf` period || ".." `T.isInfixOf` period)+    -- Dates in date: and date2: queries, including those inside expr:.+    queryDates token = case T.stripPrefix "expr:" (stripNot token) of+      Just expr -> mapMaybe queryDate (T.words (T.map (\c -> if c `elem` ("()" :: String) then ' ' else c) expr))+      Nothing -> maybeToList (queryDate token)+    queryDate token = case T.breakOn ":" (stripNot token) of+      (prefix,date) | prefix `elem` ["date", "date2"] -> Just (T.drop 1 date)+      _ -> Nothing+    stripNot token = maybe token stripNot (T.stripPrefix "not:" token) -checkDirectives :: Release -> AbsFile -> B.ByteString -> ReuseDecision ()-checkDirectives _ path bytes = do+-- Recognize relative words and dates with an implicit year. Unfamiliar date+-- syntax stays eligible, like unfamiliar options; this list can grow over time.+relativeDate :: T.Text -> Bool+relativeDate value = any relative parts+  where+    parts = T.words lowered ++ T.words (T.replace ".." " " (T.replace "to" " " lowered))+    lowered = T.toLower value+    relative part = any (`T.isPrefixOf` part) ["today", "tomorrow", "yesterday", "this", "last", "next"]+      || part `elem` ["ago", "ahead", "day", "days", "week", "weeks", "month", "months", "quarter", "quarters", "year", "years",+                     "jan", "january", "feb", "february", "mar", "march", "apr", "april", "may",+                     "jun", "june", "jul", "july", "aug", "august", "sep", "september",+                     "oct", "october", "nov", "november", "dec", "december", "q1", "q2", "q3", "q4"]+      || let digits = T.takeWhile isDigit (T.dropWhile (`elem` ("+-" :: String)) part) in not (T.null digits) && T.length digits < 4++journalIncludes :: AbsFile -> B.ByteString -> ReuseDecision [FilePath]+journalIncludes path bytes = do   ls <- physicalLines path (dropUtf8Bom bytes)-  case [(n,T.strip l) | (n,l) <- zip [1 :: Int ..] ls, not (allowed l)] of-    [] -> pure ()-    (n,value):_ -> atPath "unsupported journal directive" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)+  sequence (mapMaybe classify (zip [1 :: Int ..] ls))   where+    classify (n,raw)+      | allowed raw = Nothing+      | ("include",rest) <- T.break horizontalSpace raw,+        let target = T.dropAround horizontalSpace (T.takeWhile (/= ';') rest)+            journalPath = T.stripPrefix "journal:" target+            name = fromMaybe target journalPath,+        not (T.null name),+        T.all (\c -> isAlphaNum c || c `elem` (" ./_-" :: String)) name =+          Just $ if journalPath == Nothing && map toLower (F.takeExtension (T.unpack name)) `elem` nonJournalExtensions+            then atPath "unsupported journal include reader" (toFilePath path) (lineDetail n raw)+            else pure (T.unpack name)+      | otherwise = Just (atPath "unsupported journal directive" (toFilePath path)+                          (lineDetail n raw))+    -- hledger defaults to the journal reader for unrecognized extensions.+    nonJournalExtensions = [".csv", ".tsv", ".ssv", ".rules", ".timeclock", ".timedot"]+    lineDetail n raw = "line " <> T.pack (show n) <> ": " <> T.strip raw+    horizontalSpace c = c == ' ' || c == '\t'     allowed raw       | T.null stripped || T.isPrefixOf ";" stripped || T.isPrefixOf "#" stripped || T.isPrefixOf "*" stripped = True       | Just (c,_) <- T.uncons raw, isSpace c = False
src/Hledger/Flow/Import/IncludeFiles.hs view
@@ -22,7 +22,7 @@ import qualified Data.Text as T import qualified Data.Text.IO as T import Hledger.Flow.BaseDir (relativeToBase')-import Hledger.Flow.Common (allYearsFileName, filterPaths, groupValuesBy, textFileNeedsUpdate, writeFiles')+import Hledger.Flow.Common (allYearsFileName, filterPaths, groupValuesBy, includePreamble, textFileNeedsUpdate, writeFiles') import Hledger.Flow.Import.ImportHelpers (groupIncludesUpTo) import Hledger.Flow.Import.Inventory (manualIncludeFiles, managedIncludeFiles) import Hledger.Flow.Import.Types@@ -74,9 +74,6 @@   let lns = map (toIncludeLine $ FilePath.takeDirectory outputFile) $ preFiles ++ files ++ postFiles   T.intercalate "\n" $ lns ++ [""] -includePreamble :: T.Text-includePreamble = "### Generated by hledger-flow - DO NOT EDIT ###\n"- writeIncludesFromInventory :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> AbsDir -> [AbsFile] -> [AbsFile] -> IO [AbsFile] writeIncludesFromInventory opts ch stopAt inventory journals = do   (files, roots) <- expectedIncludes (baseDir opts) (logVerbose opts ch) stopAt inventory journals@@ -84,10 +81,10 @@   return roots  -- | Check the same contents import would write, including obsolete generated indexes.-includesNeedUpdate :: BaseDir -> AbsDir -> [AbsFile] -> [AbsFile] -> IO Bool-includesNeedUpdate base root inventory journals = do+includesNeedUpdate :: BaseDir -> AbsDir -> [AbsFile] -> [AbsFile] -> Bool -> IO Bool+includesNeedUpdate base root inventory journals compareRoot = do   (includes, _) <- expectedIncludes base (const (return ())) root inventory journals-  top <- toplevelAllYearsInclude base+  top <- if compareRoot then toplevelAllYearsInclude base else return Map.empty   anyChanged (Map.toList (Map.union top includes))   where     anyChanged [] = return False
src/Hledger/Flow/Import/Incremental.hs view
@@ -10,11 +10,11 @@ import Control.Monad (unless, when) import Data.IORef import qualified Data.Map.Strict as Map-import Data.Maybe (maybeToList)+import Data.Maybe (listToMaybe, maybeToList) import qualified Data.Set as Set import qualified Data.Text as T import Hledger.Flow.BaseDir (relativeToBase)-import Hledger.Flow.Common (directivesFile, writeStagedText)+import Hledger.Flow.Common (directivesFile, homeDirs, tildePath, writeStagedText) import Hledger.Flow.DocHelpers import Hledger.Flow.Cache (initializeCache) import Hledger.Flow.Import.Cache@@ -22,6 +22,7 @@ import Hledger.Flow.Import.Dependencies import Hledger.Flow.Import.Fingerprint import Hledger.Flow.Import.Inventory (generatedOutput)+import Hledger.Flow.Import.ShellSources import Hledger.Flow.Import.Snapshot import Hledger.Flow.Import.Types import Hledger.Flow.Logging@@ -36,16 +37,17 @@  -- Keep one path only when every occurrence of a reason names that file. data Notices = Notices !(Set.Set AbsFile) !(Map.Map T.Text (Maybe T.Text))-data IncrementalRun = IncrementalRun SnapshotRun (IO (ReuseDecision Converter)) (IORef Notices)+data IncrementalRun = IncrementalRun SnapshotRun SourceCache (IO (ReuseDecision Converter)) (IORef Notices)  withIncrementalRun :: RuntimeOptions -> TChan LogMessage -> Int -> (IncrementalRun -> IO a) -> IO a withIncrementalRun opts ch count action = withSnapshotRun (baseDir opts) $ \snapshots -> do+  scripts <- newSourceCache   reasons <- newIORef (Notices Set.empty Map.empty)   probe <- newConverterProbe opts   when (count > 0) $ do     initialized <- tryIOError $ initializeCache (baseDir opts)     either (channelErrLn ch . ("Could not initialize incremental cache: " <>) . T.pack . show) pure initialized-  finally (action (IncrementalRun snapshots probe reasons)) $ do+  finally (action (IncrementalRun snapshots scripts probe reasons)) $ do     Notices affected warnings <- readIORef reasons     unless (Map.null warnings) $ do       let scope | Set.size affected == count = "all selected statements"@@ -54,21 +56,27 @@       channelErrLn ch $ "Incremental reuse unavailable for " <> scope <> ": "         <> T.intercalate ", " (map describe (Map.toAscList warnings)) <> ". See " <> docURL IncrementalImportFallback -data HookInputs = HookInputs AbsFile FileObservation AbsFile FileObservation+data HookInputs = HookInputs AbsFile FileObservation AbsFile FileObservation [SourceInput]   deriving (Eq, Show) -observeHook :: AbsFile -> AbsFile -> IO (ReuseDecision HookInputs)-observeHook script source = checkedIO "could not check script inputs" (P.toFilePath script) $ do+observeHook :: SourceCache -> AbsFile -> AbsFile -> IO (ReuseDecision HookInputs)+observeHook scripts script source = checkedIO "could not check script inputs" (P.toFilePath script) $ do   runnable <- if Sys.os == "mingw32" then pure True else D.executable <$> D.getPermissions (P.toFilePath script)   if runnable-    then Reusable <$> (HookInputs script <$> observeFile script <*> pure source <*> observeFile source)+    then do+      observed <- observeFile script+      input <- observeFile source+      deps <- checkedIO "unresolved shell source" (P.toFilePath script) $+        Reusable <$> hookSources scripts script observed+      pure (HookInputs script observed source input <$> deps)     else pure $ Fallback (OrdinaryInputError "script is not executable" (pathText script))  hookInputsKey :: ImportDirs P.Abs -> FilePath -> T.Text -> HookInputs -> StageInputs-hookInputsKey dirs cwd stage (HookInputs script s input i) =+hookInputsKey dirs cwd stage (HookInputs script s input i deps) =   [("stage",stage),("working directory",T.pack cwd),("hook",pathText script),    ("hook digest",digestHex (observationDigest s)),("input",pathText input),    ("input digest",digestHex (observationDigest i))] ++ map ("argument",) (hookDirArgs dirs)+    ++ concatMap sourceKey deps  pathText :: AbsFile -> T.Text pathText = T.pack . P.toFilePath@@ -77,12 +85,12 @@   { engine :: Converter   , copiedInputs :: SnapshotInput   , recordedInputs :: StageInputs-  , copiedDigests :: Map.Map AbsFile Digest+  , copiedDigests :: [(AbsFile, Digest)]   , liveConstruct :: Maybe HookInputs   } deriving (Eq, Show)  importStatement :: IncrementalRun -> RuntimeOptions -> TChan LogMessage -> ImportDirs P.Abs -> AbsFile -> IO (AbsFile,Bool,Bool)-importStatement (IncrementalRun snapshots probe reasons) opts ch dirs source = do+importStatement (IncrementalRun snapshots scripts probe reasons) opts ch dirs source = do   cwd <- D.getCurrentDirectory   record <- recordPath (baseDir opts) source   old <- readRecord record@@ -90,11 +98,16 @@   let hook name = findHook opts ch name (accountDir dirs P.</> (if name == "preprocess" then [P.relfile|preprocess|] else [P.relfile|construct|]))       report OrdinaryInputError{} = pure ()       report problem = do+        homes <- homeDirs         let reason = reasonScope problem+            located file = T.pack (tildePath homes (relativeToBase opts file))             path = case problem of-              FileReason _ file _ -> Just (T.pack (relativeToBase opts file))+              FileReason _ file _ -> Just (located file)+              ConfigEntry _ file entry -> Just (located file <> ", " <> entry)               _ -> Nothing-            detail = reasonDetail problem+            detail = case problem of+              ConfigEntry{} -> ""+              _ -> reasonDetail problem             common a b = if a == b then a else Nothing         atomicModifyIORef' reasons $ \(Notices files summaries) ->           (Notices (Set.insert source files) (Map.insertWith common reason path summaries), ())@@ -112,7 +125,7 @@       input <- savedInput opts ch source intermediate       pure (input,Nothing,False)     Just script -> do-      before <- observeHook script source+      before <- observeHook scripts script source       let key = hookInputsKey dirs cwd "preprocess" <$> before           prior = old >>= preprocessRecord       hit <- case key of@@ -127,7 +140,7 @@         previous <- digestIfPresent intermediate         digest <- installOutput True intermediate (preprocess opts ch script dirs source intermediate)         when (Just digest == previous) $ logVerbose opts ch "Preprocessing ran; the CSV is unchanged."-        after <- observeHook script source+        after <- observeHook scripts script source         selected <- hook "preprocess"         usable <- case key of           Fallback reason -> when skipScripts (report reason) >> pure Nothing@@ -149,10 +162,10 @@                       ++ map ("converter",) (converterKey converter)                     configArgs = pinnedConfig ("<config>" <$ conf)                     mk inputs files args extra construct =-                      JournalInputs converter inputs (common ++ map ("hledger argument",) args ++ extra ++ [(pathText p,digestHex d) | (p,d) <- files]) (Map.fromList files) construct+                      JournalInputs converter inputs (common ++ map ("hledger argument",) args ++ extra ++ [(pathText p,digestHex d) | (p,d) <- files]) files construct                 case candidate of                   Just script -> do-                    observation <- observeHook script csv+                    observation <- observeHook scripts script csv                     pure $ (\h -> mk (ConstructInput (fst <$> conf)) (maybeToList conf)                                         (constructArguments configArgs "<journal>") (hookInputsKey dirs cwd "construct" h) (Just h)) <$> observation                   Nothing | not (isSeparatedValues csv) -> pure $ atPath "unsupported input reader" (P.toFilePath csv) ""@@ -167,9 +180,10 @@                         digest <- digestFile csv                         pure $ do                           rulesTree <- tree-                          ds <- directives-                          let files = (csv,digest) : rulesTree ++ maybeToList ds ++ maybeToList conf-                          pure $ mk (RulesInput csv rf (fst <$> ds) (fst <$> conf)) files+                          journals <- directives+                          let ds = fst <$> listToMaybe journals+                              files = (csv,digest) : rulesTree ++ journals ++ maybeToList conf+                          pure $ mk (RulesInput csv rf ds (fst <$> conf)) files                                    (rulesArguments configArgs "<input>" "<rules>" ("<directives>" <$ ds)) [("rules",pathText rf)] Nothing       ordinary = do         script <- hook "construct"@@ -186,8 +200,8 @@   let agrees inputs = case preRecord of         Nothing -> True         Just entry -> Just (stageOutput entry) == case liveConstruct inputs of-          Nothing -> Map.lookup csv (copiedDigests inputs)-          Just (HookInputs _ _ _ observed) -> Just (observationDigest observed)+          Nothing -> lookup csv (copiedDigests inputs)+          Just (HookInputs _ _ _ observed _) -> Just (observationDigest observed)       decision = rawDecision >>= \inputs -> if agrees inputs then pure inputs else Fallback changed   (journal,generated) <- case decision of     Fallback r -> fallback r@@ -209,7 +223,7 @@               contents <- convertRules opts ch executable (RulesPaths src rf ds) (RulesPaths src' rf' ds') (pin conf) translate               target <- D.canonicalizePath (P.toFilePath output) >>= P.parseAbsFile               installOutput False target (writeStagedText target contents)-            (ConstructInput _,ConstructInput conf,Just (HookInputs s _ _ _)) ->+            (ConstructInput _,ConstructInput conf,Just (HookInputs s _ _ _ _)) ->               installOutput False output (constructWith opts ch executable (pin conf) translate s dirs csv output)             _ -> ioError (userError "Inconsistent incremental conversion inputs")           again <- discover@@ -217,6 +231,8 @@           effective <- case preNow of             Nothing -> P.doesFileExist intermediate >>= \exists -> pure (if exists then intermediate else source)             Just _ -> pure intermediate+          -- Order-sensitive: copyInputs must copy in the order discovery+          -- lists inputs, or no rebuild ever counts as stable.           let stable = again == Reusable inputs && snapshotManifest snapshot == copiedDigests inputs && preNow == pre && effective == csv           if stable then pure $ Just (StageRecord (recordedInputs inputs) outDigest)                     else when (skipScripts || liveConstruct inputs == Nothing) (report changed) >> pure Nothing
src/Hledger/Flow/Import/Inventory.hs view
@@ -25,6 +25,7 @@ import qualified Data.List as List import qualified Data.Map.Strict as Map import qualified Data.Set as Set+import Hledger.Flow.Cache (initializeIgnoredDir, trashDir) import Hledger.Flow.Common (groupValuesBy, homeDirs, tildePath) import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir) import Path@@ -203,20 +204,19 @@         && (name == "all-years.journal" || (isYear (take 4 name) && drop 4 name == "-include.journal"))       _ -> False -moveToTrash :: AbsFile -> IO AbsFile-moveToTrash file = do-  let year = dirname (parent file)-      stage = dirname (parent (parent file))-      account = parent (parent (parent file))-      trash = account </> [reldir|.trash|]+-- | Move a file under the ledger's trash, keeping its path from the ledger root.+moveToTrash :: AbsDir -> AbsFile -> IO AbsFile+moveToTrash base file = do+  rel <- stripProperPrefix base (parent file)+  let trash = trashDir base       lock = trash </> [reldir|.move-lock|]-      destination = trash </> year </> stage+      destination = trash </> rel       acquire = Directory.createDirectory (toFilePath lock) `catchIOError` \err ->         if isAlreadyExistsError err           then homeDirs >>= \homes -> ioError (userError ("Trash is in use: " ++ tildePath homes (toFilePath lock)             ++ ". If no import is running, remove this empty lock directory and retry."))           else ioError err-  PathIO.createDirIfMissing True trash+  initializeIgnoredDir trash   -- Serialize destination selection and rename so two imports cannot replace   -- each other's recovery files. An interrupted move leaves a recoverable lock.   bracket_ acquire (PathIO.removeDir lock) $ do
+ src/Hledger/Flow/Import/ShellSources.hs view
@@ -0,0 +1,196 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}++module Hledger.Flow.Import.ShellSources+  ( SourceCache, newSourceCache, SourceInput, sourceKey, hookSources ) where++import Control.Monad (foldM, unless, when)+import Data.Char (isAlpha, isAlphaNum, isAscii, isControl, isSpace)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Char8 as B8+import Data.Foldable (toList)+import Data.Functor.Identity (runIdentity)+import Data.IORef+import Data.List (isPrefixOf)+import Data.Maybe (isJust)+import qualified Data.Map.Strict as Map+import qualified Data.Set as Set+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.PathHelpers (AbsFile)+import qualified Path as P+import ShellCheck.AST+import ShellCheck.ASTLib (getLiteralString)+import ShellCheck.Interface+import ShellCheck.Parser (parseScript)+import qualified System.Directory as D+import System.Environment (lookupEnv)+import qualified System.FilePath as F+import System.IO (IOMode(ReadMode), withBinaryFile)+#ifndef mingw32_HOST_OS+import qualified System.Posix.Files as Posix+#endif++-- Cache only syntax, never path resolution or filesystem observations.+type SourceCache = IORef (Map.Map Digest [Step])+newSourceCache :: IO SourceCache+newSourceCache = newIORef Map.empty++data SourceInput = SourceInput FilePath FileObservation [(String, Digest)] deriving (Eq, Show)++sourceKey :: SourceInput -> [(T.Text, T.Text)]+sourceKey (SourceInput target obs variables) =+  (T.pack target, digestHex (observationDigest obs)) :+  [("source variable " <> T.pack name, digestHex digest) | (name,digest) <- variables]++data Piece = Literal String | Variable Bool String+data Step = Include [Piece] | ShellStateChanged | Uncertain++hookSources :: SourceCache -> AbsFile -> FileObservation -> IO [SourceInput]+hookSources cache hook observed = do+  let file = P.toFilePath hook+  header <- withBinaryFile file ReadMode (fmap (B8.takeWhile (/= '\n')) . flip BS.hGet 256)+  case interpreter file header of+    Other -> pure []+    UnrecognisedShell -> uncertain file+    ShellScript -> do+      startup <- lookupEnv "BASH_ENV"+      when (maybe False (not . null) startup) (uncertain file)+      files <- newIORef []+      visits <- newIORef (0 :: Int)+      let visit parents path observation = do+            canonical <- D.canonicalizePath path+            when (Set.member canonical parents) (uncertain path)+            n <- atomicModifyIORef' visits (\i -> (i+1,i))+            when (n >= 128) (uncertain path)+            plan <- scriptSteps cache path (observationDigest observation)+            foldM (step (Set.insert canonical parents) path) True plan+          step _ _ _ ShellStateChanged = pure False+          step _ path _ Uncertain = uncertain path+          step parents path pristine (Include parts) = do+            -- Earlier commands can change working directory, variable bindings+            -- or the meaning of the source builtin.+            unless pristine (uncertain path)+            expanded <- mapM (expand path) parts+            let name = concatMap fst expanded+                variables = concatMap snd expanded+            unless ('/' `elem` name && not ("-" `isPrefixOf` name) &&+                    not (any isControl name)) (uncertain path)+            -- Test the original spelling before canonicalizing missing/../+            -- and symlink/../ components.+#ifdef mingw32_HOST_OS+            regular <- D.doesFileExist name+#else+            regular <- Posix.isRegularFile <$> Posix.getFileStatus name+#endif+            unless regular (uncertain path)+            target <- D.canonicalizePath name+            dep <- P.parseAbsFile target >>= observeFile+            modifyIORef' files (SourceInput target dep variables :)+            visit parents target dep+      _ <- visit Set.empty file observed+      reverse <$> readIORef files+  where+    uncertain f = ioError (userError ("cannot resolve shell source safely in " ++ f))+    expand _ (Literal s) = pure (s, [])+    expand f (Variable quoted name) = do+      value <- lookupEnv name+      case value of+        Just s | not (null s), quoted || not (any (\c -> isSpace c || c `elem` ("*?[" :: String)) s) ->+          pure (s, [(name, digestBytes (TE.encodeUtf8 (T.pack s)))])+        _ -> uncertain f++data Interpreter = ShellScript | Other | UnrecognisedShell++interpreter :: FilePath -> BS.ByteString -> Interpreter+#ifdef mingw32_HOST_OS+interpreter file _ | F.takeExtension file == ".ps1" = Other+#endif+interpreter _ header = case B8.stripPrefix "#!" header of+  Nothing -> if BS.all (\c -> c == 9 || c == 13 || c >= 32) header then UnrecognisedShell else Other+  Just line -> case words (B8.unpack line) of+    exe:args | F.takeFileName exe == "env" -> case args of+                 [name] | shell name -> ShellScript+                 _ | any shell args -> UnrecognisedShell+                 _ -> Other+             | shell exe -> if null args then ShellScript else UnrecognisedShell+    _ -> Other+  where shell p = F.takeFileName p `elem` ["bash","sh","dash"]++scriptSteps :: SourceCache -> FilePath -> Digest -> IO [Step]+scriptSteps cache path digest = do+  existing <- Map.lookup digest <$> readIORef cache+  case existing of+    Just plan -> pure plan+    Nothing -> do+      bytes <- BS.readFile path+      unless (digestBytes bytes == digest) (ioError (userError ("script changed while reading " ++ path)))+      let plan = case TE.decodeUtf8' bytes of+            Left _ -> [Uncertain]+            Right body ->+              let result = runIdentity $ parseScript newSystemInterface newParseSpec+                    { psScript = T.unpack body, psFilename = path, psShellTypeOverride = Just Bash, psIgnoreRC = True }+              in case prRoot result of+                Just root | all ((/= ErrorC) . cSeverity . pcComment) (prComments result) -> steps root+                _ -> [Uncertain]+      atomicModifyIORef' cache (\m -> (Map.insert digest plan m, ()))+      pure plan++-- A function or loop can execute later or repeatedly with different bindings.+steps :: Token -> [Step]+steps token = case token of+  T_Annotation _ _ t -> steps t+  T_Script _ _ ts -> concatMap steps ts+  T_Pipeline _ [] [t] -> steps t+  T_Redirecting _ [] t -> steps t+  T_AndIf _ a b | not (containsSource b) -> steps a ++ steps b+  T_OrIf _ a b | not (containsSource b) -> steps a ++ steps b+  T_SimpleCommand _ [] (cmd:args)+    | isSource cmd -> case args of+        path:rest | all sourceArgument rest, Just parts <- pieces False path -> [Include parts]+        _ -> [Uncertain]+    | Just name <- getLiteralString cmd, name `elem` ["exit","true","false",":"],+      all (isJust . getLiteralString) args -> []+    | getLiteralString cmd == Just "set",+      all (\a -> getLiteralString a `elem` map Just ["-e","-u","-eu","-euo","-o","pipefail"]) args -> []+  T_Function _ _ _ name body | name `notElem` ["source",".","set","exit","true","false",":","[","test"], not (containsSource body) -> []+  T_Condition _ _ (TC_Unary _ _ op arg) | op `elem` ["-z","-n"], isJust (pieces True arg) || positionalArgument arg -> []+  _ | containsSource token -> [Uncertain]+    | otherwise -> [ShellStateChanged]+  where+    -- Forwarded hook arguments are already captured by the hook key.+    sourceArgument t = positionalArgument t || maybe False (all literal) (pieces False t)+    literal (Literal _) = True+    literal _ = False+    positionalArgument (T_NormalWord _ [T_DoubleQuoted _ [T_DollarBraced _ _ name]]) =+      case getLiteralString name of+        Just "@" -> True+        Just n -> not (null n) && all (\c -> c >= '0' && c <= '9') n+        Nothing -> False+    positionalArgument _ = False++isSource :: Token -> Bool+isSource t = getLiteralString t `elem` [Just ".", Just "source"]++containsSource :: Token -> Bool+containsSource t@(OuterToken _ children) = case t of+  T_SimpleCommand _ _ (cmd:args)+    | isSource cmd || getLiteralString cmd == Just "eval" -> True+    | getLiteralString cmd `elem` [Just "builtin", Just "command"], any isSource args -> True+  _ -> any containsSource (toList children)++-- Parameter operators, command substitution, globs and tilde expansion need Bash.+pieces :: Bool -> Token -> Maybe [Piece]+pieces quoted token = case token of+  T_NormalWord _ ts -> concat <$> mapM (pieces quoted) ts+  T_DoubleQuoted _ ts -> concat <$> mapM (pieces True) ts+  T_SingleQuoted _ s -> Just [Literal s]+  T_Literal _ s | quoted || not (any (\c -> c `elem` ("~*?[]{}" :: String)) s) -> Just [Literal s]+  T_DollarBraced _ _ t | Just name@(c:_) <- getLiteralString t,+    isAscii c && (isAlpha c || c == '_'), all (\x -> isAscii x && (isAlphaNum x || x == '_')) name,+    not ("BASH" `isPrefixOf` name || name `elem` ["_", "PWD", "OLDPWD", "IFS", "LINENO", "RANDOM", "SRANDOM",+          "SECONDS", "SHLVL", "UID", "EUID", "PPID", "OPTIND", "OPTARG", "SHELLOPTS", "FUNCNAME",+          "DIRSTACK", "GROUPS", "PIPESTATUS", "EPOCHSECONDS", "EPOCHREALTIME"]) ->+        Just [Variable quoted name]+  _ -> Nothing
src/Hledger/Flow/Import/Snapshot.hs view
@@ -45,7 +45,7 @@  data PreparedSnapshot = PreparedSnapshot   { snapshotInputs :: SnapshotInput,-    snapshotManifest :: Map.Map AbsFile Digest,+    snapshotManifest :: [(AbsFile, Digest)],     originalPaths :: Map.Map AbsFile AbsFile   } @@ -196,9 +196,8 @@  copyInputs :: AbsDir -> AbsDir -> Release -> SnapshotInput -> IO PreparedSnapshot copyInputs base tree version input = do-  manifest <- newIORef Map.empty+  manifest <- newIORef []   paths <- newIORef Map.empty-  completed <- newIORef Set.empty   let mirror file = case stripProperPrefix base file of         Nothing -> reject "snapshot input outside ledger" (T.pack (toFilePath file))         Just rel -> pure (tree </> [reldir|l|] </> rel)@@ -212,15 +211,16 @@         -- equivalence rules. Never overwrite a copy made for another spelling.         previous <- Map.lookup target <$> readIORef paths         collision <- case previous of-          Just original -> pure (toFilePath original /= toFilePath source)+          Just (original, _) -> pure (toFilePath original /= toFilePath source)           Nothing -> D.doesPathExist (toFilePath target)         when collision $ reject "snapshot filenames collide" (T.pack (toFilePath source))       record source target digest = do-        previous <- Map.lookup source <$> readIORef manifest-        when (maybe False (/= digest) previous) $+        previous <- Map.lookup target <$> readIORef paths+        when (maybe False ((/= digest) . snd) previous) $           reject "dependency changed during snapshot" (T.pack (toFilePath source))-        modifyIORef' manifest (Map.insert source digest)-        modifyIORef' paths (Map.insert target source)+        -- Include order matters if a directory link changes during copying.+        modifyIORef' manifest ((source, digest) :)+        modifyIORef' paths (Map.insert target (source, digest))       copyBytes source target = do         destination source target         bytes <- BS.readFile (toFilePath source)@@ -232,30 +232,38 @@         () <- fromDecision (check version source bytes)         pure target       copyConfig source = validated checkConfig source (tree </> [reldir|c|] </> filename source)-      copyRule ancestors source = do-        target <- mirror source-        identity <- D.canonicalizePath (toFilePath source)-        when (Set.member identity ancestors || Set.size ancestors >= 256) $ reject "cyclic or excessively deep rules include" (T.pack (toFilePath source))-        done <- Set.member (toFilePath source) <$> readIORef completed-        unless done $ do-          bytes <- copyBytes source target-          let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes-          includes <- fromDecision (scan version source bytes)-          forM_ includes $ \spelling -> do-            child <- resolveRuleInclude base source spelling >>= fromDecision-            void $ copyRule (Set.insert identity ancestors) child-          modifyIORef' completed (Set.insert (toFilePath source))-        pure target+      copyTree kind root = do+        nodes <- readIncludeTree base kind root >>= fromDecision+        let detours = Set.fromList (concatMap includeDirectories nodes)+            common = foldl' containing base (map (parent . includePosition) nodes ++ Set.toList detours)+            -- Keep ordinary layout short; use a separate tree when a rules+            -- include needs a sibling of the ledger or a different ancestor.+            folder = tree </> if common == base then [reldir|l|] else [reldir|r|]+            staged path = (folder </>) <$> stripProperPrefix common path+        forM_ detours $ \dir -> do+          target <- if dir == common then pure folder else staged dir+          P.createDirIfMissing True target+        forM_ nodes $ \node -> do+          let source = includeSource node+              bytes = includeBytes node+          target <- staged (includePosition node)+          destination source target+          BS.writeFile (toFilePath target) bytes+          record source target (digestBytes bytes)+        staged root+      containing dir path+        | dir == path || isProperPrefixOf dir path || dir == parent dir = dir+        | otherwise = containing (parent dir) path   copied <- case input of     ConstructInput conf -> ConstructInput <$> traverse copyConfig conf     RulesInput csv rules directives conf -> do       dataCopy <- mirror csv       destination csv dataCopy       copyDigest csv dataCopy >>= record csv dataCopy-      rulesCopy <- copyRule Set.empty rules-      directivesCopy <- traverse (\source -> mirror source >>= validated checkDirectives source) directives+      rulesCopy <- copyTree (CsvRules version) rules+      directivesCopy <- traverse (copyTree JournalDirectives) directives       RulesInput dataCopy rulesCopy directivesCopy <$> traverse copyConfig conf-  PreparedSnapshot copied <$> readIORef manifest <*> readIORef paths+  PreparedSnapshot copied <$> (reverse <$> readIORef manifest) <*> (Map.map fst <$> readIORef paths)  -- hledger emits locations as "FILE:line:column:" (also "FILE:line:" and -- "FILE: explanation"). Restrict replacement to its location fields.
src/Hledger/Flow/Init.hs view
@@ -9,22 +9,25 @@   ) where -import Control.Monad (when)+import Control.Monad (forM, when) import qualified Data.Text as T import qualified Data.Text.IO as T import Hledger.Flow.BaseDir (findBaseDir) import Data.Maybe (fromMaybe)-import Hledger.Flow.Common (Shell (..), homeDirs, homeShellPath, nativeShell, shellArgument, tildePath)-import Hledger.Flow.PathHelpers (AbsDir)+import Hledger.Flow.Common (Shell (..), findHledgerConf, homeDirs, homeShellPath, nativeShell, shellArgument, tildePath)+import Hledger.Flow.DateTime (currentYear)+import Hledger.Flow.Init.Example (exampleAccount, exampleAccountPath, exampleBaseFiles, exampleImportFiles, starterOwner)+import Hledger.Flow.PathHelpers (AbsDir, RelDir, RelFile, pathOccupied, writeNewUtf8File) import Hledger.Flow.Status.Render (printNext) import Hledger.Flow.Types (BaseDir) import Path-import Path.IO (createDirIfMissing, doesDirExist)+import Path.IO (createDirIfMissing, doesDirExist, doesFileExist, renameDir, withTempDir)+import System.IO.Error (tryIOError) import qualified System.FilePath as FilePath  data InitOutcome   = -- | The new ledger, and an unrelated ledger above it, if any.-    Created BaseDir (Maybe BaseDir)+    Created BaseDir (Maybe BaseDir) RelDir [(RelFile, Bool)]   | -- | The folder already has @import/@.     AlreadyLedger BaseDir   | -- | The folder is inside this ledger's @import/@ tree.@@ -42,12 +45,34 @@       case above of         Just b | let imp = b </> [reldir|import|], imp == dir || imp `isProperPrefixOf` dir -> return (InsideLedger b)         _ -> do-          createDirIfMissing True (dir </> [reldir|import|])-          return (Created dir above)+          year <- currentYear+          owner <- starterOwner+          createDirIfMissing True dir+          baseFiles <- forM (exampleBaseFiles year) $ \(name, contents) -> do+            let target = dir </> name+            occupied <- pathOccupied target+            if occupied+              then do+                isFile <- doesFileExist target+                if isFile+                  then return (name, False)+                  else ioError (userError ("Cannot create starter file at " ++ toFilePath target))+              else do+                result <- tryIOError (writeNewUtf8File target contents)+                case result of+                  Right () -> return (name, True)+                  Left err -> do+                    exists <- doesFileExist target+                    if exists then return (name, False) else ioError err+          withTempDir dir ".hledger-flow-init-" $ \temp -> do+            let staged = temp </> [reldir|import|]+            mapM_ (\(name, contents) -> writeNewUtf8File (staged </> name) contents) (exampleImportFiles owner year)+            renameDir staged (dir </> [reldir|import|])+          return (Created dir above (exampleAccount owner) baseFiles)  -- | Run @hledger-flow init@ for a folder, printing what happened and what's next.-runInit :: Maybe FilePath -> AbsDir -> AbsDir -> IO ()-runInit hledgerPath cwd dir = do+runInit :: Maybe FilePath -> Maybe FilePath -> AbsDir -> AbsDir -> IO ()+runInit hledgerPath confOption cwd dir = do   homes <- homeDirs   let absolute = FilePath.dropTrailingPathSeparator . toFilePath       shown = T.pack . tildePath homes . absolute@@ -59,8 +84,10 @@     InsideLedger b ->       ioError . userError . T.unpack $         "That folder is inside the ledger at " <> shown b <> ". Start a new ledger outside its import/ folder."-    Created base parentLedger -> do+    Created base parentLedger account baseFiles -> do       T.putStrLn ("Started a ledger in " <> shown base)+      T.putStrLn ("Created an example account (" <> exampleAccountPath account <> ") and starter journals for manual entries and balances.")+      mapM_ (\(name, created) -> T.putStrLn ((if created then "Created " else "Kept existing ") <> T.pack (toFilePath name))) baseFiles       mapM_ (\p -> T.putStrLn ("Note: " <> shown p <> " also has an import/ folder. Commands run inside " <> shown base <> " use the new ledger.")) parentLedger       when (base /= cwd) $ do         let changeDir = case nativeShell of@@ -68,4 +95,5 @@               PowerShell -> "Set-Location -LiteralPath "             path = absolute base         T.putStrLn ("Change to it first: " <> changeDir <> fromMaybe (shellArgument nativeShell (T.pack path)) (homeShellPath nativeShell homes path))-      printNext hledgerPath base+      conf <- findHledgerConf confOption+      printNext hledgerPath conf base
+ src/Hledger/Flow/Init/Example.hs view
@@ -0,0 +1,316 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Hledger.Flow.Init.Example+  ( exampleImportFiles,+    isSampleStatement,+    exampleBaseFiles,+    starterOwner,+    exampleAccount,+    exampleAccountPath,+    sampleAccountDir,+    untouchedExample,+    directivesAreStarter,+    journalIsCommentsOnly,+  )+where++import qualified Data.ByteString as B+import Data.List (intercalate)+import Data.Maybe (fromMaybe, mapMaybe)+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import qualified Data.Text.IO as TIO+import Hledger.Flow.Common (Shell (..), nativeShell)+import Hledger.Flow.DocHelpers (DocLink (Balances, ManualJournals, RulesFiles), docURL)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile, pathOccupied)+import Hledger.Flow.Types (BaseDir)+import Path+import qualified Path.IO as PathIO+import System.Environment (lookupEnv)+import qualified System.FilePath as F+import qualified System.FilePath.Windows as Windows+import System.IO.Error (tryIOError)+import Text.Read (readMaybe)++starterOwner :: IO RelDir+starterOwner = do+  let variable = case nativeShell of+        Posix -> "USER"+        PowerShell -> "USERNAME"+  value <- either (const Nothing) id <$> tryIOError (lookupEnv variable)+  return (fromMaybe [reldir|me|] (value >>= validOwner))+  where+    validOwner name+      | Windows.isValid name,+        Windows.splitDirectories name == [name],+        not (any Windows.isPathSeparator name),+        T.strip text == text,+        not (T.isPrefixOf "." text || T.isPrefixOf "~" text),+        not (T.isSuffixOf "." text),+        ';' `notElem` name,+        not (T.isSuffixOf ".journal" folded),+        folded /= "_manual_",+        B.length (T.encodeUtf8 text) <= 255 = parseRelDir name+      | otherwise = Nothing+      where+        text = T.pack name+        folded = T.toCaseFold text++exampleAccount :: RelDir -> RelDir+exampleAccount owner = owner </> [reldir|mybank/checking|]++exampleAccountPath :: RelDir -> T.Text+exampleAccountPath account = "import/" <> T.pack (intercalate "/" parts)+  where+    parts = F.splitDirectories (F.dropTrailingPathSeparator (toFilePath account))++sampleName :: RelFile+sampleName = [relfile|hledger-flow-example.csv|]++isSampleStatement :: AbsFile -> Bool+isSampleStatement file = filename file == sampleName++rulesName :: RelFile+rulesName = [relfile|mybank-checking.rules|]++sampleCsv :: Integer -> T.Text+sampleCsv year = T.unlines $+  zipWith (<>) (sampleCsvRows year) [",Comment", ",\"" <> starterNotice <> "\"", ","]++sampleCsvRows :: Integer -> [T.Text]+sampleCsvRows year =+  ["Date,Description,Amount", date <> ",Example purchase,-5.00", date <> ",Example refund,5.00"]+  where+    date = T.pack (show year) <> "-01-02"++sampleRules :: T.Text+sampleRules = T.unlines+  [ "# Rules for the example statement.",+    "# Docs: " <> docURL RulesFiles,+    "skip 1",+    "fields date, description, amount",+    "currency USD",+    "account1 assets:mybank:checking",+    "account2 expenses:unknown",+    "",+    "if Example",+    " account2 expenses:example"+  ]++exampleImportFiles :: RelDir -> Integer -> [(RelFile, T.Text)]+exampleImportFiles starter year =+  (exampleAccount starter </> rulesName, starterComment "# " sampleRules)+    : paths ((samplePath, sampleCsv year) : manualFiles ++ balanceFiles)+  where+    account = F.dropTrailingPathSeparator (toFilePath (exampleAccount starter))+    bank = F.takeDirectory account+    owner = F.takeDirectory bank+    yearName = show year+    samplePath = account F.</> "1-in" F.</> yearName F.</> toFilePath sampleName+    date day = T.pack yearName <> "-" <> day+    entry day description postings = (date day <> " " <> description) : map ("    " <>) postings+    manual level phase help example =+      (level F.</> "_manual_" F.</> yearName F.</> (phase ++ "-import.journal"),+       journalExample ManualJournals help example)+    balance suffix help example =+      (account F.</> (yearName ++ "-" ++ suffix ++ ".journal"), journalExample Balances help example)+    manualFiles =+      [ manual "" "pre" ["Record an entry shared across owners before their imported activity."]+          (entry "01-01" "Fund the household cash box from my cash"+            ["assets:household:cash  150.00 USD", "assets:me:cash"]),+        manual "" "post" ["Record a transfer between owners after reviewing both sets of statements."]+          (entry "01-02" "Reimburse partner for shared groceries"+            ["assets:partner:cash  35.00 USD", "assets:me:cash"]),+        manual owner "pre" ["Fill a gap in a transfer between this owner's banks before their first statements."]+          (entry "01-01" "Move savings to a new bank"+            ["assets:otherbank:savings  75.00 USD", "assets:mybank:checking"]),+        manual owner "post" ["Record a transfer between this owner's banks after reviewing their statements."]+          (entry "01-02" "Move emergency savings back to checking"+            ["assets:mybank:checking  50.00 USD", "assets:otherbank:savings"]),+        manual bank "pre" ["Record a transfer between accounts at this bank before their imported activity."]+          (entry "01-01" "Move money from checking to savings"+            ["assets:mybank:savings  25.00 USD", "assets:mybank:checking"]),+        manual bank "post" ["Fill in a transfer missing from both exports after reviewing this bank's statements."]+          (entry "01-02" "Sweep savings into checking"+            ["assets:mybank:checking  12.00 USD", "assets:mybank:savings"]),+        manual account "pre"+          ["Fill a gap before this account's first statement.", "Put opening balances in the dedicated yearly opening file."]+          (entry "01-01" "Replace debit card"+            ["expenses:bank-fees  5.00 USD", "assets:mybank:checking"]),+        manual account "post"+          ["Assert this account's balance after its imported entries for the same day.",+           "Use the balance shown by your bank. Put closing balances in the dedicated yearly closing file."]+          (entry "01-02" "Check balance after the sample statement"+            ["assets:mybank:checking  0 USD = 0 USD"])+      ]+    balanceFiles =+      [ balance "opening" ["Record what this account already held just before its first imported transaction."]+          (entry "01-01" "Checking opening balance"+            ["assets:mybank:checking  100.00 USD", "equity:opening/closing balances"]),+        balance "closing"+          ["Close this account's balance to zero when you deliberately close your books.",+           "This example assumes the starter opening balance and sample statement."]+          (entry "12-31" "Checking closing balance"+            ["assets:mybank:checking  -100.00 USD = 0 USD", "equity:opening/closing balances"])+      ]++starterNotice :: T.Text+starterNotice = "Generated by hledger-flow init as a starter. This file is yours to edit. hledger-flow leaves it as you save it."++starterComment :: T.Text -> T.Text -> T.Text+starterComment prefix contents = prefix <> starterNotice <> "\n" <> contents++optionalIncludeNotice :: T.Text+optionalIncludeNotice = "; This file is optional. If absent, hledger-flow omits it from the include files it maintains."++optionalJournal :: T.Text -> T.Text+optionalJournal contents = starterComment "; " (optionalIncludeNotice <> "\n" <> contents)++journalExample :: DocLink -> [T.Text] -> [T.Text] -> T.Text+journalExample link help example = optionalJournal $ T.unlines $+  map ("; " <>) help ++ ["; Docs: " <> docURL link, "", "; Example. Remove its leading semicolons and adapt it."]+    ++ map ("; " <>) example++exampleBaseFiles :: Integer -> [(RelFile, T.Text)]+exampleBaseFiles year = paths $+  [("directives.journal", starterDirectives)]+    ++ [ (priceDir F.</> name, starterComment "; " (T.unlines+           [ "; This file is optional. Remove its include line from prices.journal to omit the pair.",+             "; Add market prices here.",+             "; P " <> date <> " " <> quote+           ]))+         | (name, quote) <- pairs+       ]+    ++ [(priceDir F.</> "prices.journal", optionalJournal (T.unlines ["include " <> T.pack name | (name, _) <- pairs]))]+  where+    priceDir = "prices" F.</> show year+    date = T.pack (show year) <> "-01-01"+    pairs = [("EUR-USD.prices", "EUR 1.10 USD"), ("BTC-USD.prices", "BTC 99000.00 USD")]++starterDirectives :: T.Text+starterDirectives = T.unlines+  [ "; Ledger-wide directives go here. https://hledger.org/dev/hledger.html#directives",+    "; " <> starterNotice,+    optionalIncludeNotice,+    "",+    "; Common commodities. https://hledger.org/dev/hledger.html#commodity-aliases",+    "commodity 1,000.00 USD  ; alias: US$",+    "commodity 1,000.00 EUR  ; alias: €",+    "commodity 1,000.00 GBP  ; alias: £",+    "commodity 1,000.00 CAD  ; alias: CA$",+    "commodity 1,000.00 AUD  ; alias: A$",+    "commodity 1,000. JPY    ; alias: JP¥",+    "commodity 1,000.00 CNY  ; alias: CN¥, alias: RMB",+    "commodity 1,000.00 INR  ; alias: ₹",+    "",+    "; Bitcoin and satoshis. https://hledger.org/dev/hledger.html#commodity-units",+    "commodity 1.00000000 BTC  ; alias: 100000000 SAT, alias: ₿",+    "commodity 1. SAT",+    "",+    "; Account types. https://hledger.org/dev/hledger.html#account-types",+    "account assets                  ; type: A",+    "account liabilities             ; type: L",+    "account equity                  ; type: E",+    "account revenues                ; type: R",+    "account expenses                ; type: X",+    "",+    "account assets:bank             ; type: C",+    "account assets:cash             ; type: C",+    "",+    "account equity:conversion       ; type: V",+    "account equity:unrealised-gain  ; type: U",+    "account revenues:gain           ; type: G"+  ]++paths :: [(FilePath, a)] -> [(RelFile, a)]+paths = mapMaybe (\(p, a) -> fmap (\path -> (path, a)) (parseRelFile p))++untouchedExample :: BaseDir -> [AbsFile] -> IO (Maybe RelDir)+untouchedExample base [source]+  | Just rel <- stripProperPrefix (importRoot base) source,+    [ownerName, "mybank", "checking", "1-in", yearName, name] <- F.splitDirectories (toFilePath rel),+    name == toFilePath sampleName,+    Just starter <- parseRelDir ownerName,+    Just year <- readMaybe yearName,+    Just generated <- parseRelFile ("3-journal" F.</> yearName F.</> "hledger-flow-example.journal"),+    Just yearIndex <- parseRelFile (yearName ++ "-include.journal") = do+      let account = sampleAccountDir base starter+      result <- tryIOError $ do+        let rules = account </> rulesName+            journal = account </> generated+            bank = parent account+            owner = parent bank+            indexes dir = [dir </> yearIndex, dir </> [relfile|all-years.journal|]]+            otherStarters =+              [ (file, contents)+                | (relative, contents) <- exampleImportFiles starter year,+                  let file = importRoot base </> relative,+                  isProperPrefixOf owner file, file /= source, file /= rules+              ]+            allowed = [source, rules, journal] ++ map fst otherStarters ++ concatMap indexes [owner, bank, account]+            dirs = concatMap (takeWhile (isProperPrefixOf owner) . iterate parent . parent) allowed+            optionalStarter (file, contents) = do+              exists <- PathIO.doesFileExist file+              if exists then matches file contents else pure True+        cleanPath <- allM plainDir [importRoot base, owner]+        cleanContents <- if cleanPath then containsOnly dirs allowed owner else return False+        if cleanContents then do+          current <- (&&) <$> matches source (sampleCsv year)+                          <*> matches rules (starterComment "# " sampleRules)+          sampleMatches <- if current then pure True+            else (&&) <$> matches source (T.unlines (sampleCsvRows year)) <*> matches rules sampleRules+          if sampleMatches then allM optionalStarter otherStarters else pure False+        else pure False+      return (case result of Right True -> Just (exampleAccount starter); _ -> Nothing)+  where+    matches file expected = (== T.encodeUtf8 expected) <$> B.readFile (toFilePath file)+untouchedExample _ _ = return Nothing++containsOnly :: [AbsDir] -> [AbsFile] -> AbsDir -> IO Bool+containsOnly allowedDirs allowedFiles dir = do+  (dirs, files) <- PathIO.listDir dir+  if all (`elem` allowedDirs) dirs && all (`elem` allowedFiles) files+    then do+      hasLinks <- (||) <$> (or <$> traverse PathIO.isSymlink dirs) <*> (or <$> traverse PathIO.isSymlink files)+      if hasLinks then return False else and <$> traverse (containsOnly allowedDirs allowedFiles) dirs+    else return False++importRoot :: BaseDir -> AbsDir+importRoot base = base </> [reldir|import|]++sampleAccountDir :: BaseDir -> RelDir -> AbsDir+sampleAccountDir base owner = importRoot base </> exampleAccount owner++-- Starter declarations and comments do not add transactions to the ledger.+directivesAreStarter :: BaseDir -> IO Bool+directivesAreStarter base = do+  let file = base </> [relfile|directives.journal|]+  exists <- pathOccupied file+  if exists+    then elem <$> readJournalBody file <*> pure [Just [], Just (journalBody starterDirectives)]+    else pure True++journalIsCommentsOnly :: AbsFile -> IO Bool+journalIsCommentsOnly file = (== Just []) <$> readJournalBody file++readJournalBody :: AbsFile -> IO (Maybe [T.Text])+readJournalBody file = do+  regular <- regularFile file+  if regular then Just . journalBody <$> TIO.readFile (toFilePath file) else pure Nothing++journalBody :: T.Text -> [T.Text]+journalBody = filter (\line -> not (T.null line || T.isPrefixOf ";" line)) . map T.strip . T.lines++plainDir :: AbsDir -> IO Bool+plainDir dir = do+  exists <- PathIO.doesDirExist dir+  if exists then not <$> PathIO.isSymlink dir else pure False++regularFile :: AbsFile -> IO Bool+regularFile file = do+  exists <- PathIO.doesFileExist file+  if exists then not <$> PathIO.isSymlink file else pure False++allM :: (a -> IO Bool) -> [a] -> IO Bool+allM check = fmap and . traverse check
+ src/Hledger/Flow/InputSnapshot.hs view
@@ -0,0 +1,138 @@+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}++module Hledger.Flow.InputSnapshot+  ( InputSnapshot,+    snapshotInputs,+    inputTargets,+    inputsMatch,+    inputsUnchanged,+  )+where++import Control.Exception (IOException, try)+import Control.Monad (filterM)+import Data.Aeson (FromJSON, ToJSON)+import Data.Char (isSpace)+import Data.List (nub)+import Data.Maybe (fromMaybe)+import qualified Data.Set as Set+import qualified Data.Text as T+import qualified Data.Text.IO as T+import Data.Time.Clock (UTCTime)+import GHC.Generics (Generic)+import Hledger.Flow.PathHelpers (AbsFile, isGlobMetacharacter)+import Path (toFilePath)+import qualified System.Directory as Directory+import qualified System.FilePath as F++data Entry = Entry FilePath FilePath UTCTime+  deriving (Eq, Generic, Show)++instance FromJSON Entry+instance ToJSON Entry++data InputSnapshot = InputSnapshot [Entry] [Entry]+  deriving (Eq, Generic, Show)++instance FromJSON InputSnapshot+instance ToJSON InputSnapshot++entry :: FilePath -> IO Entry+entry file = Entry file <$> Directory.canonicalizePath file <*> Directory.getModificationTime file++snapshotInputs :: [AbsFile] -> IO InputSnapshot+snapshotInputs inputs = do+  let files = nub (map toFilePath inputs)+  inputsWithTargets <- traverse entry files+  watched <- concat <$> traverse (\(Entry _ target _) -> globDirectories target) inputsWithTargets+  InputSnapshot inputsWithTargets <$> traverse entry (nub watched)++inputTargets :: InputSnapshot -> [FilePath]+inputTargets (InputSnapshot files _) = [target | Entry _ target _ <- files]++inputsMatch :: InputSnapshot -> IO Bool+inputsMatch (InputSnapshot files dirs) = do+  let entries = files ++ dirs+  result <- try (traverse (\(Entry file _ _) -> entry file) entries)+  return $ either (const False :: IOException -> Bool) (== entries) result++inputsUnchanged :: InputSnapshot -> UTCTime -> IO Bool+inputsUnchanged snapshot@(InputSnapshot files dirs) since =+  if all (\(Entry _ _ modified) -> modified < since) (files ++ dirs)+    then inputsMatch snapshot+    else return False++globDirectories :: FilePath -> IO [FilePath]+globDirectories file = do+  contents <- T.readFile file+  let includes =+        [ T.unpack (readerPath arg)+          | line <- T.lines (T.dropWhile (== '\xFEFF') contents),+            Just rest <- [T.stripPrefix "include" line],+            Just (first, _) <- [T.uncons rest],+            isSpace first,+            let arg = T.strip (T.takeWhile (/= ';') rest),+            not (T.null arg)+        ]+  concat <$> mapM watch includes+  where+    -- hledger-flow emits singleton character ranges for literal metacharacters.+    -- Track wildcard branches conservatively, including parents of missing branches.+    readerPath arg =+      let (prefix, path) = T.breakOn ":" arg+       in if prefix `elem` ["journal", "timeclock", "timedot", "csv", "tsv", "ssv"]+            then fromMaybe arg (T.stripPrefix ":" path)+            else arg+    watch ('~' : separator : path) | separator `elem` ("/\\" :: String) =+      Directory.getHomeDirectory >>= \home -> watchUnder home path+    watch path = watchUnder (F.takeDirectory file) path+    watchUnder parent path =+      let parts = concatMap expandRecursive (F.splitDirectories path)+          (fixed, patternParts) = span (not . snd . decodeLiteral) parts+          root = (if F.isAbsolute path then id else (parent F.</>))+            (F.joinPath (map (fst . decodeLiteral) fixed))+       in case patternParts of+            [] -> return []+            _ -> watchFrom Set.empty root patternParts++-- hledger also interprets ** with a filename suffix as a recursive glob.+expandRecursive :: String -> [String]+expandRecursive = go []+  where+    go before ('*' : '*' : suffix@(_ : _)) = (reverse before ++ "**") : go [] ('*' : suffix)+    go before (c : rest) = go (c : before) rest+    go before [] = [reverse before]++watchFrom :: Set.Set (FilePath, [FilePath]) -> FilePath -> [FilePath] -> IO [FilePath]+watchFrom _ _ [] = return []+watchFrom visited dir (part : rest) = do+  exists <- Directory.doesDirectoryExist dir+  if not exists+    then return []+    else do+      resolved <- Directory.canonicalizePath dir+      if Set.member (resolved, part : rest) visited+        then return [dir]+        else do+          let next = Set.insert (resolved, part : rest) visited+              (literal, glob) = decodeLiteral part+          descend <- case (part, glob, rest) of+            (_, _, []) | part /= "**" -> return []+            ("**", _, _) -> do+              children <- subdirs dir+              here <- watchFrom visited dir rest+              below <- concat <$> traverse (\child -> watchFrom next child (part : rest)) children+              return (here ++ below)+            (_, True, _) -> subdirs dir >>= fmap concat . traverse (\child -> watchFrom next child rest)+            _ -> watchFrom next (dir F.</> literal) rest+          return (dir : descend)+  where+    subdirs p = Directory.listDirectory p >>= filterM Directory.doesDirectoryExist . map (p F.</>)++decodeLiteral :: String -> (String, Bool)+decodeLiteral ('[' : x : '-' : y : ']' : rest) | x == y, isGlobMetacharacter x =+  let (decoded, glob) = decodeLiteral rest in (x : decoded, glob)+decodeLiteral (x : rest) =+  let (decoded, glob) = decodeLiteral rest in (x : decoded, isGlobMetacharacter x || glob)+decodeLiteral [] = ("", False)
+ src/Hledger/Flow/LedgerSelection.hs view
@@ -0,0 +1,184 @@+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}++module Hledger.Flow.LedgerSelection+  ( LedgerFile (..),+    Coverage (..),+    LedgerSelection (..),+    selectLedger,+    journalInputs,+    coverageFor,+    ledgerFileProblem,+  )+where++import Control.Exception (IOException, try)+import Control.Monad (when)+import Data.Aeson (FromJSON, ToJSON, decodeFileStrict', encodeFile)+import qualified Data.ByteString as B+import qualified Data.Text as T+import Data.Time.Clock (UTCTime)+import GHC.Generics (Generic)+import Hledger.Flow.Cache (cacheDir, initializeCache)+import Hledger.Flow.Common (configuredLedgerFile, hledgerFileArgument, runProcessCaptured, tildePath)+import Hledger.Flow.InputSnapshot+import Hledger.Flow.PathHelpers (AbsFile, withStagedOutput)+import Hledger.Flow.Types (BaseDir, HledgerInfo (..))+import Path+import qualified Path.IO as PathIO+import qualified System.Directory as Directory+import System.Exit (ExitCode (..))+import qualified System.FilePath as F++data Coverage = CoversLedger | UnrelatedLedger | UnreadableLedger T.Text+  deriving (Eq, Show)++data LedgerFile = LedgerFile+  { lfValue :: FilePath,+    lfExists :: Bool,+    lfCoverage :: Coverage+  }+  deriving (Eq, Show)++data LedgerSelection = LedgerSelection+  { selectedLedgerFile :: Maybe LedgerFile,+    aggregateJournal :: Maybe AbsFile,+    selectionError :: Maybe T.Text+  }+  deriving (Show)++data CachedInputs = CachedInputs FilePath FilePath T.Text UTCTime InputSnapshot+  deriving (Generic)++instance FromJSON CachedInputs+instance ToJSON CachedInputs++journalInputs :: HledgerInfo -> [AbsFile] -> IO (Either T.Text [AbsFile])+journalInputs info files = do+  let args = ["--no-conf", "files"] ++ concatMap (\file -> ["--file", hledgerFileArgument info (toFilePath file)]) files+  result <- try (runProcessCaptured (T.pack (toFilePath (hlPath info)), args, Nothing))+  case result of+    Left err -> return (Left (T.pack (show (err :: IOException))))+    Right (ExitFailure _, _, err) -> return (Left (T.strip err))+    Right (ExitSuccess, out, _) -> do+      let paths = filter (not . T.null) (map (T.dropWhileEnd (== '\r')) (T.lines out))+      case paths of+        [] -> return (Left "Could not read the paths printed by hledger files.")+        _ -> do+          resolved <- try (traverse (resolveLedgerPath . T.unpack) paths)+          return (either (Left . T.pack . show) Right (resolved :: Either IOException [AbsFile]))++coverageFor :: FilePath -> Maybe FilePath -> [FilePath] -> Coverage+coverageFor index directives files+  | index `elem` files && maybe True (`elem` files) directives = CoversLedger+  | otherwise = UnrelatedLedger++ledgerFileProblem :: [FilePath] -> LedgerFile -> T.Text+ledgerFileProblem homes file+  | not (lfExists file) = "LEDGER_FILE is set to " <> displayed <> ", which doesn't exist."+  | otherwise = case lfCoverage file of+      CoversLedger -> ""+      UnrelatedLedger -> "LEDGER_FILE (" <> displayed <> ") does not include this ledger."+      UnreadableLedger err -> "Could not read LEDGER_FILE (" <> displayed <> "): " <> err+  where+    displayed = T.pack (tildePath homes (lfValue file))++selectLedger :: [FilePath] -> Maybe HledgerInfo -> BaseDir -> IO LedgerSelection+selectLedger homes info base = do+  let root = base </> [relfile|all-years.journal|]+  rootExists <- Directory.doesFileExist (toFilePath root)+  configured <- configuredLedgerFile+  let missingRoot = "No all-years.journal in " <> T.pack (tildePath homes (toFilePath base)) <> " yet. Run `hledger-flow import` first."+      fallback value+        | rootExists = LedgerSelection value (Just root) Nothing+        | otherwise = LedgerSelection value Nothing (Just missingRoot)+  case configured of+    Nothing -> return (fallback Nothing)+    Just (value, path) -> do+      exists <- Directory.doesFileExist path+      if not exists+        then do+          let file = LedgerFile value False UnrelatedLedger+          return (LedgerSelection (Just file) Nothing (Just (ledgerFileProblem homes file <>+            " Point it at " <> T.pack (tildePath homes (toFilePath root)) <> ", or unset it.")))+        else do+          absolute <- resolveLedgerPath path+          canonical <- Directory.canonicalizePath (toFilePath absolute)+          rootPath <- Directory.canonicalizePath (toFilePath root)+          coverage <- if rootExists && canonical == rootPath+            then return CoversLedger+            else case info of+              Nothing -> return (UnreadableLedger "hledger is unavailable.")+              Just hl -> discoverCoverage hl base absolute+          let file = Just (LedgerFile value True coverage)+          return $ case coverage of+            CoversLedger -> LedgerSelection file (Just absolute) Nothing+            _ -> fallback file++-- Path excludes parent references. Resolve only the directory when needed+-- so hledger still reads a file symlink relative to its original location.+resolveLedgerPath :: FilePath -> IO AbsFile+resolveLedgerPath path = do+  absolute <- Directory.makeAbsolute path+  case parseAbsFile absolute of+    Just file -> return file+    Nothing -> do+      dir <- Directory.canonicalizePath (F.takeDirectory absolute)+      parseAbsFile (dir F.</> F.takeFileName absolute)++discoverCoverage :: HledgerInfo -> BaseDir -> AbsFile -> IO Coverage+discoverCoverage info base journal = do+  let cache = cacheDir base </> [relfile|ledger-coverage.json|]+      path = toFilePath journal+      executable = toFilePath (hlPath info)+      version = hlVersion info+  cached <- try (decodeFileStrict' (toFilePath cache))+  reusable <- case cached of+    Right (Just (CachedInputs oldPath oldExecutable oldVersion start snapshot))+      | (oldPath, oldExecutable, oldVersion) == (path, executable, version) -> do+          unchanged <- inputsUnchanged snapshot start+          return (if unchanged then Just (inputTargets snapshot) else Nothing)+    Left (_ :: IOException) -> return Nothing+    _ -> return Nothing+  inputs <- case reusable of+    Just paths -> return (Right paths)+    Nothing -> do+      scanned <- try $ do+        initializeCache base+        PathIO.withTempDir (parent cache) ".ledger-coverage-" $ \dir -> do+          let marker = dir </> [relfile|started|]+          B.writeFile (toFilePath marker) B.empty+          start <- PathIO.getModificationTime marker+          files <- journalInputs info [journal]+          return (Just start, files)+      (started, files) <- case scanned of+        Right result -> return result+        Left (_ :: IOException) -> do+          files <- journalInputs info [journal]+          return (Nothing, files)+      case files of+        Left err -> return (Left err)+        Right paths -> do+          snapshot <- try (snapshotInputs paths)+          case snapshot of+            Right saved -> do+              case started of+                Just start -> do+                  unchanged <- inputsUnchanged saved start+                  when unchanged $ do+                    _ <- try (withStagedOutput cache (\stage ->+                      encodeFile (toFilePath stage) (CachedInputs path executable version start saved))) :: IO (Either IOException ())+                    return ()+                Nothing -> return ()+              return (Right (inputTargets saved))+            Left (_ :: IOException) -> Right <$> traverse (Directory.canonicalizePath . toFilePath) paths+  case inputs of+    Left err -> return (UnreadableLedger err)+    Right paths -> do+      index <- Directory.canonicalizePath (toFilePath (base </> [relfile|import/all-years.journal|]))+      let directives = toFilePath (base </> [relfile|directives.journal|])+      hasDirectives <- Directory.doesFileExist directives+      required <- if hasDirectives then Just <$> Directory.canonicalizePath directives else return Nothing+      return (coverageFor index required paths)
src/Hledger/Flow/PathHelpers.hs view
@@ -1,15 +1,25 @@+{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-}  module Hledger.Flow.PathHelpers where  import Control.Monad.Catch (Exception, MonadThrow, throwM) import Control.Monad.IO.Class (MonadIO, liftIO)+import qualified Data.ByteString as B import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import Data.Time.Clock (UTCTime) import Hledger.Flow.DocHelpers (DocLink (..), docURL) import qualified Path import qualified Path.IO as Path-import System.Directory (doesDirectoryExist)+import System.Directory (doesDirectoryExist, doesPathExist, pathIsSymbolicLink)+import System.IO.Error (catchIOError, isDoesNotExistError) import qualified System.FilePath as FilePath+#ifdef mingw32_HOST_OS+import qualified System.Win32.File as Win32+#else+import qualified System.Posix.Files as Posix+#endif  type AbsFile = Path.Path Path.Abs Path.File @@ -20,14 +30,63 @@ type RelDir = Path.Path Path.Rel Path.Dir  -- | Publish a complete file by renaming it on the destination filesystem.-withStagedOutput :: AbsFile -> (AbsFile -> IO ()) -> IO ()+withStagedOutput :: AbsFile -> (AbsFile -> IO a) -> IO a withStagedOutput target writeOutput = do   let dir = Path.parent target   Path.createDirIfMissing True dir   Path.withTempDir dir ".hledger-flow-" $ \tmp -> do     let staged = tmp Path.</> Path.filename target-    writeOutput staged+    result <- writeOutput staged     Path.renameFile staged target+    return result++-- | Use this operation's own filesystem timestamp, even if another process+-- publishes a marker at the same destination.+writeTimestampMarker :: AbsFile -> IO UTCTime+writeTimestampMarker target = withStagedOutput target $ \stage -> do+  B.writeFile (Path.toFilePath stage) B.empty+  Path.getModificationTime stage++-- | Install complete UTF-8 bytes at an unused path. The final operation must+-- fail for an existing entry, including a dangling symlink.+writeNewUtf8File :: AbsFile -> T.Text -> IO ()+writeNewUtf8File target contents = do+  let dir = Path.parent target+  Path.createDirIfMissing True dir+  Path.withTempDir dir ".hledger-flow-" $ \tmp -> do+    let staged = tmp Path.</> Path.filename target+    B.writeFile (Path.toFilePath staged) (T.encodeUtf8 contents)+    publishNewFile staged target++-- | Publish a staged file exclusively, without replacing an existing target.+publishNewFile :: AbsFile -> AbsFile -> IO ()+publishNewFile staged target =+#ifdef mingw32_HOST_OS+  Win32.moveFile (extendedWindowsPath src) (extendedWindowsPath dest)+#else+  Posix.createLink src dest+#endif+  where+    src = Path.toFilePath staged+    dest = Path.toFilePath target++#ifdef mingw32_HOST_OS+-- The raw Win32 move needs the file namespace for paths beyond MAX_PATH.+-- GHC file writes already use that namespace for ordinary absolute paths.+extendedWindowsPath :: FilePath -> FilePath+extendedWindowsPath path = case FilePath.normalise path of+  absolute@('\\' : '\\' : marker : '\\' : _) | marker == '?' || marker == '.' -> absolute+  '\\' : '\\' : unc -> "\\\\?\\UNC\\" ++ unc+  absolute -> "\\\\?\\" ++ absolute+#endif++pathOccupied :: Path.Path Path.Abs t -> IO Bool+pathOccupied file = do+  let name = FilePath.dropTrailingPathSeparator (Path.toFilePath file)+  exists <- doesPathExist name+  if exists then return True else+    pathIsSymbolicLink name `catchIOError` \err ->+      if isDoesNotExistError err then return False else ioError err  isGlobMetacharacter :: Char -> Bool isGlobMetacharacter c = c `elem` ("*?[{<" :: String)
+ src/Hledger/Flow/ReportReview.hs view
@@ -0,0 +1,199 @@+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}++module Hledger.Flow.ReportReview+  ( Review (..),+    ReviewCategory (..),+    ReviewItem (..),+    ReviewFile,+    ReviewKind (..),+    captureReportInputs,+    countTransferAccounts,+    countUnknownTransactions,+    invalidateReportReview,+    loadReportReview,+    reviewFile,+    reviewHasData,+    saveReportReview,+    startReportRun,+  )+where++import Control.Exception (IOException, try)+import Data.Aeson (FromJSON, ToJSON, decodeFileStrict', encodeFile)+import Data.Char (isDigit, isSpace)+import Data.List (nub, sortOn)+import Data.Maybe (mapMaybe)+import Data.Ord (Down (..))+import qualified Data.Text as T+import qualified Data.Text.IO as T+import Data.Time.Clock (UTCTime)+import GHC.Generics (Generic)+import Hledger.Flow.InputSnapshot (InputSnapshot, inputsUnchanged, snapshotInputs)+import Hledger.Flow.Cache (cacheDir, initializeCache)+import Hledger.Flow.LedgerSelection (journalInputs)+import Hledger.Flow.PathHelpers (AbsFile, RelFile, withStagedOutput, writeTimestampMarker)+import Hledger.Flow.Types (BaseDir, HledgerConf, HledgerInfo, hledgerConfPath)+import Path+import qualified Path.IO as PathIO+import qualified System.Directory as D+import qualified System.FilePath as F+import System.IO.Error (catchIOError, isDoesNotExistError)+import Text.Read (readMaybe)++data ReviewKind = UnknownTransactions | TransferBalances+  deriving (Eq, Generic, Show, FromJSON, ToJSON)++data ReviewFile = ReviewFile ReviewKind RelFile Integer+  deriving (Generic, FromJSON, ToJSON)++data ReviewItem = ReviewItem+  { itemKind :: ReviewKind,+    itemPath :: FilePath,+    itemYear :: Integer,+    itemCount :: Maybe Int+  }+  deriving (Show)++data ReviewCategory = ReviewCategory {reviewCount :: Maybe Int, reviewFiles :: [FilePath]}+  deriving (Show)++data Review = Review+  { reviewUnknowns :: ReviewCategory,+    reviewTransfers :: ReviewCategory,+    reviewItems :: [ReviewItem]+  }+  deriving (Show)++data ReportInputs = ReportInputs+  { runStarted :: UTCTime,+    runAggregate :: AbsFile,+    runConfig :: Maybe AbsFile,+    runInputs :: InputSnapshot+  }+  deriving (Generic, FromJSON, ToJSON)++data Manifest = Manifest ReportInputs [ReviewFile]+  deriving (Generic, FromJSON, ToJSON)++manifestFile :: BaseDir -> AbsFile+manifestFile base = cacheDir base </> [relfile|reports.json|]++invalidateReportReview :: BaseDir -> IO ()+invalidateReportReview base =+  PathIO.removeFile (manifestFile base) `catchIOError` \err ->+    if isDoesNotExistError err then return () else ioError err++startReportRun :: BaseDir -> IO UTCTime+startReportRun base = do+  initializeCache base+  invalidateReportReview base+  writeTimestampMarker (cacheDir base </> [relfile|report-started|])++configPath :: Maybe HledgerConf -> IO (Maybe AbsFile)+configPath = traverse ((>>= parseAbsFile) . D.canonicalizePath . hledgerConfPath)++captureReportInputs :: HledgerInfo -> UTCTime -> AbsFile -> Maybe HledgerConf -> [[AbsFile]] -> IO (Either T.Text ReportInputs)+captureReportInputs info started aggregate conf roots = do+  discovered <- traverse (journalInputs info) (nub roots)+  case sequence discovered of+    Left err -> return (Left err)+    Right paths -> do+      result <- try $ do+        config <- configPath conf+        snapshot <- snapshotInputs (maybe id (:) config (concat paths))+        return (ReportInputs started aggregate config snapshot)+      return (either (Left . T.pack . show) Right (result :: Either IOException ReportInputs))++reviewFile :: BaseDir -> AbsFile -> Maybe ReviewFile+reviewFile base file = do+  kind <- case toFilePath (filename file) of+    "unknown-transactions.txt" -> Just UnknownTransactions+    "transfer-balance.txt" -> Just TransferBalances+    _ -> Nothing+  rel <- stripProperPrefix base file+  year <- readMaybe (F.dropTrailingPathSeparator (toFilePath (dirname (parent file))))+  return (ReviewFile kind rel year)++saveReportReview :: BaseDir -> ReportInputs -> [ReviewFile] -> IO ()+saveReportReview base inputs files =+  withStagedOutput (manifestFile base) (\stage -> encodeFile (toFilePath stage) (Manifest inputs files))++loadReportReview :: BaseDir -> AbsFile -> Maybe HledgerConf -> IO (Maybe Review)+loadReportReview base aggregate conf = do+  result <- try $ do+    saved <- decodeFileStrict' (toFilePath (manifestFile base))+    case saved of+      Nothing -> return Nothing+      Just (Manifest inputs files) -> do+        currentConfig <- configPath conf+        if runConfig inputs /= currentConfig || runAggregate inputs /= aggregate+          then return Nothing+          else do+            current <- inputsUnchanged (runInputs inputs) (runStarted inputs)+            if current then Just . summarize <$> traverse readReview files else return Nothing+  return (either (const Nothing :: IOException -> Maybe Review) id result)+  where+    readReview (ReviewFile kind rel year) = do+      text <- T.readFile (toFilePath (base </> rel))+      let body = T.unlines (drop 2 (T.lines text))+          count = if T.null (T.strip body) then Nothing else countFor kind body+      return (ReviewItem kind (toFilePath rel) year count)+    summarize items =+      let ordered = sortOn (\item -> (Down (itemYear item), itemPath item)) items+       in Review (category UnknownTransactions ordered) (category TransferBalances ordered) ordered+    category kind items =+      let relevant = filter ((== kind) . itemKind) items+       in ReviewCategory (sum <$> traverse itemCount relevant) (map itemPath relevant)++countFor :: ReviewKind -> T.Text -> Maybe Int+countFor UnknownTransactions = countUnknownTransactions+countFor TransferBalances = countTransferAccounts++reviewHasData :: ReviewKind -> T.Text -> Bool+reviewHasData kind text = case filter (not . T.null) (map T.strip (T.lines text)) of+  [] -> False+  [line] | csvHeader kind line -> False+  _ -> countFor kind text /= Just 0++csvHeader :: ReviewKind -> T.Text -> Bool+csvHeader UnknownTransactions = T.isPrefixOf "\"txnidx\",\"date\","+csvHeader TransferBalances = T.isPrefixOf "\"account\","++countUnknownTransactions :: T.Text -> Maybe Int+countUnknownTransactions text+  | null rows = Just 0+  | not (null headers) && all known rows = Just (length headers)+  | otherwise = Nothing+  where+    rows = filter (not . T.null . T.strip) (T.lines text)+    headers = filter dateHeader rows+    known row = dateHeader row || maybe False (isSpace . fst) (T.uncons row) || ";" `T.isPrefixOf` row+    dateHeader row = case T.unpack (T.take 10 row) of+      [a,b,c,d,'-',e,f,'-',g,h] ->+        all isDigit [a,b,c,d,e,f,g,h]+          && maybe True (\(end, _) -> isSpace end || end == '=') (T.uncons (T.drop 10 row))+      _ -> False++countTransferAccounts :: T.Text -> Maybe Int+countTransferAccounts text = case filter (not . T.null) (map T.strip (T.lines text)) of+  [] -> Just 0+  title : table | "Balance changes in " `T.isPrefixOf` title ->+    case break divider table of+      (heading, _ : body)+        | any (\line -> "||" `T.isInfixOf` line || "║" `T.isInfixOf` line) heading ->+            let rows = filter (not . T.isPrefixOf "└") body+             in if all accountRow rows then Just (length rows) else Nothing+      _ -> Nothing+  _ -> Nothing+  where+    divider row = "╞" `T.isPrefixOf` row || ("=" `T.isPrefixOf` row && "++" `T.isInfixOf` row)+    accountRow row = case mapMaybe (\sep -> nonemptyField sep row) ["||", "║"] of+      [_] -> True+      _ -> False+    nonemptyField sep row =+      let (label, rest) = T.breakOn sep row+       in if T.null rest || T.null (T.strip (T.dropWhile (== '│') label)) then Nothing else Just label
src/Hledger/Flow/Reports.hs view
@@ -7,9 +7,10 @@ where  import Control.Concurrent.STM-import Control.Monad (filterM, unless, when)+import Control.Monad (filterM, when) import Data.Char (toLower) import Data.Either+import Data.Maybe (mapMaybe) import qualified Data.List as List import qualified Data.Text as T import qualified Data.Text.IO as T@@ -17,14 +18,15 @@ import Hledger.Flow.Environment (renderSettings, reportSettings) import Hledger.Flow.BaseDir (relativeToBase) import Hledger.Flow.Common+import Hledger.Flow.LedgerSelection import Hledger.Flow.Logging import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile)+import Hledger.Flow.ReportReview import Hledger.Flow.RuntimeOptions import Hledger.Flow.Status (plural) import qualified Hledger.Flow.Types as FlowTypes-import Path ((</>), parent, reldir, relfile, parseRelDir, toFilePath)+import Path ((</>), parent, reldir, relfile, parseAbsFile, parseRelDir, toFilePath) import qualified Path.IO as PathIO-import qualified System.Directory as Directory import Prelude hiding (putStrLn, readFile, writeFile)  data ReportParams = ReportParams@@ -51,15 +53,21 @@  generateReports' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> IO [Either AbsFile AbsFile] generateReports' opts ch = do+  started <- startReportRun (baseDir opts)   owners <- listOwners opts-  configured <- configuredLedgerFile-  let hledgerJournal = maybe (toFilePath (baseDir opts </> [relfile|all-years.journal|])) snd configured-  hledgerJournalExists <- Directory.doesFileExist hledgerJournal+  info <- hledgerInfo opts   homes <- homeDirs-  unless hledgerJournalExists $ ioError $ userError $ case configured of-    Just (v, _) -> "LEDGER_FILE is set to " ++ v ++ ", which doesn't exist. Point it at "-      ++ tildePath homes (toFilePath (baseDir opts </> [relfile|all-years.journal|])) ++ ", or unset it."-    Nothing -> "No all-years.journal in " ++ tildePath homes (toFilePath (baseDir opts)) ++ " yet. Run `hledger-flow import` first."+  selection <- selectLedger homes (Just info) (baseDir opts)+  journal <- case (aggregateJournal selection, selectionError selection) of+    (Just file, _) -> return file+    (_, Just problem) -> ioError (userError (T.unpack problem))+    _ -> ioError (userError "No journal selected for reporting.")+  let hledgerJournal = toFilePath journal+  case selectedLedgerFile selection of+    Just file | lfCoverage file == CoversLedger ->+      channelOutLn ch ("Reporting on " <> T.pack (tildePath homes hledgerJournal) <> " (from LEDGER_FILE).")+    Just file -> channelOutLn ch (ledgerFileProblem homes file <> " Reporting on all-years.journal.")+    Nothing -> return ()   let journalWithYears = baseDir opts </> [relfile|import/all-years.journal|]   let aggregateReportDir         | any ((== toFilePath [reldir|all|]) . map toLower . toFilePath) owners = baseDir opts </> [reldir|reports/_all_|]@@ -77,7 +85,13 @@   let sharedOptions = (if prettyReports opts then ["--pretty-tables"] else []) ++ ["--depth", "2"]   let ownerWithAggregateReports = List.concat $ fmap (reportActions opts ch [incomeStatement sharedOptions, incomeMonthlyStatement sharedOptions, balanceSheet sharedOptions]) ownerWithAggregateParams   let ownerOnlyReports = List.concat $ fmap (reportActions opts ch [accountList, unknownTransactions]) ownerParams-  parAwareActions opts (aggregateOnlyReports ++ ownerWithAggregateReports ++ ownerOnlyReports)+  roots <- traverse (traverse parseAbsFile . ledgerFiles) ownerWithAggregateParams+  inputs <- captureReportInputs info started journal (hledgerConf opts) roots+  results <- parAwareActions opts (aggregateOnlyReports ++ ownerWithAggregateReports ++ ownerOnlyReports)+  case inputs of+    Left problem -> logVerbose opts ch ("Could not record report inputs: " <> problem)+    Right available -> saveReportReview (baseDir opts) available (mapMaybe (reviewFile (baseDir opts)) (rights results))+  return results  reportActions :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [ReportGenerator] -> ReportParams -> [IO (Either AbsFile AbsFile)] reportActions opts ch reports (ReportParams journals years reportsDir) = do@@ -92,7 +106,7 @@ unknownTransactions :: ReportGenerator unknownTransactions opts ch journals baseOutDir year = do   let reportArgs = ["print", "unknown"]-  generateReport opts ch journals year baseOutDir [relfile|unknown-transactions.txt|] reportArgs (not . T.null)+  generateReport opts ch journals year baseOutDir [relfile|unknown-transactions.txt|] reportArgs (reviewHasData UnknownTransactions)  incomeStatement :: [T.Text] -> ReportGenerator incomeStatement sharedOptions opts ch journals baseOutDir year = do@@ -112,7 +126,7 @@ transferBalance :: ReportGenerator transferBalance opts ch journals baseOutDir year = do   let reportArgs = ["balance"] ++ (if prettyReports opts then ["--pretty-tables"] else []) ++ ["--quarterly", "--flat", "--no-total", "transfer"]-  generateReport opts ch journals year baseOutDir [relfile|transfer-balance.txt|] reportArgs (\txt -> (length $ T.lines txt) > 4)+  generateReport opts ch journals year baseOutDir [relfile|transfer-balance.txt|] reportArgs (reviewHasData TransferBalances)  generateReport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [FilePath] -> Integer -> AbsDir -> RelFile -> [T.Text] -> (T.Text -> Bool) -> IO (Either AbsFile AbsFile) generateReport opts logCh journals year reportsDir fileName args successCheck = do
src/Hledger/Flow/RulesTemplate.hs view
@@ -8,43 +8,32 @@     templateFields,     rulesTemplate,     rulesTemplateFor,+    rulesStarterFor,+    runRules,   ) where -import Data.Char (isAlphaNum, isDigit, toLower)+import Data.Char (isAlpha, isAlphaNum, isDigit, toLower) import qualified Data.List as List import Data.Maybe (fromMaybe) import qualified Data.Set as Set import qualified Data.Text as T import Hledger.Flow.BaseDir (relativeToBase') import Hledger.Flow.DocHelpers (DocLink (..), docURL)+import Hledger.Flow.Common (homeDirs, nativeShell, tildePath)+import Hledger.Flow.Import.ImportHelpers (defaultRulesCandidates, extractImportDirs) import Hledger.Flow.Import.Inventory (findImportFiles, stageFiles)+import Hledger.Flow.Init.Example (isSampleStatement)+import Hledger.Flow.PathHelpers (AbsFile, writeNewUtf8File)+import Hledger.Flow.StatementScan+import Hledger.Flow.Status (flowCommand) import Hledger.Flow.Status.Gather (accountName, cachedStatement)-import Hledger.Flow.Types (BaseDir)+import Hledger.Flow.Types (BaseDir, HledgerConf) import Path-import qualified System.FilePath as FilePath-import System.IO (IOMode (ReadMode), hIsEOF, hSetEncoding, mkTextEncoding, withFile)+import qualified Path.IO as PathIO import qualified Data.Text.IO as T---- | The separator and the column names of a header line. Strips a UTF-8 BOM,--- surrounding quotes and spaces.--- ponytail: a quoted name containing the separator is split; use a CSV parser if that bites.-splitHeader :: T.Text -> (Char, [T.Text])-splitHeader line = (sep, map clean (T.splitOn (T.singleton sep) l))-  where-    l = T.dropWhileEnd (== '\r') (T.dropWhile (== '\xFEFF') line)-    count c = T.count (T.singleton c) l-    sep = List.foldl' (\best c -> if count c > count best then c else best) ',' [';', '\t']-    clean = T.strip . unquote . T.strip-    unquote t-      | T.length t >= 2 && T.head t == '"' && T.last t == '"' = T.init (T.tail t)-      | otherwise = t---- | False when any column looks like a date or an amount.-looksLikeHeader :: [T.Text] -> Bool-looksLikeHeader = not . any dataish-  where-    dataish f = not (T.null f) && T.all (`elem` ("0123456789-/.:,+$€£ " :: String)) f+import System.IO (Newline (..), nativeNewline)+import System.IO.Error (tryIOError, isAlreadyExistsError)  -- | hledger field names for the columns: obvious date, description and amount -- columns are named for hledger; the rest keep a normalised version of their name.@@ -91,18 +80,24 @@ -- | The starter rules file for a statement, from its contents. rulesTemplate :: T.Text -> T.Text -> FilePath -> T.Text -> Either T.Text T.Text rulesTemplate bank account path contents-  | map toLower (FilePath.takeExtension path) `notElem` [".csv", ".tsv", ".ssv"] = Left (notCsv path)-  | T.null (T.strip line) || not (looksLikeHeader names) =-      Left ("The first line of " <> T.pack path <> " looks like data, not column names. Write the fields line yourself: " <> docURL RulesFiles)-  | otherwise =-      Right . T.unlines $+  | not (isDelimitedFile path) = Left (notCsv path)+  | otherwise = renderTemplate bank account path (parseRuleSample path contents)++renderTemplate :: T.Text -> T.Text -> FilePath -> Either T.Text DelimitedStatement -> Either T.Text T.Text+renderTemplate bank account path result = case result of+  Left problem -> Left (T.pack path <> ": " <> problem <> " Write the fields line yourself: " <> docURL RulesFiles)+  Right statement -> case statementHeader statement of+    Nothing -> headerlessTemplate bank account path statement+    Just names ->+      let fields = templateFields names+       in Right . T.unlines $         [ "# Starter rules for " <> bank <> "/" <> account <> ", made by hledger-flow from the header row of",           "# " <> T.pack path <> ". Check every line, then run: hledger-flow import",           "# hledger's CSV rules reference: https://hledger.org/hledger.html#csv",           "",           "skip 1"         ]-          ++ ["separator " <> (if sep == '\t' then "TAB" else T.singleton sep) | sep /= ',']+          ++ separatorRule (statementSeparator statement)           ++ [ "fields " <> T.intercalate ", " fields,                ""              ]@@ -120,38 +115,113 @@                "account1 assets:" <> bank <> ":" <> account,                "account2 expenses:unknown"              ]+separatorRule :: Char -> [T.Text]+separatorRule sep = ["separator " <> (if sep == '\t' then "TAB" else T.singleton sep) | sep /= ',']++headerlessTemplate :: T.Text -> T.Text -> FilePath -> DelimitedStatement -> Either T.Text T.Text+headerlessTemplate bank account path statement = case statementRows statement of+  [] -> Left ("No data rows in " <> T.pack path <> ".")+  sample : _ ->+    let columns = zip [1 :: Int ..] (List.transpose (statementRows statement))+        dateCandidates = [(n, cells) | (n, column) <- columns, Just cells <- [traverse parseDateCell column]]+        dateChoice = unique dateCandidates+        amountChoice = unique [n | (n, column) <- columns, all isSignedDecimal column]+        descriptionChoice = unique+          [ n | (n, column) <- columns, all (T.any isAlpha) column, Set.size (Set.fromList column) > 1+          ]+        field n+          | Just n == fmap fst dateChoice = "date"+          | Just n == amountChoice = "amount"+          | Just n == descriptionChoice = "description"+          | otherwise = columnName n+        dateRule = case dateChoice of+          Just (n, cells) -> case inferDateFormat cells of+            Just "%Y-%m-%d" -> []+            Just layout -> ["date-format " <> layout]+            Nothing -> ["# date-format (set the order and separator for " <> sampleValue n sample <> ")"]+          Nothing -> ["# Rename the transaction date column to date, then set date-format if needed."]+     in Right . T.unlines $+          [ "# Starter rules for " <> bank <> "/" <> account <> ", made by hledger-flow from",+            "# " <> T.pack path <> ", which has no header row.",+            "# The first row, by column:"+          ]+            ++ concat (zipWith commentValue [1 :: Int ..] sample)+            ++ ["# Check every line, then run: hledger-flow import", ""]+            ++ separatorRule (statementSeparator statement)+            ++ ["fields " <> T.intercalate ", " (map (field . fst) columns)]+            ++ dateRule+            ++ ["# Rename the amount column to amount." | amountChoice == Nothing]+            ++ ["# Rename the payee column to description." | descriptionChoice == Nothing]+            ++ ["", "account1 assets:" <> bank <> ":" <> account, "account2 expenses:unknown"]   where-    line = T.takeWhile (/= '\n') contents-    (sep, names) = splitHeader line-    fields = templateFields names+    unique [one] = Just one+    unique _ = Nothing+    columnName n = "column" <> T.pack (show n)+    sampleValue n = T.filter (/= '\n') . T.intercalate " " . take 1 . drop (n - 1)+    commentValue n value = case T.splitOn "\n" value of+      first : rest -> ("#   " <> columnName n <> " =" <> if T.null first then "" else " " <> first)+        : map ("#     " <>) rest+      [] -> [] --- | The starter rules file for an account (@owner/bank/account@), from its newest statement.-rulesTemplateFor :: BaseDir -> T.Text -> IO (Either T.Text T.Text)-rulesTemplateFor base requested = do-  let account = T.dropWhileEnd (== '/') (maybe requested id (T.stripPrefix "import/" requested))+-- | The first default rules candidate and the starter for the newest statement,+-- skipping init's sample statement when the account also has real ones.+rulesStarterFor :: BaseDir -> T.Text -> IO (Either T.Text (AbsFile, T.Text))+rulesStarterFor base requested = do+  let account = T.dropWhileEnd (== '/') (fromMaybe requested (T.stripPrefix "import/" requested))       root = base </> [reldir|import|]   files <- findImportFiles root   let inputs = stageFiles root [reldir|1-in|] files-      statements = filter ((== account) . accountName root) inputs+      found = filter ((== account) . accountName root) inputs+      statements = case filter (not . isSampleStatement) found of+        [] -> found+        real -> real       known = List.nub (map (accountName root) inputs)   case reverse statements of-    [] ->-      return . Left $-        "No statements found for " <> account <> ". Accounts with statements: "-          <> (if null known then "none" else T.intercalate ", " known) <> "."-    newest : _ -> do-      source <- fromMaybe newest <$> cachedStatement newest-      let path = relativeToBase' (toFilePath base) (toFilePath source)-          isCsv = map toLower (FilePath.takeExtension path) `elem` [".csv", ".tsv", ".ssv"]-          (bank, acct) = case T.splitOn "/" account of-            [_, b, a] -> (b, a)-            _ -> (account, account)-      header <- if isCsv then firstLine (toFilePath source) else return ""-      return (rulesTemplate bank acct path header)+    [] -> return . Left $+      "No statements found for " <> account <> ". Accounts with statements: "+        <> (if null known then "none" else T.intercalate ", " known) <> "."+    newest : _ -> case extractImportDirs root newest of+      Left problem -> return (Left problem)+      Right dirs -> do+        candidates <- defaultRulesCandidates dirs+        case candidates of+          target : _ -> do+            source <- fromMaybe newest <$> cachedStatement newest+            let path = relativeToBase' (toFilePath base) (toFilePath source)+                (bank, acct) = case T.splitOn "/" account of+                  [_, b, a] -> (b, a)+                  _ -> (account, account)+            if isDelimitedFile path+              then fmap (fmap (\template -> (target, template)) . renderTemplate bank acct path) (readRuleSample source)+              else return (Left (notCsv path))+          [] -> return (Left ("No rules file target for " <> account <> ".")) --- | The first line of a file, tolerating encodings other than UTF-8.-firstLine :: FilePath -> IO T.Text-firstLine path = withFile path ReadMode $ \h -> do-  hSetEncoding h =<< mkTextEncoding "UTF-8//ROUNDTRIP"-  eof <- hIsEOF h-  if eof then return "" else T.hGetLine h+rulesTemplateFor :: BaseDir -> T.Text -> IO (Either T.Text T.Text)+rulesTemplateFor base = fmap (fmap snd) . rulesStarterFor base++runRules :: Maybe HledgerConf -> BaseDir -> T.Text -> IO ()+runRules conf base account = do+  result <- rulesStarterFor base account+  (target, template) <- either (ioError . userError . T.unpack) return result+  cwd <- PathIO.getCurrentDir+  homes <- homeDirs+  let path = toFilePath target+      shown = T.pack (if cwd == base then relativeToBase' (toFilePath base) path else tildePath homes path)+      command sub args = flowCommand nativeShell homes cwd base conf sub args+      -- Match the native line endings of status --rules-template on stdout.+      fileTemplate = case nativeNewline of+        LF -> template+        CRLF -> T.replace "\n" "\r\n" template+  written <- tryIOError (writeNewUtf8File target fileTemplate)+  case written of+    Left err -> do+      occupied <- PathIO.doesFileExist target+      let problem+            | occupied || isAlreadyExistsError err =+                shown <> " already exists. Delete it first or preview a fresh starter with: "+                  <> command "status" ["--rules-template", account]+            | otherwise = "Could not create " <> shown <> ": " <> T.pack (show err)+      ioError (userError (T.unpack problem))+    Right () -> do+      T.putStrLn ("Created " <> shown)+      T.putStrLn ("Edit it: check the fields line and the account names. Then run: " <> command "import" [])
+ src/Hledger/Flow/StatementScan.hs view
@@ -0,0 +1,285 @@+{-# LANGUAGE OverloadedStrings #-}++-- | Delimited statement records and the date evidence they contain.+module Hledger.Flow.StatementScan+  ( DelimitedStatement (..),+    parseDelimitedStatement,+    parseRuleSample,+    readRuleSample,+    readDelimitedStatement,+    splitHeader,+    looksLikeHeader,+    isDelimitedFile,+    DateCell,+    parseDateCell,+    dateCellYear,+    inferDateFormat,+    isSignedDecimal,+    StatementYear (..),+    inferStatementYear,+  )+where++import Data.Char (isDigit, isSpace, toLower)+import Data.List (intersect, nub, sort, transpose)+import Data.Maybe (catMaybes, isJust)+import qualified Data.Text as T+import qualified Data.Text.IO as T+import qualified Data.Text.Read as T+import Data.Time (fromGregorianValid)+import Hledger.Flow.PathHelpers (AbsFile)+import Path (toFilePath)+import qualified System.FilePath as F+import System.IO (IOMode (ReadMode), Handle, hIsEOF, hSetEncoding, mkTextEncoding, withFile)++data DelimitedStatement = DelimitedStatement+  { statementSeparator :: Char,+    statementHeader :: Maybe [T.Text],+    statementRows :: [[T.Text]]+  }+  deriving (Eq, Show)++isDelimitedFile :: FilePath -> Bool+isDelimitedFile path = map toLower (F.takeExtension path) `elem` [".csv", ".tsv", ".ssv"]++parseDelimitedStatement :: FilePath -> T.Text -> Either T.Text DelimitedStatement+parseDelimitedStatement = parseStatementSample (const Nothing)++parseRuleSample :: FilePath -> T.Text -> Either T.Text DelimitedStatement+parseRuleSample = parseStatementSample (Just . ruleRows)++parseStatementSample :: (DelimitedStatement -> Maybe Int) -> FilePath -> T.Text -> Either T.Text DelimitedStatement+parseStatementSample limit file contents =+  case logicalRecords (T.lines (T.dropWhile (== '\xFEFF') contents)) of+    [] -> Left noColumns+    first : rest -> do+      initial <- first >>= startStatement file+      rows <- traverse (>>= parseRow initial) (maybe rest (\n -> take (remaining n initial) rest) (limit initial))+      return initial {statementRows = statementRows initial ++ rows}++readDelimitedStatement :: Maybe Int -> AbsFile -> IO (Either T.Text DelimitedStatement)+readDelimitedStatement = readSample . const++readRuleSample :: AbsFile -> IO (Either T.Text DelimitedStatement)+readRuleSample = readSample (Just . ruleRows)++ruleRows :: DelimitedStatement -> Int+ruleRows initial = case statementHeader initial of+  Just _ -> 0+  Nothing -> 50++readSample :: (DelimitedStatement -> Maybe Int) -> AbsFile -> IO (Either T.Text DelimitedStatement)+readSample limit path = withFile (toFilePath path) ReadMode $ \handle -> do+  hSetEncoding handle =<< mkTextEncoding "UTF-8//ROUNDTRIP"+  first <- readRecord handle+  case first >>= traverse (startStatement (toFilePath path) . T.dropWhile (== '\xFEFF')) of+    Left err -> return (Left err)+    Right Nothing -> return (Left noColumns)+    Right (Just initial) -> do+      rows <- readRows handle initial (fmap (\n -> remaining n initial) (limit initial)) []+      return $ fmap (\xs -> initial {statementRows = statementRows initial ++ xs}) rows++remaining :: Int -> DelimitedStatement -> Int+remaining n initial = n - length (statementRows initial)++readRows :: Handle -> DelimitedStatement -> Maybe Int -> [[T.Text]] -> IO (Either T.Text [[T.Text]])+readRows _ _ (Just n) rows | n <= 0 = return (Right (reverse rows))+readRows handle initial limit rows = do+  next <- readRecord handle+  case next >>= traverse (parseRow initial) of+    Left err -> return (Left err)+    Right Nothing -> return (Right (reverse rows))+    Right (Just row) -> readRows handle initial (subtract 1 <$> limit) (row : rows)++readRecord :: Handle -> IO (Either T.Text (Maybe T.Text))+readRecord handle = do+  eof <- hIsEOF handle+  if eof then return (Right Nothing) else do+    line <- stripCR <$> T.hGetLine handle+    if T.null (T.strip line) then readRecord handle else finish (openQuote False line) [line]+  where+    finish False chunks = return (Right (Just (T.intercalate "\n" (reverse chunks))))+    finish True chunks = do+      eof <- hIsEOF handle+      if eof then return (Left "Unclosed quoted field.") else do+        line <- stripCR <$> T.hGetLine handle+        finish (openQuote True line) (line : chunks)++logicalRecords :: [T.Text] -> [Either T.Text T.Text]+logicalRecords = next . map stripCR+  where+    next [] = []+    next (line : rest)+      | T.null (T.strip line) = next rest+      | otherwise = collect (openQuote False line) [line] rest+    collect False chunks rest = Right (T.intercalate "\n" (reverse chunks)) : next rest+    collect True _ [] = [Left "Unclosed quoted field."]+    collect True chunks (line : rest) = collect (openQuote True line) (line : chunks) rest++stripCR :: T.Text -> T.Text+stripCR = T.dropWhileEnd (== '\r')++openQuote :: Bool -> T.Text -> Bool+openQuote = T.foldl' (\inside c -> if c == '"' then not inside else inside)++noColumns :: T.Text+noColumns = "Could not determine the statement's columns."++startStatement :: FilePath -> T.Text -> Either T.Text DelimitedStatement+startStatement file record = do+  let sep = chooseSeparator (map toLower (F.takeExtension file)) record+  fields <- splitFields sep record+  case fields of+    _ : _ : _ | looksLikeHeader fields -> return (DelimitedStatement sep (Just fields) [])+              | otherwise -> return (DelimitedStatement sep Nothing [fields])+    _ -> Left noColumns++parseRow :: DelimitedStatement -> T.Text -> Either T.Text [T.Text]+parseRow initial record = do+  row <- splitFields (statementSeparator initial) record+  let width = case (statementHeader initial, statementRows initial) of+        (Just header, _) -> length header+        (_, first : _) -> length first+        _ -> 0+  if length row == width then Right row else Left "Statement rows have different numbers of columns."++splitHeader :: T.Text -> (Char, [T.Text])+splitHeader line =+  let record = stripCR (T.dropWhile (== '\xFEFF') line)+      sep = chooseSeparator ".csv" record+   in (sep, either (const []) id (splitFields sep record))++chooseSeparator :: String -> T.Text -> Char+chooseSeparator ".tsv" _ = '\t'+chooseSeparator ".ssv" _ = ';'+chooseSeparator _ record = foldl' pick ',' [';', '\t']+  where+    pick best other | count other > count best = other+                    | otherwise = best+    count sep = snd (T.foldl' (step sep) (False, 0 :: Int) record)+    step _ (quoted, n) '"' = (not quoted, n)+    step sep (quoted, n) c = (quoted, if not quoted && c == sep then n + 1 else n)++splitFields :: Char -> T.Text -> Either T.Text [T.Text]+splitFields sep = field [] . T.unpack+  where+    field fields rest+      | '"' : contents <- dropWhile (\c -> isSpace c && c /= sep) rest = quoted fields [] contents+      | otherwise =+          let (value, tail') = break (== sep) rest+           in if '"' `elem` value then Left "Unexpected quote in a delimited field."+              else end fields (T.strip (T.pack value)) tail'+    quoted _ _ [] = Left "Unclosed quoted field."+    quoted fields value ('"' : '"' : rest) = quoted fields ('"' : value) rest+    quoted fields value ('"' : rest) =+      end fields (T.strip (T.pack (reverse value))) (dropWhile (\c -> isSpace c && c /= sep) rest)+    quoted fields value (c : rest) = quoted fields (c : value) rest+    end fields value [] = Right (reverse (value : fields))+    end fields value (c : rest)+      | c == sep = field (value : fields) rest+      | otherwise = Left "Unexpected text after a quoted field."++looksLikeHeader :: [T.Text] -> Bool+looksLikeHeader = not . any dataish+  where+    dataish f = isJust (parseDateCell f) ||+      (not (T.null f) && T.all (`elem` ("0123456789-/.:,+$€£ " :: String)) f)++isSignedDecimal :: T.Text -> Bool+isSignedDecimal raw = case T.splitOn "." (dropSign raw) of+  [whole] -> digits whole+  [whole, fraction] -> digits whole && digits fraction+  _ -> False+  where+    dropSign t = case T.uncons t of+      Just (sign, rest) | sign `elem` ['+', '-'] -> rest+      _ -> t++digits :: T.Text -> Bool+digits t = not (T.null t) && T.all (\c -> c >= '0' && c <= '9') t++decimal :: T.Text -> Maybe Int+decimal t+  | digits t, Right (n, "") <- T.decimal t = Just n+  | otherwise = Nothing++data DateCell = DateCell {dateCellYear :: Integer, dateFormats :: [T.Text]}+  deriving (Eq, Show)++parseDateCell :: T.Text -> Maybe DateCell+parseDateCell value = do+  let (date, suffix) = T.break (\c -> isSpace c || c == 'T') (T.strip value)+  (sep, _) <- T.uncons (T.dropWhile isDigit date)+  if sep `elem` ['-', '/', '.'] then Just () else Nothing+  -- hledger accepts both padded and unpadded components with %-m and %-d.+  case T.splitOn (T.singleton sep) date of+    [a,b,c]+      | T.length a == 4 && short b && short c -> do+          year <- parseYear a+          m <- decimal b+          d <- decimal c+          -- Year-first inputs use conventional ISO order wherever it fits.+          let formats = if valid year m d then [fmt sep ["%Y","%-m","%-d"]]+                        else [fmt sep ["%Y","%-d","%-m"] | valid year d m]+          cell year formats suffix+      | short a && short b && T.length c == 4 -> do+          year <- parseYear c+          lo <- decimal a+          hi <- decimal b+          cell year ([fmt sep ["%-m","%-d","%Y"] | valid year lo hi] +++                     [fmt sep ["%-d","%-m","%Y"] | valid year hi lo]) suffix+    _ -> Nothing+  where+    short t = T.length t <= 2 && digits t+    valid y m d = isJust (fromGregorianValid y m d)+    parseYear t = do+      n <- decimal t+      if n >= 1900 && n <= 2099 then Just (toInteger n) else Nothing+    fmt sep = T.intercalate (T.singleton sep)+    cell _ [] _ = Nothing+    cell year formats suffix = case timeSuffix suffix of+      Nothing -> Nothing+      Just layout -> Just (DateCell year (maybe [] (\timeFormat -> map (<> timeFormat) formats) layout))++-- | An unfamiliar but time-like suffix supplies a year without a format guess.+timeSuffix :: T.Text -> Maybe (Maybe T.Text)+timeSuffix "" = Just (Just "")+timeSuffix suffix = do+  let (joiner, time) = case T.stripPrefix "T" suffix of+        Just t -> ("T", t)+        Nothing -> (" ", T.stripStart suffix)+  (hour, tail') <- T.uncons time+  if isDigit hour && ":" `T.isInfixOf` tail' then Just () else Nothing+  return $ case T.splitOn ":" time of+    [h,m] | within 23 h && within 59 m -> Just (joiner <> "%H:%M")+    [h,m,s] | within 23 h && within 59 m && within 59 s -> Just (joiner <> "%H:%M:%S")+    _ -> Nothing+  where+    within bound t = T.length t == 2 && maybe False (<= bound) (decimal t)++inferDateFormat :: [DateCell] -> Maybe T.Text+inferDateFormat cells = case map dateFormats cells of+  first : rest | [one] <- foldl' intersect first rest -> Just one+  _ -> Nothing++data StatementYear = UnknownYear | SingleYear Integer | SpanningYears [Integer]+  deriving (Eq, Show)++inferStatementYear :: DelimitedStatement -> StatementYear+inferStatementYear statement = case traverse agreed (transpose dateColumns) of+  Just years -> case sort (nub years) of+    [year] -> SingleYear year+    [] -> UnknownYear+    many -> SpanningYears many+  Nothing -> UnknownYear+  where+    dateColumns =+      [ dates+        | column <- transpose (statementRows statement),+          let dates = map parseDateCell column,+          any isJust dates,+          and (zipWith (\cell date -> T.null cell || isJust date) column dates)+      ]+    agreed cells = case nub (map dateCellYear (catMaybes cells)) of+      [year] -> Just year+      _ -> Nothing
src/Hledger/Flow/Status.hs view
@@ -12,28 +12,39 @@     statementExtensions,     suggestPlacement,     misplacedFiles,+    placementDestination,+    misplacedMoveCommand,     nextStep,+    firstStatementStep,+    flowCommand,+    ledgerFileNote,+    balanceHint,+    withConfigCheck,     hledgerVersionNumber,     plural,   ) where  import Data.Char (isDigit, toLower)+import Data.List (nub) import Data.Maybe (fromMaybe, mapMaybe) import qualified Data.Set as Set import qualified Data.Text as T-import Hledger.Flow.Common (Shell (..), hledgerVersionNumber, homeShellPath, posixLiteral, powerShellLiteral, shellArgument, tildePath)+import Hledger.Flow.Common (Shell (..), createDirectoryCommand, hledgerFileArgument, hledgerVersionNumber, homeShellPath, moveFileCommand, posixLiteral, powerShellLiteral, shellArgument) import Hledger.Flow.DocHelpers (DocLink (..), docURL)+import Hledger.Flow.HledgerConfig (ConfigConflict, ConfigUse (..), firstConfigConflict, renderConfigConflict) import Hledger.Flow.Import.Inventory (stageFiles)-import Hledger.Flow.PathHelpers (AbsDir, AbsFile)-import Hledger.Flow.Types (BaseDir, HledgerInfo)-import Path (reldir, relfile, stripProperPrefix, toFilePath, (</>))+import Hledger.Flow.Init.Example (exampleAccountPath)+import Hledger.Flow.LedgerSelection (Coverage (..), LedgerFile (..), ledgerFileProblem)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile)+import Hledger.Flow.ReportReview+import Hledger.Flow.StatementScan (StatementYear (..))+import Hledger.Flow.Types (BaseDir, HledgerConf (..), HledgerInfo)+import Path (parseRelDir, parseRelFile, reldir, relfile, stripProperPrefix, toFilePath, (</>)) import qualified System.FilePath as FilePath  data Situation = InLedger BaseDir | EmptyDir | Elsewhere deriving (Eq, Show) -data LedgerFile = LedgerFile {lfValue :: FilePath, lfExists :: Bool} deriving (Eq, Show)- data AccountStatus = AccountStatus   { asAccount :: T.Text,     asStatements :: Int,@@ -41,6 +52,8 @@     asRulesFile :: Maybe FilePath,     asRulesCandidates :: [FilePath],     asWithoutRules :: Int,+    asOtherRules :: [FilePath],+    asRulesTargetOccupied :: Bool,     asPreprocess :: Bool,     asConstruct :: Bool,     asJournalsMissing :: Int,@@ -49,17 +62,31 @@   deriving (Eq, Show)  -- | Both paths are relative to the base directory and start with @import@.-data Misplaced = Misplaced {mpPath :: FilePath, mpSuggestion :: FilePath} deriving (Eq, Show)+data Misplaced = Misplaced+  { mpPath :: FilePath,+    mpSuggestion :: FilePath,+    mpYear :: StatementYear,+    mpDestinationExists :: Bool+  }+  deriving (Eq, Show)  data LedgerStatus = LedgerStatus   { lsBaseDir :: BaseDir,     lsHledger :: Maybe HledgerInfo,     lsLedgerFile :: Maybe LedgerFile,+    lsAggregateJournal :: Maybe AbsFile,+    lsHledgerConf :: Maybe HledgerConf,+    lsConfigConflicts :: [ConfigConflict],     lsAccounts :: [AccountStatus],     lsMisplaced :: [Misplaced],     lsManualJournals :: Int,+    lsHasManualWork :: Bool,     lsAllYearsExists :: Bool,-    lsNeedsReconciliation :: Bool+    lsNeedsReconciliation :: Bool,+    lsNeedsPrices :: Bool,+    lsReview :: Maybe Review,+    lsFirstStatementOwner :: RelDir,+    lsUntouchedExample :: Maybe RelDir   }   deriving (Show) @@ -101,69 +128,218 @@     misplaced f = do       rel <- stripProperPrefix root f       let parts = FilePath.splitDirectories (toFilePath rel)-      return (Misplaced (FilePath.joinPath ("import" : parts)) (suggestPlacement parts))+      let evidence = case filter isYear (take (length parts - 1) parts) of+            first : _ -> SingleYear (read first)+            [] -> UnknownYear+      return (Misplaced (FilePath.joinPath ("import" : parts)) (suggestPlacement parts) evidence False) +placementDestination :: Misplaced -> Maybe RelFile+placementDestination item = case mpYear item of+  SingleYear _ | null (missingParts item) -> parseRelFile (mpSuggestion item)+  _ -> Nothing++missingParts :: Misplaced -> [T.Text]+missingParts item =+  [T.pack part | part <- ["owner", "bank", "account"],+    ("<" ++ part ++ ">") `elem` FilePath.splitDirectories (FilePath.takeDirectory (mpSuggestion item))]++misplacedMoveCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Misplaced -> Maybe T.Text+misplacedMoveCommand _ _ _ _ item | mpDestinationExists item = Nothing+misplacedMoveCommand shell homes cwd base item = do+  source <- parseRelFile (mpPath item)+  target <- placementDestination item+  return (moveFileCommand shell homes cwd base source target)++placementDetails :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Misplaced -> [T.Text]+placementDetails shell homes cwd base item =+  [T.pack (mpPath item) <> " → " <> T.pack (mpSuggestion item)]+    ++ ["Choose the " <> T.intercalate ", " parts <> "." | parts@(_ : _) <- [missingParts item]]+    ++ case mpYear item of+      SpanningYears years ->+        ["This statement contains transactions from " <> T.intercalate ", " (map (T.pack . show) years)+          <> ". Split it into one file per year."]+      UnknownYear -> ["Couldn't tell the year from the file. Use the year of its transactions."]+      SingleYear _ | mpDestinationExists item ->+        ["The destination already exists. Compare the two paths above before moving the statement."]+      SingleYear _ -> ["Move it with: " <> cmd | Just cmd <- [misplacedMoveCommand shell homes cwd base item]]+ plural :: Int -> T.Text -> T.Text plural 1 noun = "1 " <> noun plural n noun = T.pack (show n) <> " " <> noun <> "s" +firstStatementStep :: Integer -> Shell -> [FilePath] -> AbsDir -> BaseDir -> RelDir -> NextStep+firstStatementStep year shell homes cwd base owner =+  NextStep+    "Put your first statement in import/<owner>/<bank>/<account>/1-in/<year>/"+    Nothing+    ( [ "owner: whose money it is, for example " <> ownerName <> ", a partner or a business",+        "bank: the institution, for example mybank",+        "account: one account at that bank, for example checking"+      ]+        ++ ["Change the names, then run: " <> createDirectoryCommand shell homes cwd+              (base </> [reldir|import|] </> owner </> [reldir|mybank/checking/1-in|] </> folder)+            | Just folder <- [parseRelDir (show year)]]+    )+    (Just (docURL InputFiles))+  where+    ownerName = T.pack (FilePath.dropTrailingPathSeparator (toFilePath owner))+ -- | The single most important next step. The first matching rule wins. -- Paths under one of @homes@ show as @~@, and as @"$HOME/…"@ in commands. nextStep :: Integer -> Shell -> [FilePath] -> AbsDir -> LedgerStatus -> NextStep nextStep year shell homes cwd s   | Nothing <- lsHledger s =       NextStep "Install hledger, or pass its location with --hledger-path" Nothing [] (Just "https://hledger.org/install.html")-  | statements == 0 && null (lsMisplaced s) && lsManualJournals s == 0 && not (lsNeedsReconciliation s) =-      NextStep-        "Put your first statement in import/<owner>/<bank>/<account>/1-in/<year>/"-        Nothing-        [ "For example: import/me/mybank/checking/1-in/" <> T.pack (show year) <> "/statement.csv",-          "Then check with: " <> command "status"-        ]-        (Just (docURL InputFiles))+  | statements == 0 && null (lsMisplaced s) && not (lsHasManualWork s) && not (lsNeedsReconciliation s) =+      firstStatement   | not (null (lsMisplaced s)) =       NextStep         ("Move " <> plural (length (lsMisplaced s)) "file" <> " to where hledger-flow looks")         Nothing-        [T.pack (mpPath m) <> " → " <> T.pack (mpSuggestion m) | m <- lsMisplaced s]-        (Just (docURL InputFiles))+        (concatMap (placementDetails shell homes cwd (lsBaseDir s)) (lsMisplaced s))+        (Just (docURL (if any spansYears (lsMisplaced s) then StatementYearFolders else InputFiles)))+  | a : _ <- filter needsRules (lsAccounts s),+    source : other <- asOtherRules a =+      case other of+        [] | not (asRulesTargetOccupied a), Just commandLine <- renameRules a source ->+          NextStep ("Rename " <> T.pack source <> " to " <> T.pack (relativeCandidate a))+            (Just commandLine) ["Files it includes can keep their names."] (Just (docURL RulesFiles))+        _ ->+          NextStep ("Choose the rules to save as " <> T.pack (relativeCandidate a)) Nothing+            (map T.pack (source : other) +++              ["The target already exists. Check it before making changes." | asRulesTargetOccupied a] +++              ["Rename the rules for this account. Files it includes can keep their names."])+            (Just (docURL RulesFiles))   | a : _ <- filter needsRules (lsAccounts s) =       NextStep-        ("Create " <> T.pack (tildePath homes (firstCandidate a)) <> " (rules for " <> asAccount a <> ")")-        (Just (command "status" <> " --rules-template " <> shellArgument shell (asAccount a)))-        ["Save its output as that file, then edit it."]+        ("Create " <> T.pack (relativeCandidate a) <> " (rules for " <> asAccount a <> ")")+        (Just (commandArgs "rules" [asAccount a]))+        ["It creates that file from the statement\'s columns, for you to edit."]         (Just (docURL RulesFiles))   | toImport > 0 =-      NextStep ("Import " <> plural toImport "new or changed statement") (Just importCommand) [] Nothing-  | not (lsAllYearsExists s) =-      NextStep "Run import to create all-years.journal" (Just importCommand) [] Nothing+      withConfigCheck ImportConfig s $+        NextStep ("Import " <> plural toImport "new or changed statement") (Just importCommand) [] Nothing+  | Nothing <- lsAggregateJournal s, not (lsAllYearsExists s), not missingConfigured =+      withConfigCheck ImportConfig s $+        NextStep "Run import to create all-years.journal" (Just importCommand) [] Nothing   | lsNeedsReconciliation s =-      NextStep "Run import to reconcile generated files and journal includes" (Just importCommand) [] Nothing-  | maybe True (not . lfExists) (lsLedgerFile s) =+      withConfigCheck ImportConfig s $+        NextStep "Run import to reconcile generated files and journal includes" (Just importCommand) [] Nothing+  | lsNeedsPrices s =+      withConfigCheck ImportConfig s $+        NextStep+          "Add market prices for your commodities"+          Nothing+          [ "Save each pair's P directives in prices/<year>/<commodity>-<quote>.prices, such as XAU-USD.prices.",+            "Include each pair file from prices/<year>/prices.journal, then run " <> importCommand <> "."+          ]+          (Just (docURL PriceFiles))+  | Just lf <- lsLedgerFile s, not (lfExists lf) =       NextStep         "Point LEDGER_FILE at this ledger so hledger finds it"-        (Just exportLine)-        [ "LEDGER_FILE is set to " <> T.pack (lfValue lf) <> ", which doesn't exist."-          | Just lf <- [lsLedgerFile s]-        ]+        (Just (setLedgerFileCommand shell homes s))+        [ledgerFileProblem homes lf]         (Just (docURL SetLedgerFile))-  | otherwise =-      NextStep "Generate reports, or explore with hledger" (Just (command "report")) ["Or try: hledger balance"] Nothing+  | otherwise = case lsReview s of+      Nothing -> withConfigCheck ReportConfig s $+        NextStep "Generate reports, or explore with hledger" (Just (command "report"))+          ["Or try: " <> hint | Nothing <- [ledgerFileNote shell homes s], Just hint <- [balanceHint shell homes s]] Nothing+      Just review+        | Just account <- lsUntouchedExample s, null (reviewItems review) ->+            NextStep "Adapt the example account to your ledger" Nothing+              [ "Rename the owner, bank and account folders in " <> exampleAccountPath account <> " to match your ledger.",+                "Replace the sample CSV under 1-in/ with your statement in the folder for its year.",+                "Replace the sample rules with your own <bank>-<account>.rules in the account folder.",+                "Use your account names in any starter journals you fill in.",+                "Then run " <> importCommand <> " and " <> command "report" <> "."+              ]+              (Just (docURL InputFiles))+        | otherwise -> reviewStep importCommand firstStatement review   where-    command subcommand = "hledger-flow " <> subcommand <> baseArgument-    baseArgument-      | cwd == lsBaseDir s = ""-      | otherwise = " " <> pasteable (shellArgument shell) (FilePath.dropTrailingPathSeparator (toFilePath (lsBaseDir s)))-    pasteable literal path = fromMaybe (literal (T.pack path)) (homeShellPath shell homes path)+    commandArgs = flowCommand shell homes cwd (lsBaseDir s) (lsHledgerConf s)+    command subcommand = commandArgs subcommand []+    spansYears item = case mpYear item of SpanningYears _ -> True; _ -> False+    missingConfigured = maybe False (not . lfExists) (lsLedgerFile s)+    firstStatement = firstStatementStep year shell homes cwd (lsBaseDir s) (lsFirstStatementOwner s)     importCommand = command "import"     statements = sum (map asStatements (lsAccounts s))     toImport = sum [asJournalsMissing a + asJournalsOutdated a | a <- lsAccounts s]     needsRules a = asWithoutRules a > 0 && not (asConstruct a)-    firstCandidate a = case asRulesCandidates a of-      c : _ | cwd /= lsBaseDir s -> toFilePath (lsBaseDir s) FilePath.</> c-            | otherwise -> c-      [] -> "a rules file"-    allYears = toFilePath (lsBaseDir s </> [relfile|all-years.journal|])-    exportLine = case shell of-      Posix -> "export LEDGER_FILE=" <> pasteable posixLiteral allYears-      PowerShell -> "$env:LEDGER_FILE = " <> pasteable powerShellLiteral allYears+    relativeCandidate a = fromMaybe "a rules file" (case asRulesCandidates a of+      target : _ -> Just target+      [] -> Nothing)+    renameRules a source = do+      old <- parseRelFile source+      new <- parseRelFile (relativeCandidate a)+      return (moveFileCommand shell homes cwd (lsBaseDir s) old new)++reviewStep :: T.Text -> NextStep -> Review -> NextStep+reviewStep importCommand firstStatement review+  | not (null unknowns) =+      NextStep (maybe "Review unknown transactions" (\n -> "Categorise " <> plural n "unknown transaction")+        (reviewCount (reviewUnknowns review))) Nothing+        (map (\path -> "Review " <> T.pack path <> ".") (take 1 unknowns) +++          ["Add if rules to the relevant account's rules file, then run: " <> importCommand] +++          ["Or start with transfers: " <> T.pack path | path : _ <- [transfers]])+        (Just (docURL ReviewUnknownTransactions))+  | not (null transfers) =+      NextStep ("Check transfers in " <> plural (length years) "year") Nothing+        (["Review " <> T.pack path <> "." | path : _ <- [transfers]] +++          ["A transfer may need review when the matching statement hasn't arrived yet."])+        (Just (docURL ReviewTransfers))+  | otherwise = firstStatement {nsSummary = "Everything is imported and reported"}+  where+    unknowns = reviewFiles (reviewUnknowns review)+    transfers = reviewFiles (reviewTransfers review)+    years = nub [itemYear item | item <- reviewItems review, itemKind item == TransferBalances]++flowCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Maybe HledgerConf -> T.Text -> [T.Text] -> T.Text+flowCommand shell homes cwd base conf subcommand arguments =+  T.unwords (["hledger-flow"] ++ config ++ [subcommand] ++ map (shellArgument shell) arguments ++ directory)+  where+    config = case conf of+      Just (GivenConf file) | subcommand `elem` ["import", "report", "rules"] ->+        ["--hledger-conf", shellPath shell homes file]+      _ -> []+    directory = [shellPath shell homes (FilePath.dropTrailingPathSeparator (toFilePath base)) | cwd /= base]++shellPath :: Shell -> [FilePath] -> FilePath -> T.Text+shellPath shell homes file =+  fromMaybe (shellArgument shell (T.pack file)) (homeShellPath shell homes file)++setLedgerFileCommand :: Shell -> [FilePath] -> LedgerStatus -> T.Text+setLedgerFileCommand shell homes s = case shell of+  Posix -> "export LEDGER_FILE=" <> setting posixLiteral+  PowerShell -> "$env:LEDGER_FILE = " <> setting powerShellLiteral+  where+    path = toFilePath (lsBaseDir s </> [relfile|all-years.journal|])+    setting literal = fromMaybe (literal (T.pack path)) (homeShellPath shell homes path)++balanceHint :: Shell -> [FilePath] -> LedgerStatus -> Maybe T.Text+balanceHint shell homes s = do+  case firstConfigConflict BalanceConfig (lsConfigConflicts s) of+    Just _ -> Nothing+    Nothing -> Just ()+  info <- lsHledger s+  journal <- lsAggregateJournal s+  let config = case lsHledgerConf s of+        Just (GivenConf configPath) -> ["--conf", shellPath shell homes configPath]+        _ -> []+      path = T.unpack (hledgerFileArgument info (toFilePath journal))+  return (T.unwords (["hledger"] ++ config ++ ["-f", shellPath shell homes path, "balance"]))++ledgerFileNote :: Shell -> [FilePath] -> LedgerStatus -> Maybe T.Text+ledgerFileNote shell homes s = do+  _ <- lsAggregateJournal s+  message <- case lsLedgerFile s of+    Nothing -> Just "LEDGER_FILE isn't set, so hledger won't find this ledger by default."+    Just file | lfExists file && lfCoverage file /= CoversLedger -> Just (ledgerFileProblem homes file)+    _ -> Nothing+  return (T.intercalate "\n" ([message, "hledger-flow report uses all-years.journal.", "For hledger, run: " <> setLedgerFileCommand shell homes s]+      ++ ["Or try: " <> hint | Just hint <- [balanceHint shell homes s]]))++withConfigCheck :: ConfigUse -> LedgerStatus -> NextStep -> NextStep+withConfigCheck use status next = case firstConfigConflict use (lsConfigConflicts status) of+  Just problem -> NextStep "Fix hledger.conf before continuing" Nothing+    (T.lines (renderConfigConflict problem)) (Just (docURL StatusCommand))+  Nothing -> next
src/Hledger/Flow/Status/Gather.hs view
@@ -5,14 +5,17 @@ module Hledger.Flow.Status.Gather   ( classifyDir,     gatherStatus,+    gatherStatusWithConfig,     accountName,     cachedStatement,     hookExists,+    mainRulesCandidates,   ) where  import Control.Applicative ((<|>)) import Control.Monad (forM, when)+import Data.Char (toLower) import qualified Data.List as List import qualified Data.Map.Strict as Map import qualified Data.Set as Set@@ -21,17 +24,23 @@ import qualified Data.Text.IO as T import Data.Time.Clock (UTCTime) import Hledger.Flow.BaseDir (findBaseDir, relativeToBase')-import Hledger.Flow.Common (configuredLedgerFile, firstExistingFile, groupValuesBy)+import Hledger.Flow.Common (firstExistingFile, groupValuesBy, hledgerFileArgument, runProcessCaptured)+import Hledger.Flow.HledgerConfig (findConfigConflicts) import Hledger.Flow.Import.ImportHelpers (defaultRulesCandidates, extractImportDirs, rulesFileCandidates) import Hledger.Flow.Import.IncludeFiles (includesNeedUpdate) import Hledger.Flow.Import.Inventory (classifyGeneratedOutputs, findImportFiles, generatedOutput, isManualJournal, managedIncludeFiles, stageFiles) import Hledger.Flow.Import.Types (ImportDirs (..))-import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelFile)+import Hledger.Flow.Init.Example (directivesAreStarter, journalIsCommentsOnly, sampleAccountDir, starterOwner, untouchedExample)+import Hledger.Flow.LedgerSelection+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelFile, pathOccupied)+import Hledger.Flow.ReportReview (loadReportReview, reviewItems)+import Hledger.Flow.StatementScan (StatementYear (..), inferStatementYear, isDelimitedFile, readDelimitedStatement) import Hledger.Flow.Status-import Hledger.Flow.Types (BaseDir, HledgerInfo)+import Hledger.Flow.Types (BaseDir, HledgerConf, HledgerInfo (..)) import Path import qualified Path.IO as PathIO import qualified System.Directory as Directory+import System.Exit (ExitCode (..)) import qualified System.FilePath as FilePath import qualified System.Info as Sys import System.IO.Error (tryIOError)@@ -48,32 +57,107 @@       return (if all ("." `List.isPrefixOf`) names then EmptyDir else Elsewhere)  gatherStatus :: Maybe HledgerInfo -> BaseDir -> IO LedgerStatus-gatherStatus hledger base = do+gatherStatus hledger = gatherStatusWithConfig hledger Nothing++gatherStatusWithConfig :: Maybe HledgerInfo -> Maybe HledgerConf -> BaseDir -> IO LedgerStatus+gatherStatusWithConfig hledger conf base = do   let root = base </> [reldir|import|]   files <- findImportFiles root   let statements = stageFiles root [reldir|1-in|] files+      manualJournals = filter (isManualJournal root) files       groups = Map.toAscList (groupValuesBy (accountName root) statements)   accounts <- forM groups (uncurry (accountStatus base))-  configured <- configuredLedgerFile-  ledgerFile <- forM configured $ \(value, path) -> LedgerFile value <$> Directory.doesFileExist path-  allYears <- PathIO.doesFileExist (base </> [relfile|all-years.journal|])+  conflicts <- maybe (pure []) (\hl -> findConfigConflicts hl conf) hledger+  selection <- selectLedger [] hledger base+  let defaultJournal = base </> [relfile|all-years.journal|]+  allYears <- PathIO.doesFileExist defaultJournal+  let selected = aggregateJournal selection   (orphans, journals) <- classifyGeneratedOutputs root files statements   -- A newly initialised, empty ledger has no indexes to reconcile yet.-  reconciliation <- if allYears && null orphans-    then includesNeedUpdate base root files journals+  reconciliation <- if (allYears || isJust selected) && null orphans+    then includesNeedUpdate base root files journals allYears     else return (not (null orphans) || not (null (managedIncludeFiles root files)))+  needsPrices <- case (hledger, selected) of+    (Just info, Just journal) | not reconciliation -> journalNeedsPrices info journal+    _ -> return False+  review <- maybe (pure Nothing) (\journal -> loadReportReview base journal conf) selected+  owner <- starterOwner+  examplePresent <- either (const True) id <$> tryIOError (pathOccupied (sampleAccountDir base owner))+  let personal = [reldir|personal|]+      alternate = if FilePath.equalFilePath (toFilePath owner) (toFilePath personal) then [reldir|me|] else personal+      firstOwner = if examplePresent then alternate else owner+  example <- case review of+    Just current | null (reviewItems current) -> untouchedExample base statements+    _ -> return Nothing+  manualWork <- case manualJournals of+    [] -> pure False+    -- A wrapper can contribute journal entries of its own.+    _ | not (null statements) || maybe False (/= defaultJournal) selected -> pure True+    _ -> either (const True) not <$> tryIOError (onlyStarterJournals base manualJournals)+  misplaced <- traverse (inspectMisplaced base) (List.sortOn mpPath (misplacedFiles root files))   return     LedgerStatus       { lsBaseDir = base,         lsHledger = hledger,-        lsLedgerFile = ledgerFile,+        lsLedgerFile = selectedLedgerFile selection,+        lsAggregateJournal = selected,+        lsHledgerConf = conf,+        lsConfigConflicts = conflicts,         lsAccounts = accounts,-        lsMisplaced = List.sortOn mpPath (misplacedFiles root files),-        lsManualJournals = length (filter (isManualJournal root) files),+        lsMisplaced = misplaced,+        lsManualJournals = length manualJournals,+        lsHasManualWork = manualWork,         lsAllYearsExists = allYears,-        lsNeedsReconciliation = reconciliation+        lsNeedsReconciliation = reconciliation,+        lsNeedsPrices = needsPrices,+        lsReview = review,+        lsFirstStatementOwner = firstOwner,+        lsUntouchedExample = example       } +-- The starter ledger contains declarations and comments only.+onlyStarterJournals :: BaseDir -> [AbsFile] -> IO Bool+onlyStarterJournals base [] = directivesAreStarter base+onlyStarterJournals base (file : rest) = do+  commentOnly <- journalIsCommentsOnly file+  if commentOnly then onlyStarterJournals base rest else return False++inspectMisplaced :: BaseDir -> Misplaced -> IO Misplaced+inspectMisplaced base item = do+  evidence <- case mpYear item of+    UnknownYear | isDelimitedFile (mpPath item), Just path <- parseRelFile (mpPath item) -> do+      scanned <- tryIOError (readDelimitedStatement Nothing (base </> path))+      return $ case scanned of+        Right (Right statement) -> inferStatementYear statement+        _ -> UnknownYear+    known -> return known+  let suggestion = case evidence of+        SingleYear year -> FilePath.joinPath+          [if part == "<year>" then show year else part | part <- FilePath.splitDirectories (mpSuggestion item)]+        _ -> mpSuggestion item+      enriched = item {mpYear = evidence, mpSuggestion = suggestion}+  occupied <- case placementDestination enriched of+    Nothing -> return False+    Just path -> either (const True) id <$> tryIOError (pathOccupied (base </> path))+  return enriched {mpDestinationExists = occupied}++-- Ask hledger to resolve includes and parse the journal. If it cannot read+-- the journal, status can still report what it found in the import tree.+journalNeedsPrices :: HledgerInfo -> AbsFile -> IO Bool+journalNeedsPrices info journal = do+  commodities <- query ["commodities", "--used"]+  case T.lines <$> commodities of+    Just (_ : _ : _) -> (== Just "") . fmap T.strip <$> query ["prices"]+    _ -> return False+  where+    query args = do+      let executable = T.pack (toFilePath (hlPath info))+          fileArgs = ["--no-conf", "--file", hledgerFileArgument info (toFilePath journal)]+      result <- tryIOError (runProcessCaptured (executable, args ++ fileArgs, Nothing))+      return $ case result of+        Right (ExitSuccess, output, _) -> Just output+        _ -> Nothing+ accountName :: AbsDir -> AbsFile -> T.Text accountName root file = case stripProperPrefix root file of   Just rel -> T.intercalate "/" (map T.pack (take 3 (FilePath.splitDirectories (toFilePath rel))))@@ -92,15 +176,25 @@   directives <- firstExistingFile [base </> [relfile|directives.journal|]]   sources <- forM statements $ \s -> do     cached <- cachedStatement s-    rules <- rulesFileCandidates (fromMaybe s cached) dirs >>= firstExistingFile-    return (s, cached, rules)-  let resolved = [rules | (_, _, rules) <- sources]+    candidates <- rulesFileCandidates (fromMaybe s cached) dirs+    rules <- firstExistingFile candidates+    return (s, cached, rules, candidates)+  let resolved = [rules | (_, _, rules, _) <- sources]+      withoutRules = length (filter isNothing resolved)+  (otherRules, targetOccupied) <- if withoutRules > 0 && isNothing construct+    then do+      other <- tryIOError (mainRulesCandidates (accountDir dirs) (concat [xs | (_, _, _, xs) <- sources]))+      occupied <- case defaults of+        target : _ -> either (const True) id <$> tryIOError (pathOccupied target)+        _ -> return False+      return (either (const []) id other, occupied)+    else return ([], False)   -- Statements in an account commonly share one rules file.   ruleTimes <- case construct of     Just _ -> return Map.empty     Nothing -> traverse (tryIOError . latestRuleChange) $       Map.fromSet id (Set.fromList (catMaybes resolved))-  journals <- forM sources $ \(s, cached, rules) -> do+  journals <- forM sources $ \(s, cached, rules, _) -> do     let inputs = s : catMaybes [cached, preprocess, construct <|> directives]     journal <- generatedOutput [reldir|3-journal|] ".journal" s     journalState inputs (rules >>= (`Map.lookup` ruleTimes)) journal@@ -111,7 +205,9 @@         asYears = List.nub (List.sort (map yearOf statements)),         asRulesFile = relative <$> rulesFile,         asRulesCandidates = map relative defaults,-        asWithoutRules = length (filter isNothing resolved),+        asWithoutRules = withoutRules,+        asOtherRules = map relative otherRules,+        asRulesTargetOccupied = targetOccupied,         asPreprocess = isJust preprocess,         asConstruct = isJust construct,         asJournalsMissing = length (filter (== Missing) journals),@@ -130,12 +226,27 @@       key <- Directory.canonicalizePath file       when (Set.member key ancestors) $ ioError (userError ("cyclic rules include: " ++ file))       modified <- Directory.getModificationTime file-      contents <- T.readFile file-      let children = [FilePath.takeDirectory file FilePath.</> T.unpack (T.strip path)-                     | line <- T.lines (T.dropWhile (== '\xFEFF') contents)-                     , Just path <- [T.stripPrefix "include " line]]+      children <- ruleIncludes file       times <- mapM (visit (Set.insert key ancestors)) children       return (maximum (modified : times))++ruleIncludes :: FilePath -> IO [FilePath]+ruleIncludes file = do+  contents <- T.readFile file+  return [FilePath.takeDirectory file FilePath.</> T.unpack (T.strip path)+         | line <- T.lines (T.dropWhile (== '\xFEFF') contents),+           Just path <- [T.stripPrefix "include " line]]++mainRulesCandidates :: AbsDir -> [AbsFile] -> IO [AbsFile]+mainRulesCandidates dir recognized = do+  (_, files) <- PathIO.listDir dir+  let local = List.sort [file | file <- files, map toLower (FilePath.takeExtension (toFilePath file)) == ".rules"]+      canonical = Directory.canonicalizePath . toFilePath+  recognizedTargets <- Set.fromList <$> traverse canonical recognized+  children <- concat <$> traverse (ruleIncludes . toFilePath) local+  childTargets <- Set.fromList <$> traverse Directory.canonicalizePath children+  pairs <- traverse (\file -> (,) file <$> canonical file) local+  return [file | (file, target) <- pairs, Set.notMember target (Set.union recognizedTargets childTargets)]  journalState :: [AbsFile] -> Maybe (Either IOError UTCTime) -> AbsFile -> IO JournalState journalState inputs rulesTime journal = do
src/Hledger/Flow/Status/Render.hs view
@@ -22,11 +22,15 @@ import Hledger.Flow.DateTime (currentYear) import Hledger.Flow.Environment (coresSetting, environmentJSON, environmentSettings, renderSettings) import Hledger.Flow.Internals (SystemInfo, systemInfo)-import Hledger.Flow.Status.Gather (gatherStatus)+import Hledger.Flow.Status.Gather (gatherStatusWithConfig)+import Hledger.Flow.PathHelpers (AbsDir) import Hledger.Flow.Types (BaseDir) import Path.IO (getCurrentDir) import qualified Data.Text as T import Hledger.Flow.Status+import Hledger.Flow.ReportReview+import Hledger.Flow.StatementScan (StatementYear (..))+import Hledger.Flow.LedgerSelection (Coverage (..)) import Hledger.Flow.Types (HledgerConf, HledgerInfo (..)) import Path (toFilePath) import Hledger.Flow.Logging (bold, green, red, useColor, yellow)@@ -39,7 +43,7 @@  renderStatus :: Bool -> [FilePath] -> LedgerStatus -> NextStep -> T.Text renderStatus color homes s next =-  T.unlines (header : accountBlock ++ misplacedBlock) <> renderNext color next+  T.unlines (header : accountBlock ++ misplacedBlock ++ noteBlock ++ reviewBlock) <> renderNext color next   where     header = bold color "Ledger: " <> T.pack (tildePath homes (baseText s)) <> "   (" <> hledgerText <> ")"     hledgerText = case lsHledger s of@@ -69,6 +73,21 @@                   ++ [tshow outdated <> " outdated" | outdated > 0]               txt = T.intercalate ", " parts            in if missing + outdated > 0 then yellow color ("• " <> txt) else green color ("✓ " <> txt)+    noteBlock = case T.lines <$> ledgerFileNote nativeShell homes s of+      Just (first : rest) -> "" : ("  Note: " <> first) : map ("        " <>) rest+      _ -> []+    reviewBlock = case maybe [] reviewItems (lsReview s) of+      [] -> []+      items -> "" : "  To review:" : map showReview (take 5 items)+        ++ ["    and " <> tshow (length items - 5) <> " more" | length items > 5]+    showReview item =+      let label = case itemKind item of+            UnknownTransactions -> "unknown transaction"+            TransferBalances -> "transfer account to check"+          count = case itemKind item of+            TransferBalances -> fmap (\n -> plural n "transfer account" <> " to check") (itemCount item)+            UnknownTransactions -> fmap (\n -> plural n label) (itemCount item)+       in "    " <> maybe "count unavailable" id count <> "  " <> T.pack (itemPath item)     misplacedBlock = case length (lsMisplaced s) of       0 -> []       n -> ["", "  " <> yellow color ("⚠ " <> plural n "file" <> " not where hledger-flow looks")]@@ -83,16 +102,20 @@ tshow :: Show a => a -> T.Text tshow = T.pack . show -statusJSON :: SystemInfo -> Maybe HledgerConf -> LedgerStatus -> NextStep -> Value-statusJSON sys conf s next =+statusJSON :: SystemInfo -> Maybe HledgerConf -> AbsDir -> LedgerStatus -> NextStep -> Value+statusJSON sys conf cwd s next =   object     [ "schemaVersion" .= (1 :: Int),       "environment" .= environmentJSON sys conf,       "baseDir" .= baseText s,       "hledger" .= fmap hledgerJSON (lsHledger s),-      "ledgerFile" .= fmap (\lf -> object ["value" .= lfValue lf, "exists" .= lfExists lf]) (lsLedgerFile s),+      "ledgerFile" .= fmap (\lf -> object ["value" .= lfValue lf, "exists" .= lfExists lf, "coversLedger" .= (lfCoverage lf == CoversLedger)]) (lsLedgerFile s),+      "aggregateJournal" .= fmap toFilePath (lsAggregateJournal s),+      "ledgerFileNote" .= ledgerFileNote nativeShell [] s,       "accounts" .= map accountJSON (lsAccounts s),-      "misplaced" .= [object ["path" .= mpPath m, "suggestion" .= mpSuggestion m] | m <- lsMisplaced s],+      "misplaced" .= map misplacedJSON (lsMisplaced s),+      "reportsCurrent" .= maybe False (const True) (lsReview s),+      "review" .= fmap reviewJSON (lsReview s),       "manualJournals" .= lsManualJournals s,       "allYearsJournalExists" .= lsAllYearsExists s,       "next"@@ -104,6 +127,17 @@           ]     ]   where+    reviewJSON review = object ["unknownTransactions" .= category (reviewUnknowns review),+                                "transferAccounts" .= category (reviewTransfers review)]+    category review = object ["count" .= reviewCount review, "files" .= reviewFiles review]+    misplacedJSON item = object+      [ "path" .= mpPath item,+        "suggestion" .= mpSuggestion item,+        "year" .= (case mpYear item of SingleYear year -> Just year; _ -> Nothing),+        "years" .= (case mpYear item of SpanningYears years -> years; _ -> []),+        "destinationExists" .= mpDestinationExists item,+        "command" .= misplacedMoveCommand nativeShell [] cwd (lsBaseDir s) item+      ]     hledgerJSON hl = object ["path" .= toFilePath (hlPath hl), "version" .= hledgerVersionNumber (hlVersion hl)]     accountJSON a =       object@@ -113,12 +147,14 @@           "rulesFile" .= asRulesFile a,           "rulesCandidates" .= asRulesCandidates a,           "statementsWithoutRules" .= asWithoutRules a,+          "otherRules" .= asOtherRules a,+          "rulesTargetOccupied" .= asRulesTargetOccupied a,           "hooks" .= object ["preprocess" .= asPreprocess a, "construct" .= asConstruct a],           "journalsMissing" .= asJournalsMissing a,           "journalsOutdated" .= asJournalsOutdated a         ] --- | The next step for people to read with @homes@, or for scripts with @[]@ (absolute paths).+-- | The next step for people to read with @homes@, or for scripts with @[]@ (no home abbreviation). nextStepHere :: [FilePath] -> LedgerStatus -> IO NextStep nextStepHere homes status = do   year <- currentYear@@ -137,23 +173,23 @@ printStatus hledgerPath confOpt json base = do   hledger <- findHledger hledgerPath   conf <- findHledgerConf confOpt-  status <- gatherStatus hledger base+  status <- gatherStatusWithConfig hledger conf base   sys <- systemInfo+  cwd <- getCurrentDir+  year <- currentYear+  let next homes = nextStep year nativeShell homes cwd status   if json-    then do-      next <- nextStepHere [] status-      TL.putStrLn (encodeToLazyText (statusJSON sys conf status next))+    then TL.putStrLn (encodeToLazyText (statusJSON sys conf cwd status (next [])))     else do       color <- useColor stdout       homes <- homeDirs-      next <- nextStepHere homes status-      T.putStr (renderStatus color homes status next <> renderEnvironment color homes sys conf status)+      T.putStr (renderStatus color homes status (next homes) <> renderEnvironment color homes sys conf status)  -- | Print only the next step for a ledger.-printNext :: Maybe FilePath -> BaseDir -> IO ()-printNext hledgerPath base = do+printNext :: Maybe FilePath -> Maybe HledgerConf -> BaseDir -> IO ()+printNext hledgerPath conf base = do   hledger <- findHledger hledgerPath-  status <- gatherStatus hledger base+  status <- gatherStatusWithConfig hledger conf base   homes <- homeDirs   next <- nextStepHere homes status   color <- useColor stdout
test/CSVImport/IncludeFilesTests.hs view
@@ -17,6 +17,7 @@ testIncludeYears =   TestCase     ( do+        assertEqual "An empty generated index has no report years" (Right []) (includeYears' (includePreamble <> "\n"))         let txterr = "Some text without years"         let expectederr = ["Unable to extract years from the following text:", txterr, "Errors:"]         let actualerr = case lefts [includeYears' txterr] of
test/Common/Integration.hs view
@@ -13,6 +13,7 @@ import Data.Time.Clock (UTCTime(..), addUTCTime, secondsToDiffTime) import Data.Time.Calendar (fromGregorian) import Hledger.Flow.Common+import Hledger.Flow.PathHelpers (writeNewUtf8File) import qualified Hledger.Flow.RuntimeOptions as RT import Hledger.Flow.Types (HledgerConf (..), LogMessage (StdErr)) import qualified Path@@ -96,7 +97,8 @@   assertEqual "no existing files" Nothing absent  testVerboseTestFile :: Test-testVerboseTestFile = TestCase $ withTestDir $ \base -> do+-- Pin colour off: stderr is a terminal when the suite runs interactively.+testVerboseTestFile = TestCase $ withEnv "NO_COLOR" "1" $ withTestDir $ \base -> do   let opts = (defaultOpts base) {RT.verbose = True}       file = base Path.</> [Path.relfile|statement.csv|]   ch <- newTChanIO@@ -247,5 +249,15 @@   entries <- Directory.listDirectory dir   assertBool ("no staged files left: " ++ show entries) (not (any (List.isPrefixOf ".") entries)) +testWriteNewFileAtLongPath :: Test+testWriteNewFileAtLongPath = TestLabel "create a file at a long path without replacing it" $ TestCase $ withTestDir $ \base -> do+  name <- Path.parseRelFile (replicate 100 'a' </> replicate 100 'b' </> replicate 100 'c' </> "starter.rules")+  let target = base Path.</> name+  writeNewUtf8File target "first\n"+  second <- tryIOError (writeNewUtf8File target "second\n")+  case second of+    Left _ -> T.readFile (Path.toFilePath target) >>= assertEqual "preserve the first write" "first\n"+    Right () -> assertFailure "an existing file was replaced"+ tests :: Test-tests = TestList [testWriteGeneratedText, testHledgerLookup, testFindHledgerConf, testHomeDirs, testFilterPaths, testFirstExistingFile, testVerboseTestFile, testNeedsRegeneration, testBatchOrder, testBatchFailure]+tests = TestList [testWriteGeneratedText, testWriteNewFileAtLongPath, testHledgerLookup, testFindHledgerConf, testHomeDirs, testFilterPaths, testFirstExistingFile, testVerboseTestFile, testNeedsRegeneration, testBatchOrder, testBatchFailure]
+ test/HledgerConfig/Unit.hs view
@@ -0,0 +1,30 @@+{-# LANGUAGE OverloadedStrings #-}++module HledgerConfig.Unit (tests) where++import Data.Maybe (isJust, isNothing)+import qualified Data.Text as T+import Hledger.Flow.HledgerConfig+import Test.HUnit++fromText :: [T.Text] -> [ConfigConflict]+fromText = configConflicts "hledger.conf" . parseConfigLines True++tests :: Test+tests = "hledger config sources" ~: TestList+  [ TestList [TestCase $ do+      let problem = firstConfigConflict ImportConfig (fromText ["# comment", "[print]", input])+      fmap (T.takeWhile (/= '\n') . renderConfigConflict) problem @?=+        Just ("hledger.conf line 3 [print] adds a journal: " <> input)+    | input <- ["-f other", "-fother", "--file other", "--file=other"]],+    TestList [isNothing (firstConfigConflict ReportConfig (fromText ["[general]", option <> " \"-f=value\""])) ~?= True+    | option <- ["--alias", "--pivot", "-b", "--begin", "-e", "--end", "-p", "--period",+                 "--depth", "-o", "--output-file", "-O", "--output-format", "--rules",+                 "--rules-file", "--today", "--value", "--conf"]],+    isJust (firstConfigConflict ReportConfig (fromText ["[balancesheet]", "--flat -f other"])) ~?= True,+    isNothing (firstConfigConflict ImportConfig (fromText ["[balance]", "--file other"])) ~?= True,+    isNothing (firstConfigConflict ReportConfig (fromText ["[files]", "--file other"])) ~?= True,+    isNothing (firstConfigConflict ReportConfig (fromText ["# -f file", "--pretty # -f file"])) ~?= True,+    isNothing (firstConfigConflict BalanceConfig (fromText ["-- --file other"])) ~?= True,+    isJust (firstConfigConflict ReportConfig (fromText ["  [balance] @arguments"])) ~?= True+  ]
test/ImportHelpers/Integration.hs view
@@ -110,14 +110,16 @@ testTrashLock :: Test testTrashLock = TestCase $ withSystemTempDir "hlflow-trash" $ \root -> do   let source = root </> [relfile|import/owner/bank/account/3-journal/2020/march.journal|]-      lock = root </> [reldir|import/owner/bank/account/.trash/.move-lock|]-      target = root </> [relfile|import/owner/bank/account/.trash/2020/3-journal/march.journal|]+      lock = root </> [reldir|.hledger-flow/trash/.move-lock|]+      target = root </> [relfile|.hledger-flow/trash/import/owner/bank/account/3-journal/2020/march.journal|]+      ignore = root </> [relfile|.hledger-flow/trash/.gitignore|]   touchFile source   createDirIfMissing True lock-  result <- tryIOError (moveToTrash source)+  result <- tryIOError (moveToTrash root source)   assertBool "A concurrent trash move must not race for a destination" (isLeft result)   T.readFile (toFilePath source) >>= assertEqual "Busy trash leaves source intact" "x"   doesFileExist target >>= assertBool "Busy trash writes no destination" . not   removeDir lock-  actual <- moveToTrash source+  actual <- moveToTrash root source   assertEqual "Retry after the other move completes succeeds" target actual+  T.readFile (toFilePath ignore) >>= assertEqual "Git ignores the trash" "*\n"
test/Incremental/Dependencies.hs view
@@ -97,19 +97,28 @@           no release s = fallback (checkConfig release file s)       forM_ [V1524,V1996,OtherSupported] $ \release -> do         ok release "# comment\n[balance] --period=thismonth\n[register]\n@file\n[general]\n[print]\n"-        forM_ ["--period=thismonth", "[general] --debug", "[print]\n@indirect", "[txns]", "[bal]", "[b]", "[alias]", "[missing", "[general", "[balance]\n[print] --period=thismonth"] (no release)-      ok V1996 "[print] --layout=hledger1 # layout\n"-      ok OtherSupported "[print] --layout=hledger1 # layout\n"-      no OtherSupported "[print]\n [balance]\n--empty"-      no OtherSupported "[holdings]"-      no OtherSupported "[demo]"-      no V1524 "[print]\n--layout=hledger1"-      no V1996 "[general]\n--layout=hledger1"-      ok V1996 "[holdings]"-      no V1524 "[print]\n [balance]\n--empty"-      no V1524 "[holdings]"-      ok V1524 "[demo]"-      no V1996 "[demo]"+        ok release "--invert\n[print] -x --round soft --layout=55 # literal values\n"+        forM_ ["not:archive", "--layout=hledger1 not:archive", "--unknown", "--round",+               "--alias /old/=new", "--begin=2025-01-01", "--period=2025", "date:2025..2026",+               "--forecast=2025", "--forecast=2025..2026", "--value=then", "--value=2025-06-30,EUR",+               "--today=2026-06-30\ndate:thismonth", "--today 2026-06-30 -V", "-B", "desc:\"in 30 days\"",+               "desc:today", "acct:next", "--period=\"monthly in 2025\"", "desc:\"a # b\"", "--round=soft # date:today"] $+          ok release . ("[print]\n" <>)+        forM_ ["[general] --debug", "[txns]", "[bal]", "[b]", "[alias]", "[missing", "[general",+               "[future-command] --period=thismonth", "[balance] date:today"] (ok release)+        forM_ ["--period=thismonth", "--begin\ntoday", "-p 'this month'", "-pthismonth", "--begin today", "-etomorrow",+               "date:today", "not:date:thisyear", "date2:'last month'", "--period=1/1..4/1",+               "--forecast", "--forecast=thismonth", "--forecast=2025..", "--value=now", "--exchange=EUR,now", "-XEUR,now",+               "-V", "-X USD", "--value=end", "--value=end,EUR", "date:'-7 days..'", "-b '2 weeks ago'",+               "expr:'tag:recurring and date:thismonth'", "expr:'not:(date:thisyear)'", "--today=2026-01-01\n--forecast"] $ \setting ->+          case checkConfig release file ("[print]\n" <> setting) of+            Fallback reason -> reasonScope reason @?= "non-deterministic hledger config entry"+            _ -> assertFailure ("expected clock dependency: " ++ show setting)+        forM_ ["--file=extra.journal", "-fextra.journal", "--rules-file extra.rules", "@indirect",+               "--new", "--output-file saved.txt", "-osaved.txt", "--locations"] $+          no release . ("[print]\n" <>)+      ok V1996 "[print]\n [balance]\n--period=thismonth"+      no V1524 "[print]\n [balance]\n--period=thismonth"       forM_ ["[balance]\n--period=thismonth", "[balance]\n--period=thisyear"] $ \bytes -> do         B.writeFile (toFilePath file) bytes         discoverConfig V1996 (Just (GivenConf (toFilePath file))) >>= (@?= Reusable (Just (file,digestBytes bytes)))@@ -123,9 +132,35 @@       )   , "directives eligibility" ~: withTestDir (\base -> do       let file = base </> [relfile|directives.journal|]-          check = checkDirectives V1996 file . T.encodeUtf8-      forM_ ["account assets:bank:current  ; type: C","commodity 1,000.00000000 BTC  ; lots:","commodity €1,000.00","P 2000-01-01 EUR €1","P 2000/01/01 BTC 12.30 EUR","; tags\n# prices\n* prices\n\n"] $ \s -> check s @?= Reusable ()+          check = journalIncludes file . T.encodeUtf8+      forM_ ["account assets:bank:current  ; type: C","commodity 1,000.00000000 BTC  ; lots:","commodity €1,000.00","P 2000-01-01 EUR €1","P 2000/01/01 BTC 12.30 EUR","; tags\n# prices\n* prices\n\n"] $ \s -> check s @?= Reusable []+      check "include accounts.journal  ; declarations\ninclude\ttax/vat codes.journal" @?= Reusable ["accounts.journal","tax/vat codes.journal"]+      check "include vat-codes\ninclude vat.CUSTOM\ninclude journal:vat.CSV" @?= Reusable ["vat-codes","vat.CUSTOM","vat.CSV"]+      forM_ ["", "journal:", "vat.CsV", "vat.tsv", "vat.ssv", "vat.rules", "vat.timedot", "vat.timeclock"] $+        fallback . check . ("include " <>)       forM_ ["account assets:any\n    note", "comment\ncommodity €1\nend comment","include *.journal","P today EUR €1","P 2000-01-01 EUR €1 trailing","commodity 1,234 ABC extra", "2025/01/01 A"] (fallback . check))+  , "unmirrorable journal includes" ~: withTestDir (\tmp -> do+      let base = tmp </> [reldir|ledger|]+          root = base </> [relfile|directives.journal|]+          nested = base </> [relfile|sub/accounts.journal|]+          write path = B.writeFile (toFilePath path)+      P.createDirIfMissing True (parent nested)+      write root "include sub/accounts.journal\n"+      write nested "include ../directives.journal\n"+      discoverDirectives root >>= fallback+      write nested "include ../../outside.journal\n"+      write (tmp </> [relfile|outside.journal|]) "account expenses:cafe\n"+      discoverDirectives root >>= fallback+#ifndef mingw32_HOST_OS+      let link = base </> [relfile|alias.journal|]+      D.createFileLink (toFilePath nested) (toFilePath link)+      write root "include alias.journal\n"+      write nested "include prices.journal\n"+      write (base </> [relfile|prices.journal|]) "; different directory\n"+      write (parent nested </> [relfile|prices.journal|]) "; canonical directory\n"+      discoverDirectives root >>= fallback+#endif+      )   , "converter identity" ~: bracket getLocaleEncoding setLocaleEncoding (\_ -> withTestDir (\base -> do       setLocaleEncoding utf8       let real = base </> [relfile|actual|]
test/Incremental/Snapshot.hs view
@@ -13,7 +13,6 @@ import GHC.IO.Encoding (getLocaleEncoding, setLocaleEncoding, utf8) import qualified Data.ByteString as BS import qualified Data.ByteString.Char8 as B8-import qualified Data.Map.Strict as Map import qualified Data.Text as T import Hledger.Flow.Import.Cache import Hledger.Flow.Import.Dependencies@@ -50,18 +49,24 @@           rules = base </> [relfile|account/bank.rules|]           category = base </> [relfile|shared/coffee.rules|]           directives = base </> [relfile|directives.journal|]+          accounts = base </> [relfile|shared/accounts.journal|]           a = "account2 expenses:cafe\n"       P.createDirIfMissing True (parent statement)       P.createDir (parent category)       BS.writeFile (toFilePath statement) "2026-01-01,Coffee,3\n"       BS.writeFile (toFilePath rules) "include ../shared/coffee.rules\n"       BS.writeFile (toFilePath category) a-      BS.writeFile (toFilePath directives) "account expenses:cafe\n"+      BS.writeFile (toFilePath directives) "include shared/accounts.journal\n"+      BS.writeFile (toFilePath accounts) "account expenses:cafe\n"       result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (RulesInput statement rules (Just directives) Nothing) $ \snapshot -> do         RulesInput dataCopy rulesCopy (Just directivesCopy) Nothing <- pure (snapshotInputs snapshot)-        let deps = snapshotManifest snapshot-        assertEqual "manifest includes nested rule" (Just (digestBytes a)) (Map.lookup category deps)-        assertEqual "all four files" 4 (Map.size deps)+        let expected = [statement, rules, category, directives, accounts]+        digests <- mapM digestFile expected+        assertEqual "manifest follows the order of conversion inputs"+          (zip expected digests) (snapshotManifest snapshot)+        BS.writeFile (toFilePath accounts) "account expenses:food\n"+        BS.readFile (toFilePath (parent directivesCopy </> [relfile|shared/accounts.journal|]))+          >>= assertEqual "private copy retains original declarations" "account expenses:cafe\n"         assertEqual "CSV basename" (filename statement) (filename dataCopy)         canonicalCopy <- D.canonicalizePath (toFilePath rulesCopy)         let location = ":1:2:\n"
test/Init/Integration.hs view
@@ -2,21 +2,28 @@  module Init.Integration (tests) where +import Hledger.Flow.DateTime (currentYear) import Hledger.Flow.Init (InitOutcome (..), initLedger)+import Hledger.Flow.Init.Example (exampleAccount, exampleBaseFiles, exampleImportFiles, starterOwner) import Path-import Path.IO (doesDirExist)+import Path.IO (doesDirExist, doesFileExist) import Test.HUnit hiding (Path) import TestHelpers (withTestDir)  testInit :: Test testInit = TestCase $ withTestDir $ \tmp -> do   let fresh = tmp </> [reldir|fin|]-  (@?= Created fresh Nothing) =<< initLedger fresh+  year <- currentYear+  owner <- starterOwner+  let created = [(name, True) | (name, _) <- exampleBaseFiles year]+  (@?= Created fresh Nothing (exampleAccount owner) created) =<< initLedger fresh+  mapM_ (\(file, _) -> doesFileExist (fresh </> [reldir|import|] </> file) >>= assertBool (toFilePath file))+    (exampleImportFiles owner year)   doesDirExist (fresh </> [reldir|import|]) >>= assertBool "import/ created"   (@?= AlreadyLedger fresh) =<< initLedger fresh   (@?= InsideLedger fresh) =<< initLedger (fresh </> [reldir|import/me|])   let nested = fresh </> [reldir|elsewhere|]-  (@?= Created nested (Just fresh)) =<< initLedger nested+  (@?= Created nested (Just fresh) (exampleAccount owner) created) =<< initLedger nested  tests :: Test tests = TestList [testInit]
+ test/LedgerSelection/Unit.hs view
@@ -0,0 +1,14 @@+{-# LANGUAGE OverloadedStrings #-}++module LedgerSelection.Unit (tests) where++import Hledger.Flow.LedgerSelection (Coverage (..), coverageFor)+import Test.HUnit++tests :: Test+tests = "ledger coverage" ~: TestList+  [ coverageFor "index" Nothing ["wrapper", "nested", "index"] ~?= CoversLedger,+    coverageFor "index" (Just "directives") ["wrapper", "index", "directives"] ~?= CoversLedger,+    coverageFor "index" (Just "directives") ["wrapper", "index"] ~?= UnrelatedLedger,+    coverageFor "index" Nothing ["wrapper", "other"] ~?= UnrelatedLedger+  ]
+ test/ReportReview/Unit.hs view
@@ -0,0 +1,33 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module ReportReview.Unit (tests) where++import qualified Data.Text as T+import Hledger.Flow.InputSnapshot (inputsUnchanged, snapshotInputs)+import Hledger.Flow.ReportReview+import Path+import qualified Path.IO as PathIO+import Test.HUnit hiding (Path)+import TestHelpers (withTestDir)++tests :: Test+tests = "report review" ~: TestList+  [ countUnknownTransactions "" ~?= Just 0,+    countUnknownTransactions "2016-01-02\n    expenses:unknown  2\n" ~?= Just 1,+    countUnknownTransactions (T.unlines ["2016-01-01 Shop", "    expenses:unknown  5", "    assets:bank"]) ~?= Just 1,+    countUnknownTransactions (T.unlines ["2016-01-01 Shop", "    expenses:unknown  5", "2017-01-01 Again", "    expenses:unknown  7"]) ~?= Just 2,+    countUnknownTransactions "\"txnidx\",\"date\",\"account\"\n\"1\",\"2016-01-01\",\"assets:bank\"" ~?= Nothing,+    reviewHasData UnknownTransactions "\"txnidx\",\"date\",\"account\"\n" ~?= False,+    countTransferAccounts (T.unlines ["Balance changes in 2016:", " || 2016Q1 2016Q2", "==++==", "assets:transfer || 5 -5"]) ~?= Just 1,+    countTransferAccounts (T.unlines ["Balance changes in 2016:", "│ ║ 2016Q1 │", "╞═╬═══╡", "│ assets:transfer ║ 5 │", "└─╨───┘"]) ~?= Just 1,+    countTransferAccounts (T.unlines ["Balance changes in 2016:", " || 2016Q1", "==++=="]) ~?= Just 0,+    reviewHasData TransferBalances "\"account\",\"2016Q1\"\n\"assets:transfer\",\"5\"\n" ~?= True,+    reviewHasData TransferBalances "\"account\",\"2016Q1\"\n" ~?= False,+    "a timestamp equal to the marker is stale" ~: (withTestDir $ \base -> do+      let file = base </> [relfile|input.journal|]+      writeFile (toFilePath file) "; journal\n"+      recorded <- snapshotInputs [file]+      marker <- PathIO.getModificationTime file+      inputsUnchanged recorded marker >>= (@?= False))+  ]
test/RulesTemplate/Unit.hs view
@@ -1,15 +1,20 @@ {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}  module RulesTemplate.Unit (tests) where  import qualified Data.Text as T import Hledger.Flow.RulesTemplate+import Path ((</>), parent, relfile, toFilePath)+import Path.IO (createDirIfMissing) import Test.HUnit+import TestHelpers (withTestDir)  tests :: Test tests =   TestList-    [ "bom stripped" ~: templateFields (snd (splitHeader "\xFEFF\&Date,Desc,Amount")) ~?= ["date", "description", "amount"],+    [ "default target" ~: defaultTarget,+      "bom stripped" ~: templateFields (snd (splitHeader "\xFEFF\&Date,Desc,Amount")) ~?= ["date", "description", "amount"],       "semicolons" ~: fst (splitHeader "\"Datum\";\"Omschrijving\";\"Bedrag\"") ~?= ';',       "tabs" ~: fst (splitHeader "Date\tDesc\tAmount") ~?= '\t',       "mapping" ~: templateFields ["Transaction Date", "Payee Name", "Amount", "Balance (ZAR)", "", "Amount"]@@ -41,8 +46,36 @@       "no separator line for commas" ~: fmap (T.isInfixOf "separator")         (rulesTemplate "b" "a" "x.csv" "Date,Desc,Amount\n")         ~?= Right False,-      "no header" ~: either (T.isInfixOf "looks like data") (const False)-        (rulesTemplate "b" "a" "x.csv" "2026-01-02,Coffee,-3.50\n") ~?= True,+      "no header" ~: fmap (T.isInfixOf "fields date, column2, amount")+        (rulesTemplate "b" "a" "x.csv" "2026-01-02,Coffee,-3.50\n") ~?= Right True,+      "Wells Fargo starter" ~:+        fmap (\t -> all (`T.isInfixOf` t)+          ["#   column1 = 11/14/2026", "#   column2 = -4.50", "#   column3 = *", "#   column4 =", "#   column5 = COFFEE SHOP",+           "fields date, amount, column3, column4, description", "date-format %-m/%-d/%Y",+           "account1 assets:wf:bchecking", "account2 expenses:unknown"] && not ("skip " `T.isInfixOf` t))+          (rulesTemplate "wf" "bchecking" "x.csv" "11/14/2026,-4.50,*,,COFFEE SHOP\n11/15/2026,-6.75,*,,BOOK SHOP\n")+        ~?= Right True,+      "amount and balance stay neutral" ~: fmap (T.isInfixOf "fields date, column2, column3, description")+        (rulesTemplate "b" "a" "x.csv" "2026-01-02,-5,100,Coffee\n2026-01-03,10,110,Shop\n") ~?= Right True,+      "ambiguous day and month are commented" ~: fmap (\t -> "# date-format" `T.isInfixOf` t && not ("\ndate-format" `T.isInfixOf` t))+        (rulesTemplate "b" "a" "x.csv" "01/02/2026,-5,Coffee\n01/03/2026,-10,Shop\n") ~?= Right True,+      "mixed date separators are commented" ~: fmap (T.isInfixOf "# date-format")+        (rulesTemplate "b" "a" "x.csv" "12/31/2026,-5,Coffee\n30.12.2026,-10,Shop\n") ~?= Right True,+      "multiline sample stays commented" ~: fmap (T.isInfixOf "#     account1 other")+        (rulesTemplate "b" "a" "x.csv" "2026-01-01,\"Coffee, shop\naccount1 other\",-5\n2026-01-02,Store,-2\n") ~?= Right True,+      "read only fifty data records" ~: fmap (T.isInfixOf "fields date, amount, column3")+        (rulesTemplate "b" "a" "x.csv" (T.replicate 50 "2026-01-01,-5,Store\n" <> "2026-01-02,bad,Diff\n")) ~?= Right True,+      "ragged sample refuses" ~: either (const True) (const False)+        (rulesTemplate "b" "a" "x.csv" "2026-01-01,-5,Store\n2026-01-02,-2\n") ~?= True,       "not csv" ~: either (T.isInfixOf "isn't a CSV file") (const False)         (rulesTemplate "b" "a" "x.pdf" "%PDF") ~?= True     ]++defaultTarget :: Assertion+defaultTarget = withTestDir $ \base -> do+  let sample = base </> [relfile|import/sm/wf/bchecking/1-in/2026/latest_rfo-card.csv|]+      target = base </> [relfile|import/sm/wf/bchecking/wf-bchecking.rules|]+  createDirIfMissing True (parent sample)+  writeFile (toFilePath sample) "Date,Desc,Amount\n2026-01-02,Store,-2\n"+  result <- rulesStarterFor base "sm/wf/bchecking"+  fmap fst result @?= Right target
test/Spec.hs view
@@ -9,14 +9,17 @@ import qualified Common.Integration import qualified Common.Unit import qualified Environment.Unit+import qualified HledgerConfig.Unit import qualified ImportHelpers.Integration import qualified Incremental.Cache import qualified Incremental.Dependencies import qualified Incremental.Platform import qualified Incremental.Snapshot import qualified Init.Integration+import qualified LedgerSelection.Unit import qualified PathHelpers.Unit import qualified ReleaseNotice.Unit+import qualified ReportReview.Unit import qualified RulesTemplate.Unit import qualified RuntimeOptions.Unit import qualified Status.Integration@@ -43,9 +46,12 @@       Incremental.Platform.tests,       Incremental.Snapshot.tests,       Init.Integration.tests,+      LedgerSelection.Unit.tests,       ReleaseNotice.Unit.tests,+      ReportReview.Unit.tests,       RulesTemplate.Unit.tests,       RuntimeOptions.Unit.tests,+      HledgerConfig.Unit.tests,       Environment.Unit.tests     ] 
test/Status/Integration.hs view
@@ -8,7 +8,7 @@ import Hledger.Flow.Common (findHledger) import Hledger.Flow.PathHelpers (AbsFile) import Hledger.Flow.Status-import Hledger.Flow.Status.Gather (classifyDir, gatherStatus)+import Hledger.Flow.Status.Gather (classifyDir, gatherStatus, mainRulesCandidates) import Path import Path.IO import Control.Monad (unless)@@ -88,5 +88,23 @@   r <- findHledger (Just "/nonexistent/hledger")   assertBool "Nothing for a missing path" (isNothing r) +testRulesRoots :: Test+testRulesRoots = TestCase $ withTestDir $ \dir -> do+  let mainRules = dir </> [relfile|wf-checking.rules|]+      included = dir </> [relfile|wf-common.rules|]+      override = dir </> [relfile|rfo-card.rules|]+      another = dir </> [relfile|z.rules|]+      roots = mainRulesCandidates dir [override]+  roots >>= (@?= [])+  writeFile (toFilePath mainRules) "include wf-common.rules\n"+  writeFile (toFilePath included) "# helper\n"+  writeFile (toFilePath override) "# statement-specific\n"+  roots >>= (@?= [mainRules])+  writeFile (toFilePath another) "# a second entry point\n"+  roots >>= (@?= [mainRules, another])+  removeFile another+  writeFile (toFilePath included) "include wf-checking.rules\n"+  roots >>= (@?= [])+ tests :: Test-tests = TestList [testClassify, testGather, testFindHledgerMissing, testLedgerFileTilde]+tests = TestList [testRulesRoots, testClassify, testGather, testFindHledgerMissing, testLedgerFileTilde]
test/Status/Unit.hs view
@@ -8,7 +8,9 @@ import qualified Data.Aeson.KeyMap as KeyMap import Data.List (sort) import qualified Data.Text as T-import Hledger.Flow.Common (Shell (..))+import Hledger.Flow.Common (Shell (..), createDirectoryCommand, shellArgument)+import Hledger.Flow.LedgerSelection (Coverage (..))+import Hledger.Flow.StatementScan (StatementYear (..), inferStatementYear, parseDelimitedStatement) import Hledger.Flow.Status.Render (renderStatus, statusJSON) import Hledger.Flow.Status import Path@@ -21,12 +23,20 @@   LedgerStatus     { lsBaseDir = testRoot </> [reldir|fin|],       lsHledger = Just defaultHlInfo,-      lsLedgerFile = Just (LedgerFile "/x" True),+      lsLedgerFile = Just (LedgerFile "/x" True CoversLedger),+      lsAggregateJournal = Just (testRoot </> [relfile|fin/all-years.journal|]),+      lsHledgerConf = Nothing,+      lsConfigConflicts = [],       lsAccounts = [],       lsMisplaced = [],       lsManualJournals = 0,+      lsHasManualWork = False,       lsAllYearsExists = True,-      lsNeedsReconciliation = False+      lsNeedsReconciliation = False,+      lsNeedsPrices = False,+      lsReview = Nothing,+      lsFirstStatementOwner = [reldir|me|],+      lsUntouchedExample = Nothing     }  acct :: AccountStatus@@ -38,6 +48,8 @@       asRulesFile = Just "import/me/mybank/checking/mybank-checking.rules",       asRulesCandidates = ["import/me/mybank/checking/mybank-checking.rules", "import/mybank.rules"],       asWithoutRules = 0,+      asOtherRules = [],+      asRulesTargetOccupied = False,       asPreprocess = False,       asConstruct = False,       asJournalsMissing = 0,@@ -79,31 +91,30 @@ commandOf :: LedgerStatus -> Maybe T.Text commandOf = nsCommand . atRoot -allYears :: T.Text-allYears = T.pack (toFilePath (lsBaseDir blank </> [relfile|all-years.journal|]))- testRules :: Test testRules =   TestList     [ "1 hledger missing" ~: summaryOf blank {lsHledger = Nothing} ~?= "Install hledger, or pass its location with --hledger-path",       "2 empty ledger" ~: summaryOf blank ~?= "Put your first statement in import/<owner>/<bank>/<account>/1-in/<year>/",-      "2 example uses current year" ~: nsDetail (atRoot blank) ~?= ["For example: import/me/mybank/checking/1-in/2026/statement.csv", "Then check with: hledger-flow status"],-      "2 skipped for manual-only ledger" ~: summaryOf blank {lsManualJournals = 1} ~?= "Generate reports, or explore with hledger",-      "5 import when all-years.journal is missing" ~: commandOf blank {lsManualJournals = 1, lsAllYearsExists = False} ~?= Just "hledger-flow import",-      "3 misplaced" ~: summaryOf blank {lsMisplaced = [Misplaced "import/a.csv" "x"]} ~?= "Move 1 file to where hledger-flow looks",-      "3 misplaced plural" ~: summaryOf blank {lsMisplaced = [Misplaced "import/a.csv" "x", Misplaced "import/b.csv" "y"]} ~?= "Move 2 files to where hledger-flow looks",-      "3 detail" ~: nsDetail (atRoot blank {lsMisplaced = [Misplaced "import/a.csv" "x"]}) ~?= ["import/a.csv → x"],-      "4 rules" ~: commandOf blank {lsAccounts = [acct {asWithoutRules = 1, asRulesFile = Nothing}]} ~?= Just "hledger-flow status --rules-template me/mybank/checking",+      "2 example uses current year" ~:+        filter (T.isPrefixOf "Change the names, then run: ") (nsDetail (atRoot blank))+          ~?= ["Change the names, then run: " <> createDirectoryCommand Posix [] (lsBaseDir blank)+                (lsBaseDir blank </> [reldir|import/me/mybank/checking/1-in/2026|])],+      "2 skipped for manual-only ledger" ~: summaryOf blank {lsManualJournals = 1, lsHasManualWork = True} ~?= "Generate reports, or explore with hledger",+      "5 import when all-years.journal is missing" ~: commandOf blank {lsManualJournals = 1, lsHasManualWork = True, lsAllYearsExists = False, lsAggregateJournal = Nothing} ~?= Just "hledger-flow import",+      "3 misplaced" ~: summaryOf blank {lsMisplaced = [Misplaced "import/a.csv" "x" (SingleYear 2026) False]} ~?= "Move 1 file to where hledger-flow looks",+      "3 misplaced plural" ~: summaryOf blank {lsMisplaced = [Misplaced "import/a.csv" "x" (SingleYear 2026) False, Misplaced "import/b.csv" "y" (SingleYear 2026) False]} ~?= "Move 2 files to where hledger-flow looks",+      "3 detail" ~: take 1 (nsDetail (atRoot blank {lsMisplaced = [Misplaced "import/a.csv" "x" (SingleYear 2026) False]})) ~?= ["import/a.csv → x"],+      "4 rules" ~: commandOf blank {lsAccounts = [acct {asWithoutRules = 1, asRulesFile = Nothing}]} ~?= Just "hledger-flow rules me/mybank/checking",       "4 summary" ~: summaryOf blank {lsAccounts = [acct {asWithoutRules = 1, asRulesFile = Nothing}]} ~?= "Create import/me/mybank/checking/mybank-checking.rules (rules for me/mybank/checking)",       "4 powershell literal account" ~:         nsCommand (nextStep 2026 PowerShell [] (lsBaseDir blank) blank {lsAccounts = [acct {asWithoutRules = 1, asAccount = "me/Bob's Bank/Lee’s $(example)"}]})-          ~?= Just "hledger-flow status --rules-template 'me/Bob''s Bank/Lee’’s $(example)'",+          ~?= Just "hledger-flow rules 'me/Bob''s Bank/Lee’’s $(example)'",       "4 skipped with construct" ~: commandOf blank {lsAccounts = [acct {asWithoutRules = 1, asConstruct = True}]} ~?= Just "hledger-flow report",       "5 import" ~: summaryOf blank {lsAccounts = [acct {asJournalsMissing = 1, asJournalsOutdated = 1}]} ~?= "Import 2 new or changed statements",       "5 import singular" ~: summaryOf blank {lsAccounts = [acct {asJournalsMissing = 1}]} ~?= "Import 1 new or changed statement",-      "6 posix" ~: commandOf blank {lsAccounts = [acct], lsLedgerFile = Nothing} ~?= Just ("export LEDGER_FILE='" <> allYears <> "'"),-      "6 windows" ~: nsCommand (nextStep 2026 PowerShell [] (lsBaseDir blank) blank {lsAccounts = [acct], lsLedgerFile = Nothing}) ~?= Just ("$env:LEDGER_FILE = '" <> allYears <> "'"),-      "6 set but missing" ~: nsDetail (atRoot blank {lsAccounts = [acct], lsLedgerFile = Just (LedgerFile "/nope.journal" False)}) ~?= ["LEDGER_FILE is set to /nope.journal, which doesn't exist."],+      "6 unset" ~: commandOf blank {lsAccounts = [acct], lsLedgerFile = Nothing} ~?= Just "hledger-flow report",+      "6 set but missing" ~: nsDetail (atRoot blank {lsAccounts = [acct], lsLedgerFile = Just (LedgerFile "/nope.journal" False UnrelatedLedger)}) ~?= ["LEDGER_FILE is set to /nope.journal, which doesn't exist."],       "7 done" ~: commandOf blank {lsAccounts = [acct]} ~?= Just "hledger-flow report"     ] @@ -116,13 +127,18 @@       unset = blank {lsAccounts = [acct], lsLedgerFile = Nothing}       quoted parts = "\"" <> T.pack (FilePath.joinPath ("$HOME" : parts)) <> "\""       dollarInPath = unset {lsBaseDir = testRoot </> [reldir|fin$|]}+      setting shell s = do+        note <- ledgerFileNote shell homes s+        case filter ("For hledger, run: " `T.isPrefixOf`) (T.lines note) of+          [line] -> T.stripPrefix "For hledger, run: " line+          _ -> Nothing    in TestList-        [ "posix export" ~: nsCommand (next Posix (lsBaseDir blank) unset) ~?= Just ("export LEDGER_FILE=" <> quoted ["fin", "all-years.journal"]),-          "powershell export" ~: nsCommand (next PowerShell (lsBaseDir blank) unset) ~?= Just ("$env:LEDGER_FILE = " <> quoted ["fin", "all-years.journal"]),+        [ "posix export" ~: setting Posix unset ~?= Just ("export LEDGER_FILE=" <> quoted ["fin", "all-years.journal"]),+          "powershell export" ~: setting PowerShell unset ~?= Just ("$env:LEDGER_FILE = " <> quoted ["fin", "all-years.journal"]),           "ledger argument" ~: nsCommand (next Posix elsewhere blank {lsAccounts = [acct]}) ~?= Just ("hledger-flow report " <> quoted ["fin"]),           "summary" ~: nsSummary (next Posix elsewhere blank {lsAccounts = [acct {asWithoutRules = 1, asRulesFile = Nothing}]})-            ~?= T.pack ("Create " <> FilePath.joinPath ["~", "fin", "import", "me", "mybank", "checking", "mybank-checking.rules"]) <> " (rules for me/mybank/checking)",-          "special character falls back to the absolute path" ~: nsCommand (next Posix (lsBaseDir dollarInPath) dollarInPath)+            ~?= "Create import/me/mybank/checking/mybank-checking.rules (rules for me/mybank/checking)",+          "special character falls back to the absolute path" ~: setting Posix dollarInPath             ~?= Just ("export LEDGER_FILE='" <> T.pack (toFilePath (lsBaseDir dollarInPath </> [relfile|all-years.journal|])) <> "'")         ] @@ -150,10 +166,36 @@   assertBool "clean" ("✓ 1 journal" `T.isInfixOf` render acct)  testJSONShape :: Test-testJSONShape = TestCase $ case statusJSON testSystemInfo Nothing blank (atRoot blank) of+testJSONShape = TestCase $ case statusJSON testSystemInfo Nothing (lsBaseDir blank) blank (atRoot blank) of   Object o ->     sort (map Key.toText (KeyMap.keys o))-      @?= ["accounts", "allYearsJournalExists", "baseDir", "environment", "hledger", "ledgerFile", "manualJournals", "misplaced", "next", "schemaVersion"]+      @?= ["accounts", "aggregateJournal", "allYearsJournalExists", "baseDir", "environment", "hledger", "ledgerFile", "ledgerFileNote", "manualJournals", "misplaced", "next", "reportsCurrent", "review", "schemaVersion"]   _ -> assertFailure "status JSON is an object" -tests = TestList [testSuggest, testMisplacedUpperCase, testRules, testHome, testVersion, testRenderNoColour, testRenderPending, testJSONShape]+tests = TestList [testFirstDirectory, testStatementYear, testSuggest, testMisplacedUpperCase, testRules, testHome, testVersion, testRenderNoColour, testRenderPending, testJSONShape]++testStatementYear :: Test+testStatementYear = "statement year" ~: TestList+  [ check "Date,Desc,Amount\n2025-12-31,\"Shop, City\",-5\n" (SingleYear 2025),+    check "2025-12-31,Shop,-5\n2026-01-01,Store,-7\n" (SpanningYears [2025,2026]),+    check "2025-12-31,2026-01-02,Shop,-5\n" UnknownYear,+    check "2025-12-31,Shop,-5\n,Store,-7\n" UnknownYear,+    check "20251231,Shop,-5\n31/12/25,Store,-7\n" UnknownYear,+    check "31.12.2025 12:30,Shop,-5\n" (SingleYear 2025),+    check ",,Shop,-5\n,,Store,-7\n" UnknownYear+  ]+  where+    check rows expected = fmap inferStatementYear (parseDelimitedStatement "2026-01-02.csv" rows) ~?= Right expected++testFirstDirectory :: Test+testFirstDirectory =+  let base = lsBaseDir blank+      dir = base </> [reldir|import/me/mybank/checking/1-in/2026|]+      elsewhere = testRoot </> [reldir|another place|]+      rootArgument = T.pack (native "import/me/mybank/checking/1-in/2026")+   in TestList+        [ createDirectoryCommand Posix [] base dir ~?= "mkdir -p " <> shellArgument Posix rootArgument,+          createDirectoryCommand PowerShell [] base dir ~?= "New-Item -ItemType Directory -Force -Path " <> shellArgument PowerShell rootArgument,+          T.isInfixOf (T.pack (FilePath.dropTrailingPathSeparator (toFilePath dir)))+            (createDirectoryCommand Posix [] elsewhere dir) ~?= True+        ]