hledger-flow 0.20.1 → 0.21.0
raw patch · 40 files changed
+2860/−384 lines, 40 filesdep +ShellCheckPVP ok
version bump matches the API change (PVP)
Dependencies added: ShellCheck
API changes (from Hackage documentation)
- Hledger.Flow.Import.Dependencies: checkDirectives :: Release -> AbsFile -> ByteString -> ReuseDecision ()
- Hledger.Flow.Import.Dependencies: resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)
- Hledger.Flow.Status: instance GHC.Classes.Eq Hledger.Flow.Status.LedgerFile
- Hledger.Flow.Status: instance GHC.Internal.Show.Show Hledger.Flow.Status.LedgerFile
+ Hledger.Flow.Cache: initializeIgnoredDir :: AbsDir -> IO ()
+ Hledger.Flow.Cache: trashDir :: AbsDir -> AbsDir
+ Hledger.Flow.Common: createDirectoryCommand :: Shell -> [FilePath] -> AbsDir -> AbsDir -> Text
+ Hledger.Flow.Common: includePreamble :: Text
+ Hledger.Flow.Common: makeDirectory :: Shell -> Text -> Text
+ Hledger.Flow.Common: moveFileCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> RelFile -> RelFile -> Text
+ Hledger.Flow.DocHelpers: Balances :: DocLink
+ Hledger.Flow.DocHelpers: ManualJournals :: DocLink
+ Hledger.Flow.DocHelpers: PriceFiles :: DocLink
+ Hledger.Flow.DocHelpers: ReviewTransfers :: DocLink
+ Hledger.Flow.DocHelpers: ReviewUnknownTransactions :: DocLink
+ Hledger.Flow.DocHelpers: StatementYearFolders :: DocLink
+ Hledger.Flow.HledgerConfig: BalanceConfig :: ConfigUse
+ Hledger.Flow.HledgerConfig: ConfigLine :: Text -> Int -> Text -> [Text] -> Bool -> ConfigLine
+ Hledger.Flow.HledgerConfig: ImportConfig :: ConfigUse
+ Hledger.Flow.HledgerConfig: ReportConfig :: ConfigUse
+ Hledger.Flow.HledgerConfig: [configArguments] :: ConfigLine -> [Text]
+ Hledger.Flow.HledgerConfig: [configLineNumber] :: ConfigLine -> Int
+ Hledger.Flow.HledgerConfig: [configSection] :: ConfigLine -> Text
+ Hledger.Flow.HledgerConfig: [configSource] :: ConfigLine -> Text
+ Hledger.Flow.HledgerConfig: [configStartsSection] :: ConfigLine -> Bool
+ Hledger.Flow.HledgerConfig: configConflicts :: FilePath -> [ConfigLine] -> [ConfigConflict]
+ Hledger.Flow.HledgerConfig: data ConfigConflict
+ Hledger.Flow.HledgerConfig: data ConfigLine
+ Hledger.Flow.HledgerConfig: data ConfigUse
+ Hledger.Flow.HledgerConfig: findConfigConflict :: HledgerInfo -> ConfigUse -> Maybe HledgerConf -> IO (Maybe ConfigConflict)
+ Hledger.Flow.HledgerConfig: findConfigConflicts :: HledgerInfo -> Maybe HledgerConf -> IO [ConfigConflict]
+ Hledger.Flow.HledgerConfig: firstConfigConflict :: ConfigUse -> [ConfigConflict] -> Maybe ConfigConflict
+ Hledger.Flow.HledgerConfig: instance GHC.Classes.Eq Hledger.Flow.HledgerConfig.ConfigConflict
+ Hledger.Flow.HledgerConfig: instance GHC.Classes.Eq Hledger.Flow.HledgerConfig.ConfigLine
+ Hledger.Flow.HledgerConfig: instance GHC.Classes.Eq Hledger.Flow.HledgerConfig.ConfigUse
+ Hledger.Flow.HledgerConfig: instance GHC.Internal.Show.Show Hledger.Flow.HledgerConfig.ConfigConflict
+ Hledger.Flow.HledgerConfig: instance GHC.Internal.Show.Show Hledger.Flow.HledgerConfig.ConfigLine
+ Hledger.Flow.HledgerConfig: instance GHC.Internal.Show.Show Hledger.Flow.HledgerConfig.ConfigUse
+ Hledger.Flow.HledgerConfig: parseConfigLines :: Bool -> [Text] -> [ConfigLine]
+ Hledger.Flow.HledgerConfig: renderConfigConflict :: ConfigConflict -> Text
+ Hledger.Flow.HledgerConfig: requireSafeConfig :: ConfigUse -> RuntimeOptions -> IO ()
+ Hledger.Flow.Import.Dependencies: ConfigEntry :: Text -> FilePath -> Text -> FallbackReason
+ Hledger.Flow.Import.Dependencies: CsvRules :: Release -> IncludeFile
+ Hledger.Flow.Import.Dependencies: IncludeNode :: AbsFile -> AbsFile -> [AbsDir] -> ByteString -> IncludeNode
+ Hledger.Flow.Import.Dependencies: JournalDirectives :: IncludeFile
+ Hledger.Flow.Import.Dependencies: [includeBytes] :: IncludeNode -> ByteString
+ Hledger.Flow.Import.Dependencies: [includeDirectories] :: IncludeNode -> [AbsDir]
+ Hledger.Flow.Import.Dependencies: [includePosition] :: IncludeNode -> AbsFile
+ Hledger.Flow.Import.Dependencies: [includeSource] :: IncludeNode -> AbsFile
+ Hledger.Flow.Import.Dependencies: data IncludeFile
+ Hledger.Flow.Import.Dependencies: data IncludeNode
+ Hledger.Flow.Import.Dependencies: instance GHC.Classes.Eq Hledger.Flow.Import.Dependencies.IncludeFile
+ Hledger.Flow.Import.Dependencies: instance GHC.Internal.Show.Show Hledger.Flow.Import.Dependencies.IncludeFile
+ Hledger.Flow.Import.Dependencies: journalIncludes :: AbsFile -> ByteString -> ReuseDecision [FilePath]
+ Hledger.Flow.Import.Dependencies: readIncludeTree :: AbsDir -> IncludeFile -> AbsFile -> IO (ReuseDecision [IncludeNode])
+ Hledger.Flow.Import.Dependencies: resolveInclude :: IncludeFile -> AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)
+ Hledger.Flow.Import.Dependencies: scanIncludes :: IncludeFile -> Bool -> AbsFile -> ByteString -> ReuseDecision [FilePath]
+ Hledger.Flow.Import.ShellSources: data SourceInput
+ Hledger.Flow.Import.ShellSources: hookSources :: SourceCache -> AbsFile -> FileObservation -> IO [SourceInput]
+ Hledger.Flow.Import.ShellSources: instance GHC.Classes.Eq Hledger.Flow.Import.ShellSources.SourceInput
+ Hledger.Flow.Import.ShellSources: instance GHC.Internal.Show.Show Hledger.Flow.Import.ShellSources.SourceInput
+ Hledger.Flow.Import.ShellSources: newSourceCache :: IO SourceCache
+ Hledger.Flow.Import.ShellSources: sourceKey :: SourceInput -> [(Text, Text)]
+ Hledger.Flow.Import.ShellSources: type SourceCache = IORef Map Digest [Step]
+ Hledger.Flow.Init.Example: directivesAreStarter :: BaseDir -> IO Bool
+ Hledger.Flow.Init.Example: exampleAccount :: RelDir -> RelDir
+ Hledger.Flow.Init.Example: exampleAccountPath :: RelDir -> Text
+ Hledger.Flow.Init.Example: exampleBaseFiles :: Integer -> [(RelFile, Text)]
+ Hledger.Flow.Init.Example: exampleImportFiles :: RelDir -> Integer -> [(RelFile, Text)]
+ Hledger.Flow.Init.Example: isSampleStatement :: AbsFile -> Bool
+ Hledger.Flow.Init.Example: journalIsCommentsOnly :: AbsFile -> IO Bool
+ Hledger.Flow.Init.Example: sampleAccountDir :: BaseDir -> RelDir -> AbsDir
+ Hledger.Flow.Init.Example: starterOwner :: IO RelDir
+ Hledger.Flow.Init.Example: untouchedExample :: BaseDir -> [AbsFile] -> IO (Maybe RelDir)
+ Hledger.Flow.InputSnapshot: data InputSnapshot
+ Hledger.Flow.InputSnapshot: inputTargets :: InputSnapshot -> [FilePath]
+ Hledger.Flow.InputSnapshot: inputsMatch :: InputSnapshot -> IO Bool
+ Hledger.Flow.InputSnapshot: inputsUnchanged :: InputSnapshot -> UTCTime -> IO Bool
+ Hledger.Flow.InputSnapshot: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: instance GHC.Classes.Eq Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance GHC.Classes.Eq Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: instance GHC.Internal.Generics.Generic Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance GHC.Internal.Generics.Generic Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: instance GHC.Internal.Show.Show Hledger.Flow.InputSnapshot.Entry
+ Hledger.Flow.InputSnapshot: instance GHC.Internal.Show.Show Hledger.Flow.InputSnapshot.InputSnapshot
+ Hledger.Flow.InputSnapshot: snapshotInputs :: [AbsFile] -> IO InputSnapshot
+ Hledger.Flow.LedgerSelection: CoversLedger :: Coverage
+ Hledger.Flow.LedgerSelection: LedgerFile :: FilePath -> Bool -> Coverage -> LedgerFile
+ Hledger.Flow.LedgerSelection: LedgerSelection :: Maybe LedgerFile -> Maybe AbsFile -> Maybe Text -> LedgerSelection
+ Hledger.Flow.LedgerSelection: UnreadableLedger :: Text -> Coverage
+ Hledger.Flow.LedgerSelection: UnrelatedLedger :: Coverage
+ Hledger.Flow.LedgerSelection: [aggregateJournal] :: LedgerSelection -> Maybe AbsFile
+ Hledger.Flow.LedgerSelection: [lfCoverage] :: LedgerFile -> Coverage
+ Hledger.Flow.LedgerSelection: [lfExists] :: LedgerFile -> Bool
+ Hledger.Flow.LedgerSelection: [lfValue] :: LedgerFile -> FilePath
+ Hledger.Flow.LedgerSelection: [selectedLedgerFile] :: LedgerSelection -> Maybe LedgerFile
+ Hledger.Flow.LedgerSelection: [selectionError] :: LedgerSelection -> Maybe Text
+ Hledger.Flow.LedgerSelection: coverageFor :: FilePath -> Maybe FilePath -> [FilePath] -> Coverage
+ Hledger.Flow.LedgerSelection: data Coverage
+ Hledger.Flow.LedgerSelection: data LedgerFile
+ Hledger.Flow.LedgerSelection: data LedgerSelection
+ Hledger.Flow.LedgerSelection: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.LedgerSelection.CachedInputs
+ Hledger.Flow.LedgerSelection: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.LedgerSelection.CachedInputs
+ Hledger.Flow.LedgerSelection: instance GHC.Classes.Eq Hledger.Flow.LedgerSelection.Coverage
+ Hledger.Flow.LedgerSelection: instance GHC.Classes.Eq Hledger.Flow.LedgerSelection.LedgerFile
+ Hledger.Flow.LedgerSelection: instance GHC.Internal.Generics.Generic Hledger.Flow.LedgerSelection.CachedInputs
+ Hledger.Flow.LedgerSelection: instance GHC.Internal.Show.Show Hledger.Flow.LedgerSelection.Coverage
+ Hledger.Flow.LedgerSelection: instance GHC.Internal.Show.Show Hledger.Flow.LedgerSelection.LedgerFile
+ Hledger.Flow.LedgerSelection: instance GHC.Internal.Show.Show Hledger.Flow.LedgerSelection.LedgerSelection
+ Hledger.Flow.LedgerSelection: journalInputs :: HledgerInfo -> [AbsFile] -> IO (Either Text [AbsFile])
+ Hledger.Flow.LedgerSelection: ledgerFileProblem :: [FilePath] -> LedgerFile -> Text
+ Hledger.Flow.LedgerSelection: selectLedger :: [FilePath] -> Maybe HledgerInfo -> BaseDir -> IO LedgerSelection
+ Hledger.Flow.PathHelpers: pathOccupied :: Path Abs t -> IO Bool
+ Hledger.Flow.PathHelpers: publishNewFile :: AbsFile -> AbsFile -> IO ()
+ Hledger.Flow.PathHelpers: writeNewUtf8File :: AbsFile -> Text -> IO ()
+ Hledger.Flow.PathHelpers: writeTimestampMarker :: AbsFile -> IO UTCTime
+ Hledger.Flow.ReportReview: Review :: ReviewCategory -> ReviewCategory -> [ReviewItem] -> Review
+ Hledger.Flow.ReportReview: ReviewCategory :: Maybe Int -> [FilePath] -> ReviewCategory
+ Hledger.Flow.ReportReview: ReviewItem :: ReviewKind -> FilePath -> Integer -> Maybe Int -> ReviewItem
+ Hledger.Flow.ReportReview: TransferBalances :: ReviewKind
+ Hledger.Flow.ReportReview: UnknownTransactions :: ReviewKind
+ Hledger.Flow.ReportReview: [itemCount] :: ReviewItem -> Maybe Int
+ Hledger.Flow.ReportReview: [itemKind] :: ReviewItem -> ReviewKind
+ Hledger.Flow.ReportReview: [itemPath] :: ReviewItem -> FilePath
+ Hledger.Flow.ReportReview: [itemYear] :: ReviewItem -> Integer
+ Hledger.Flow.ReportReview: [reviewCount] :: ReviewCategory -> Maybe Int
+ Hledger.Flow.ReportReview: [reviewFiles] :: ReviewCategory -> [FilePath]
+ Hledger.Flow.ReportReview: [reviewItems] :: Review -> [ReviewItem]
+ Hledger.Flow.ReportReview: [reviewTransfers] :: Review -> ReviewCategory
+ Hledger.Flow.ReportReview: [reviewUnknowns] :: Review -> ReviewCategory
+ Hledger.Flow.ReportReview: captureReportInputs :: HledgerInfo -> UTCTime -> AbsFile -> Maybe HledgerConf -> [[AbsFile]] -> IO (Either Text ReportInputs)
+ Hledger.Flow.ReportReview: countTransferAccounts :: Text -> Maybe Int
+ Hledger.Flow.ReportReview: countUnknownTransactions :: Text -> Maybe Int
+ Hledger.Flow.ReportReview: data Review
+ Hledger.Flow.ReportReview: data ReviewCategory
+ Hledger.Flow.ReportReview: data ReviewFile
+ Hledger.Flow.ReportReview: data ReviewItem
+ Hledger.Flow.ReportReview: data ReviewKind
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReportReview.Manifest
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReportReview.ReportInputs
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReportReview.ReviewFile
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.FromJSON.FromJSON Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReportReview.Manifest
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReportReview.ReportInputs
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReportReview.ReviewFile
+ Hledger.Flow.ReportReview: instance Data.Aeson.Types.ToJSON.ToJSON Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: instance GHC.Classes.Eq Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: instance GHC.Internal.Generics.Generic Hledger.Flow.ReportReview.Manifest
+ Hledger.Flow.ReportReview: instance GHC.Internal.Generics.Generic Hledger.Flow.ReportReview.ReportInputs
+ Hledger.Flow.ReportReview: instance GHC.Internal.Generics.Generic Hledger.Flow.ReportReview.ReviewFile
+ Hledger.Flow.ReportReview: instance GHC.Internal.Generics.Generic Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: instance GHC.Internal.Show.Show Hledger.Flow.ReportReview.Review
+ Hledger.Flow.ReportReview: instance GHC.Internal.Show.Show Hledger.Flow.ReportReview.ReviewCategory
+ Hledger.Flow.ReportReview: instance GHC.Internal.Show.Show Hledger.Flow.ReportReview.ReviewItem
+ Hledger.Flow.ReportReview: instance GHC.Internal.Show.Show Hledger.Flow.ReportReview.ReviewKind
+ Hledger.Flow.ReportReview: invalidateReportReview :: BaseDir -> IO ()
+ Hledger.Flow.ReportReview: loadReportReview :: BaseDir -> AbsFile -> Maybe HledgerConf -> IO (Maybe Review)
+ Hledger.Flow.ReportReview: reviewFile :: BaseDir -> AbsFile -> Maybe ReviewFile
+ Hledger.Flow.ReportReview: reviewHasData :: ReviewKind -> Text -> Bool
+ Hledger.Flow.ReportReview: saveReportReview :: BaseDir -> ReportInputs -> [ReviewFile] -> IO ()
+ Hledger.Flow.ReportReview: startReportRun :: BaseDir -> IO UTCTime
+ Hledger.Flow.RulesTemplate: rulesStarterFor :: BaseDir -> Text -> IO (Either Text (AbsFile, Text))
+ Hledger.Flow.RulesTemplate: runRules :: Maybe HledgerConf -> BaseDir -> Text -> IO ()
+ Hledger.Flow.StatementScan: DelimitedStatement :: Char -> Maybe [Text] -> [[Text]] -> DelimitedStatement
+ Hledger.Flow.StatementScan: SingleYear :: Integer -> StatementYear
+ Hledger.Flow.StatementScan: SpanningYears :: [Integer] -> StatementYear
+ Hledger.Flow.StatementScan: UnknownYear :: StatementYear
+ Hledger.Flow.StatementScan: [statementHeader] :: DelimitedStatement -> Maybe [Text]
+ Hledger.Flow.StatementScan: [statementRows] :: DelimitedStatement -> [[Text]]
+ Hledger.Flow.StatementScan: [statementSeparator] :: DelimitedStatement -> Char
+ Hledger.Flow.StatementScan: data DateCell
+ Hledger.Flow.StatementScan: data DelimitedStatement
+ Hledger.Flow.StatementScan: data StatementYear
+ Hledger.Flow.StatementScan: dateCellYear :: DateCell -> Integer
+ Hledger.Flow.StatementScan: inferDateFormat :: [DateCell] -> Maybe Text
+ Hledger.Flow.StatementScan: inferStatementYear :: DelimitedStatement -> StatementYear
+ Hledger.Flow.StatementScan: instance GHC.Classes.Eq Hledger.Flow.StatementScan.DateCell
+ Hledger.Flow.StatementScan: instance GHC.Classes.Eq Hledger.Flow.StatementScan.DelimitedStatement
+ Hledger.Flow.StatementScan: instance GHC.Classes.Eq Hledger.Flow.StatementScan.StatementYear
+ Hledger.Flow.StatementScan: instance GHC.Internal.Show.Show Hledger.Flow.StatementScan.DateCell
+ Hledger.Flow.StatementScan: instance GHC.Internal.Show.Show Hledger.Flow.StatementScan.DelimitedStatement
+ Hledger.Flow.StatementScan: instance GHC.Internal.Show.Show Hledger.Flow.StatementScan.StatementYear
+ Hledger.Flow.StatementScan: isDelimitedFile :: FilePath -> Bool
+ Hledger.Flow.StatementScan: isSignedDecimal :: Text -> Bool
+ Hledger.Flow.StatementScan: looksLikeHeader :: [Text] -> Bool
+ Hledger.Flow.StatementScan: parseDateCell :: Text -> Maybe DateCell
+ Hledger.Flow.StatementScan: parseDelimitedStatement :: FilePath -> Text -> Either Text DelimitedStatement
+ Hledger.Flow.StatementScan: parseRuleSample :: FilePath -> Text -> Either Text DelimitedStatement
+ Hledger.Flow.StatementScan: readDelimitedStatement :: Maybe Int -> AbsFile -> IO (Either Text DelimitedStatement)
+ Hledger.Flow.StatementScan: readRuleSample :: AbsFile -> IO (Either Text DelimitedStatement)
+ Hledger.Flow.StatementScan: splitHeader :: Text -> (Char, [Text])
+ Hledger.Flow.Status: [asOtherRules] :: AccountStatus -> [FilePath]
+ Hledger.Flow.Status: [asRulesTargetOccupied] :: AccountStatus -> Bool
+ Hledger.Flow.Status: [lfCoverage] :: LedgerFile -> Coverage
+ Hledger.Flow.Status: [lsAggregateJournal] :: LedgerStatus -> Maybe AbsFile
+ Hledger.Flow.Status: [lsConfigConflicts] :: LedgerStatus -> [ConfigConflict]
+ Hledger.Flow.Status: [lsFirstStatementOwner] :: LedgerStatus -> RelDir
+ Hledger.Flow.Status: [lsHasManualWork] :: LedgerStatus -> Bool
+ Hledger.Flow.Status: [lsHledgerConf] :: LedgerStatus -> Maybe HledgerConf
+ Hledger.Flow.Status: [lsNeedsPrices] :: LedgerStatus -> Bool
+ Hledger.Flow.Status: [lsReview] :: LedgerStatus -> Maybe Review
+ Hledger.Flow.Status: [lsUntouchedExample] :: LedgerStatus -> Maybe RelDir
+ Hledger.Flow.Status: [mpDestinationExists] :: Misplaced -> Bool
+ Hledger.Flow.Status: [mpYear] :: Misplaced -> StatementYear
+ Hledger.Flow.Status: balanceHint :: Shell -> [FilePath] -> LedgerStatus -> Maybe Text
+ Hledger.Flow.Status: firstStatementStep :: Integer -> Shell -> [FilePath] -> AbsDir -> BaseDir -> RelDir -> NextStep
+ Hledger.Flow.Status: flowCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Maybe HledgerConf -> Text -> [Text] -> Text
+ Hledger.Flow.Status: ledgerFileNote :: Shell -> [FilePath] -> LedgerStatus -> Maybe Text
+ Hledger.Flow.Status: misplacedMoveCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Misplaced -> Maybe Text
+ Hledger.Flow.Status: placementDestination :: Misplaced -> Maybe RelFile
+ Hledger.Flow.Status: withConfigCheck :: ConfigUse -> LedgerStatus -> NextStep -> NextStep
+ Hledger.Flow.Status.Gather: gatherStatusWithConfig :: Maybe HledgerInfo -> Maybe HledgerConf -> BaseDir -> IO LedgerStatus
+ Hledger.Flow.Status.Gather: mainRulesCandidates :: AbsDir -> [AbsFile] -> IO [AbsFile]
- Hledger.Flow.Import.Dependencies: discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile, Digest)))
+ Hledger.Flow.Import.Dependencies: discoverDirectives :: AbsFile -> IO (ReuseDecision [(AbsFile, Digest)])
- Hledger.Flow.Import.ImportHelpers: withStagedOutput :: AbsFile -> (AbsFile -> IO ()) -> IO ()
+ Hledger.Flow.Import.ImportHelpers: withStagedOutput :: AbsFile -> (AbsFile -> IO a) -> IO a
- Hledger.Flow.Import.IncludeFiles: includesNeedUpdate :: BaseDir -> AbsDir -> [AbsFile] -> [AbsFile] -> IO Bool
+ Hledger.Flow.Import.IncludeFiles: includesNeedUpdate :: BaseDir -> AbsDir -> [AbsFile] -> [AbsFile] -> Bool -> IO Bool
- Hledger.Flow.Import.Inventory: moveToTrash :: AbsFile -> IO AbsFile
+ Hledger.Flow.Import.Inventory: moveToTrash :: AbsDir -> AbsFile -> IO AbsFile
- Hledger.Flow.Import.Snapshot: snapshotManifest :: PreparedSnapshot -> Map AbsFile Digest
+ Hledger.Flow.Import.Snapshot: snapshotManifest :: PreparedSnapshot -> [(AbsFile, Digest)]
- Hledger.Flow.Init: Created :: BaseDir -> Maybe BaseDir -> InitOutcome
+ Hledger.Flow.Init: Created :: BaseDir -> Maybe BaseDir -> RelDir -> [(RelFile, Bool)] -> InitOutcome
- Hledger.Flow.Init: runInit :: Maybe FilePath -> AbsDir -> AbsDir -> IO ()
+ Hledger.Flow.Init: runInit :: Maybe FilePath -> Maybe FilePath -> AbsDir -> AbsDir -> IO ()
- Hledger.Flow.PathHelpers: withStagedOutput :: AbsFile -> (AbsFile -> IO ()) -> IO ()
+ Hledger.Flow.PathHelpers: withStagedOutput :: AbsFile -> (AbsFile -> IO a) -> IO a
- Hledger.Flow.Status: AccountStatus :: Text -> Int -> [Integer] -> Maybe FilePath -> [FilePath] -> Int -> Bool -> Bool -> Int -> Int -> AccountStatus
+ Hledger.Flow.Status: AccountStatus :: Text -> Int -> [Integer] -> Maybe FilePath -> [FilePath] -> Int -> [FilePath] -> Bool -> Bool -> Bool -> Int -> Int -> AccountStatus
- Hledger.Flow.Status: LedgerFile :: FilePath -> Bool -> LedgerFile
+ Hledger.Flow.Status: LedgerFile :: FilePath -> Bool -> Coverage -> LedgerFile
- Hledger.Flow.Status: LedgerStatus :: BaseDir -> Maybe HledgerInfo -> Maybe LedgerFile -> [AccountStatus] -> [Misplaced] -> Int -> Bool -> Bool -> LedgerStatus
+ Hledger.Flow.Status: LedgerStatus :: BaseDir -> Maybe HledgerInfo -> Maybe LedgerFile -> Maybe AbsFile -> Maybe HledgerConf -> [ConfigConflict] -> [AccountStatus] -> [Misplaced] -> Int -> Bool -> Bool -> Bool -> Bool -> Maybe Review -> RelDir -> Maybe RelDir -> LedgerStatus
- Hledger.Flow.Status: Misplaced :: FilePath -> FilePath -> Misplaced
+ Hledger.Flow.Status: Misplaced :: FilePath -> FilePath -> StatementYear -> Bool -> Misplaced
- Hledger.Flow.Status.Render: printNext :: Maybe FilePath -> BaseDir -> IO ()
+ Hledger.Flow.Status.Render: printNext :: Maybe FilePath -> Maybe HledgerConf -> BaseDir -> IO ()
- Hledger.Flow.Status.Render: statusJSON :: SystemInfo -> Maybe HledgerConf -> LedgerStatus -> NextStep -> Value
+ Hledger.Flow.Status.Render: statusJSON :: SystemInfo -> Maybe HledgerConf -> AbsDir -> LedgerStatus -> NextStep -> Value
Files
- ChangeLog.md +16/−6
- app/Main.hs +26/−8
- hledger-flow.cabal +13/−2
- site/go-links.json +8/−1
- src/Hledger/Flow/Cache.hs +13/−6
- src/Hledger/Flow/Common.hs +48/−7
- src/Hledger/Flow/DocHelpers.hs +12/−0
- src/Hledger/Flow/HledgerConfig.hs +165/−0
- src/Hledger/Flow/Import/CSVImport.hs +1/−1
- src/Hledger/Flow/Import/Dependencies.hs +198/−64
- src/Hledger/Flow/Import/IncludeFiles.hs +4/−7
- src/Hledger/Flow/Import/Incremental.hs +39/−23
- src/Hledger/Flow/Import/Inventory.hs +8/−8
- src/Hledger/Flow/Import/ShellSources.hs +196/−0
- src/Hledger/Flow/Import/Snapshot.hs +33/−25
- src/Hledger/Flow/Init.hs +39/−11
- src/Hledger/Flow/Init/Example.hs +316/−0
- src/Hledger/Flow/InputSnapshot.hs +138/−0
- src/Hledger/Flow/LedgerSelection.hs +184/−0
- src/Hledger/Flow/PathHelpers.hs +62/−3
- src/Hledger/Flow/ReportReview.hs +199/−0
- src/Hledger/Flow/Reports.hs +27/−13
- src/Hledger/Flow/RulesTemplate.hs +127/−57
- src/Hledger/Flow/StatementScan.hs +285/−0
- src/Hledger/Flow/Status.hs +222/−46
- src/Hledger/Flow/Status/Gather.hs +133/−22
- src/Hledger/Flow/Status/Render.hs +52/−16
- test/CSVImport/IncludeFilesTests.hs +1/−0
- test/Common/Integration.hs +14/−2
- test/HledgerConfig/Unit.hs +30/−0
- test/ImportHelpers/Integration.hs +6/−4
- test/Incremental/Dependencies.hs +50/−15
- test/Incremental/Snapshot.hs +10/−5
- test/Init/Integration.hs +10/−3
- test/LedgerSelection/Unit.hs +14/−0
- test/ReportReview/Unit.hs +33/−0
- test/RulesTemplate/Unit.hs +36/−3
- test/Spec.hs +6/−0
- test/Status/Integration.hs +20/−2
- test/Status/Unit.hs +66/−24
ChangeLog.md view
@@ -31,6 +31,16 @@ existing destination belongs in release-procedure.md. --> +## 0.21.0++- `import` moves orphaned generated files to `.hledger-flow/trash/` at the ledger root instead of each account's `.trash` folder, keeping their path from the root. The trash has its own `.gitignore`. Recover what you need from existing account `.trash` folders, then delete them and any `.trash/` line in your `.gitignore`. See [generated files and recovery](https://hledger-flow.github.io/reference/input-files/#generated-files-and-recovery).+- `import` and `report` now stop before writing output if the hledger config they use supplies `-f`, `--file` or an argument file such as `@args.txt`. The error names the file and line. Remove the setting or pass a separate config with `--hledger-conf`. See [hledger config file](https://hledger-flow.github.io/reference/status/#hledger-config-file).+- If `LEDGER_FILE` points to an unrelated ledger, `report` now reads the local `all-years.journal` for aggregate reports. To keep using a custom journal, make it include `import/all-years.journal` and, if you have one, `directives.journal`. `status` shows which journal was selected. See [journal used for reports](https://hledger-flow.github.io/reference/status/#journal-used-for-reports).+- `init` creates a working example account under your username, with starter journals for manual entries, opening and closing balances, directives and prices. The [tutorial](https://hledger-flow.github.io/tutorial/) starts from it. `status` then prints a command for your shell that creates your first account's input directory, keeping real statements out of the example.+- Run `hledger-flow rules OWNER/BANK/ACCOUNT` to create a starter rules file in the account folder. `status --rules-template` also prints [starter rules for statements without a header](https://hledger-flow.github.io/reference/rules-files/#starter-rules-for-exports-without-a-header), with a sample row. Generated date formats accept days and months with or without leading zeroes.+- `status` guides more steps. It helps you [reuse existing account rules](https://hledger-flow.github.io/reference/rules-files/#reuse-existing-rules) when the main rules file needs a different name, and suggests [year folders](https://hledger-flow.github.io/reference/input-files/#year-folders) for misplaced delimited statements, with safe move commands when the destination is known. After a successful report, it points to [unknown transactions and transfers that need review](https://hledger-flow.github.io/reference/status/#review-unknown-transactions). When the imported journal uses more than one commodity and has no market price directives, it suggests adding [market prices](https://hledger-flow.github.io/reference/price-files/), with one file per currency pair.+- `import --incremental` reuses journals in more cases and notices more changes. It follows CSV rules that include shared files outside the ledger by relative path, literal journal includes in `directives.journal`, and, with `--incremental=skip-unchanged-scripts`, [sourced shell helpers](https://hledger-flow.github.io/reference/incremental-import/#sourced-shell-helpers). Editing one of these files rebuilds the statements that depend on it. More `hledger.conf` settings allow reuse. When a setting such as a date relative to today or a reference to an untracked file prevents it, the notice quotes the setting and its line. See [what gets checked](https://hledger-flow.github.io/reference/incremental-import/#what-gets-checked).+ ## 0.20.1 - Use `hledger-flow import --incremental` with hledger 1.52.4 or later to reuse journals when their inputs, including nested rules files, still match. Choose `--incremental=skip-unchanged-scripts` to also reuse the outputs of unchanged scripts. See the [incremental import reference](https://hledger-flow.github.io/reference/incremental-import/).@@ -38,7 +48,7 @@ - On macOS, incremental import checks the hledger executable faster and still detects a replacement build that reports the same version. - `hledger-flow import --verbose` and `hledger-flow report --verbose` are easier to read. Each statement's lines appear together, files are labelled- by what import looked for, and every hledger or hook command is shown ready+ by what import looked for, and every hledger or import script command is shown ready to paste into a shell at the base directory, with paths there starting with `./`. On a terminal, the output uses colour unless `NO_COLOR` is set.@@ -157,9 +167,9 @@ Each binary archive now includes a [quick-start](https://github.com/apauley/hledger-flow/blob/master/QUICKSTART.md) covering installation, download verification and a first import. - Windows imports now run `preprocess.ps1` and `construct.ps1` directly- with PowerShell. Native hooks can use `preprocess.exe` and `construct.exe`.- Existing extensionless hooks take precedence. See- [Windows import hooks](https://github.com/apauley/hledger-flow/blob/master/docs/README.md#windows-import-hooks).+ with PowerShell. Native executables can be named `preprocess.exe` and `construct.exe`.+ Existing extensionless scripts take precedence. See+ [Windows import scripts](https://github.com/apauley/hledger-flow/blob/master/docs/README.md#windows-import-hooks). - Manual-only years and ledgers now get include files without dummy inputs, including `_manual_` files at the import, owner, bank, and account levels. Fixes [#103](https://github.com/apauley/hledger-flow/issues/103).@@ -180,9 +190,9 @@ ## 0.17.0 -- If you use a custom `preprocess` hook, write to the path supplied as its+- If you use a custom `preprocess` script, write to the path supplied as its second argument. It now points into a temporary directory and keeps the- final filename and extension. Failed hooks preserve the previous+ final filename and extension. Failed scripts preserve the previous preprocessed CSV or constructed journal. - If you use the `Hledger.Flow.*` library, update calls that use Turtle types to use `path` and `Text` types. The `TurtlePath` alias has been
app/Main.hs view
@@ -54,10 +54,11 @@ import Hledger.Flow.Status.Render ( printStatus ) import qualified Data.Text as Text import Path.IO ( getCurrentDir, resolveDir' )+import Hledger.Flow.HledgerConfig (ConfigUse (..), requireSafeConfig) import Hledger.Flow.Init ( runInit ) import Hledger.Flow.DocHelpers ( DocLink (..), docURL, siteURL ) import Hledger.Flow.Logging ( useColor )-import Hledger.Flow.RulesTemplate ( rulesTemplateFor )+import Hledger.Flow.RulesTemplate ( rulesTemplateFor, runRules ) import Data.Maybe (fromMaybe) import Hledger.Flow.Internals (availableCores, versionInfo, systemInfo) import Hledger.Flow.BaseDir ( determineBaseDir, findBaseDir )@@ -92,7 +93,7 @@ newtype InitParams = InitParams { initDir :: Maybe FilePath } deriving (Show) -data Command = Import ImportParams | Report ReportParams | Status StatusParams | Init InitParams deriving (Show)+data Command = Import ImportParams | Report ReportParams | Status StatusParams | Rules String (Maybe FilePath) | Init InitParams deriving (Show) data MainParams = MainParams { verbosity :: Int , hledgerPathOpt :: Maybe FilePath@@ -151,15 +152,23 @@ T.putStrLn $ versionInfo sysInfo Command mainParams' (Just (Import subParams)) -> do (base, runDir) <- determineBaseDir (maybeImportBaseDir subParams)- withReleaseNotice base $ (toRuntimeOptionsImport base runDir mainParams' subParams >>= importCSVs) `catch` reportFatal+ opts <- toRuntimeOptionsImport base runDir mainParams' subParams+ requireSafeConfig ImportConfig opts+ withReleaseNotice base $ importCSVs opts `catch` reportFatal Command mainParams' (Just (Report subParams)) -> do (base, _) <- determineBaseDir (maybeReportBaseDir subParams)- withReleaseNotice base $ (toRuntimeOptionsReport base mainParams' subParams >>= generateReports) `catch` reportFatal+ opts <- toRuntimeOptionsReport base mainParams' subParams+ requireSafeConfig ReportConfig opts+ withReleaseNotice base $ generateReports opts `catch` reportFatal Command mainParams' (Just (Status subParams)) -> runStatus mainParams' subParams+ Command mainParams' (Just (Rules account dir)) -> do+ base <- statusBaseDir dir+ conf <- findHledgerConf (hledgerConfOpt mainParams')+ runRules conf base (Text.pack account) Command mainParams' (Just (Init subParams)) -> do cwd <- getCurrentDir dir <- maybe (return cwd) resolveDir' (initDir subParams)- runInit (hledgerPathOpt mainParams') cwd dir+ runInit (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') cwd dir Command mainParams' Nothing -> runBare mainParams' -- | (name, one-line description) of each command, in display order.@@ -167,6 +176,7 @@ commandSummaries = [ ("status", "Show what's in your ledger and what to do next"), ("init", "Start a new ledger in a folder"),+ ("rules", "Create a starter rules file for an account"), ("import", "Turn statements into journals"), ("report", "Generate reports from your journals") ]@@ -184,14 +194,18 @@ runStatus :: MainParams -> StatusParams -> IO () runStatus mainParams' params = do- cwd <- getCurrentDir- dir <- maybe (return cwd) (resolveExistingDir cwd) (statusDir params)- base <- findBaseDir dir >>= maybe (throwIO (MissingBaseDir dir)) return+ base <- statusBaseDir (statusDir params) case statusRulesTemplate params of Just account -> rulesTemplateFor base (Text.pack account) >>= either (ioError . userError . Text.unpack) T.putStr Nothing | statusJson params -> printStatus (hledgerPathOpt mainParams') (hledgerConfOpt mainParams') True base | otherwise -> printInteractiveStatus mainParams' base +statusBaseDir :: Maybe FilePath -> IO BaseDir+statusBaseDir target = do+ cwd <- getCurrentDir+ dir <- maybe (return cwd) (resolveExistingDir cwd) target+ findBaseDir dir >>= maybe (throwIO (MissingBaseDir dir)) return+ printInteractiveStatus :: MainParams -> BaseDir -> IO () printInteractiveStatus params base = withCachedReleaseNotice base $ printStatus (hledgerPathOpt params) (hledgerConfOpt params) False base@@ -264,6 +278,10 @@ ("hledger-flow status --rules-template me/mybank/checking", "Print a starter rules file for one account") ] <> subcommand "init" GettingStarted (Init . InitParams <$> optional (argument str (metavar "DIR" <> help "The folder for the new ledger, created if missing. Default: the current folder."))) [ ("hledger-flow init ~/finances", "Start a ledger in ~/finances") ]+ <> subcommand "rules" RulesFiles+ (Rules <$> argument str (metavar "ACCOUNT" <> help "Account as owner/bank/account.")+ <*> optional (argument str (metavar "DIR" <> help "Inspect the ledger containing this folder.")))+ [ ("hledger-flow rules me/mybank/checking", "Write a starter rules file for this account") ] <> subcommand "import" InputFiles (Import <$> subcommandParserImport colour) [ ("hledger-flow import", "Import the current folder"), ("hledger-flow import import/me/mybank", "Import one bank's statements"),
hledger-flow.cabal view
@@ -5,7 +5,7 @@ -- see: https://github.com/sol/hpack name: hledger-flow-version: 0.20.1+version: 0.21.0 synopsis: An hledger workflow focusing on automated statement import and classification. description: hledger-flow uses hledger's CSV rules to turn each bank statement into a separate journal. It combines those journals with your manual entries for@@ -38,6 +38,7 @@ Hledger.Flow.DateTime Hledger.Flow.DocHelpers Hledger.Flow.Environment+ Hledger.Flow.HledgerConfig Hledger.Flow.Import.Cache Hledger.Flow.Import.Conversion Hledger.Flow.Import.CSVImport@@ -48,16 +49,22 @@ Hledger.Flow.Import.Incremental Hledger.Flow.Import.Inventory Hledger.Flow.Import.PrivateDirectory+ Hledger.Flow.Import.ShellSources Hledger.Flow.Import.Snapshot Hledger.Flow.Import.Types Hledger.Flow.Init+ Hledger.Flow.Init.Example+ Hledger.Flow.InputSnapshot Hledger.Flow.Internals+ Hledger.Flow.LedgerSelection Hledger.Flow.Logging Hledger.Flow.PathHelpers Hledger.Flow.ReleaseNotice+ Hledger.Flow.ReportReview Hledger.Flow.Reports Hledger.Flow.RulesTemplate Hledger.Flow.RuntimeOptions+ Hledger.Flow.StatementScan Hledger.Flow.Status Hledger.Flow.Status.Gather Hledger.Flow.Status.Render@@ -68,7 +75,8 @@ src ghc-options: -Wall build-depends:- aeson >=2.2.5.1 && <2.3+ ShellCheck ==0.10.*+ , aeson >=2.2.5.1 && <2.3 , ansi-terminal >=1.1.5 && <1.2 , async >=2.2.6 && <2.3 , base >=4.20 && <4.22@@ -136,14 +144,17 @@ CSVImport.Integration CSVImport.Unit Environment.Unit+ HledgerConfig.Unit ImportHelpers.Integration Incremental.Cache Incremental.Dependencies Incremental.Platform Incremental.Snapshot Init.Integration+ LedgerSelection.Unit PathHelpers.Unit ReleaseNotice.Unit+ ReportReview.Unit RulesTemplate.Unit RuntimeOptions.Unit Status.Integration
site/go-links.json view
@@ -2,10 +2,17 @@ "docs": "/quickstart/", "getting-started": "/quickstart/#import-your-first-statement", "input-files": "/reference/input-files/",+ "statement-year-folders": "/reference/input-files/#year-folders", "incremental-import-fallback": "/reference/incremental-import/#when-reuse-is-unavailable", "rules-files": "/reference/rules-files/",+ "price-files": "/reference/price-files/",+ "manual-journals": "/reference/manual-journals/",+ "balances": "/reference/balances/", "preprocess-script": "/reference/preprocess/", "set-ledger-file": "/quickstart/#import-your-first-statement", "status": "/reference/status/",- "reports": "/tutorial/part-2/#reports"+ "reports": "/tutorial/part-3/#reports",+ "review-unknown-transactions": "/reference/status/#review-unknown-transactions",+ "review-transfers": "/reference/status/#review-transfers",+ "sample-ledger": "/guides/sample-ledger/" }
src/Hledger/Flow/Cache.hs view
@@ -1,7 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} -module Hledger.Flow.Cache (cacheDir, initializeCache) where+module Hledger.Flow.Cache (cacheDir, initializeCache, trashDir, initializeIgnoredDir) where import Control.Monad (unless) import qualified Data.ByteString as BS@@ -14,14 +14,21 @@ cacheDir :: AbsDir -> AbsDir cacheDir base = base P.</> [P.reldir|.hledger-flow/cache|] +-- | Where import moves generated files whose statement is gone.+trashDir :: AbsDir -> AbsDir+trashDir base = base P.</> [P.reldir|.hledger-flow/trash|]+ initializeCache :: AbsDir -> IO ()-initializeCache base = do- let cache = cacheDir base- ignore = cache P.</> [P.relfile|.gitignore|]- lock = cache P.</> [P.relfile|initialize.lock|]+initializeCache = initializeIgnoredDir . cacheDir++-- | Create a directory whose contents Git ignores.+initializeIgnoredDir :: AbsDir -> IO ()+initializeIgnoredDir dir = do+ let ignore = dir P.</> [P.relfile|.gitignore|]+ lock = dir P.</> [P.relfile|initialize.lock|] exists <- P.doesFileExist ignore unless exists $ do- P.createDirIfMissing True cache+ P.createDirIfMissing True dir -- Serialize first use across processes. Closing the handle releases the lock. withBinaryFile (P.toFilePath lock) AppendMode $ \handle -> do hLock handle ExclusiveLock
src/Hledger/Flow/Common.hs view
@@ -23,10 +23,10 @@ import Hledger.Flow.DateTime (timeAction) import Hledger.Flow.BaseDir (relativeToBase) import Hledger.Flow.Logging-import Hledger.Flow.PathHelpers (AbsFile, RelDir, literalGlobPath, withStagedOutput)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile, literalGlobPath, withStagedOutput) import qualified Hledger.Flow.RuntimeOptions as RT import Hledger.Flow.Types-import Path (reldir, relfile, toFilePath)+import Path (reldir, relfile, stripProperPrefix, toFilePath) import qualified Path import qualified Path.IO as Path import qualified System.Directory as Directory@@ -269,6 +269,42 @@ logFun errorMessage exitWith (ExitFailure exitStatus) +-- | Move to a file path, with the same collision rules in either shell.+moveFileCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> RelFile -> RelFile -> T.Text+moveFileCommand shell homes cwd base source target = case shell of+ Posix ->+ makeDirectory Posix (path dir) <> " && [ ! -e " <> dest <> " ] && [ ! -L " <> dest+ <> " ] && mv -n " <> path src <> " " <> dest+ PowerShell ->+ makeDirectory PowerShell (literal dir) <> " -ErrorAction Stop | Out-Null; "+ <> "[System.IO.File]::Move(" <> nativePath src <> ", " <> nativePath file <> ")"+ where+ atRoot = cwd == base+ fromHere rel = if atRoot then toFilePath rel else toFilePath (base Path.</> rel)+ src = fromHere source+ file = fromHere target+ dir = FilePath.takeDirectory file+ literal = powerShellLiteral . T.pack+ -- .NET resolves paths against the process directory, not PowerShell's location.+ nativePath name+ | atRoot = "(Join-Path $PWD.ProviderPath " <> literal name <> ")"+ | otherwise = literal name+ path name = fromMaybe (shellArgument shell (T.pack name)) (homeShellPath shell homes name)+ dest = path file++createDirectoryCommand :: Shell -> [FilePath] -> AbsDir -> AbsDir -> T.Text+createDirectoryCommand shell homes cwd dir = makeDirectory shell argument+ where+ absolute = FilePath.dropTrailingPathSeparator (toFilePath dir)+ relative = FilePath.dropTrailingPathSeparator . toFilePath <$> stripProperPrefix cwd dir+ argument = case relative of+ Just path -> shellArgument shell (T.pack path)+ Nothing -> fromMaybe (shellArgument shell (T.pack absolute)) (homeShellPath shell homes absolute)++makeDirectory :: Shell -> T.Text -> T.Text+makeDirectory Posix argument = "mkdir -p " <> argument+makeDirectory PowerShell argument = "New-Item -ItemType Directory -Force -Path " <> argument+ -- | An external program hledger-flow runs, described for the people reading its output. data External = External { -- | Who printed the output and who failed: @hledger@, or a hook script's path.@@ -549,12 +585,17 @@ return [] Right years -> return years +includePreamble :: T.Text+includePreamble = "### Generated by hledger-flow - DO NOT EDIT ###\n"+ includeYears' :: T.Text -> Either T.Text [Integer]-includeYears' txt = case partitionEithers (includeYears'' txt) of- (errors, []) -> do- let msg = "Unable to extract years from the following text:\n" <> txt <> "\nErrors:\n" <> T.intercalate "\n" (map T.pack errors)- Left msg- (_, years) -> Right years+includeYears' txt+ | T.stripEnd txt == T.stripEnd includePreamble = Right []+ | otherwise = case partitionEithers (includeYears'' txt) of+ (errors, []) -> do+ let msg = "Unable to extract years from the following text:\n" <> txt <> "\nErrors:\n" <> T.intercalate "\n" (map T.pack errors)+ Left msg+ (_, years) -> Right years includeYears'' :: T.Text -> [Either String Integer] includeYears'' txt = map extractDigits (T.lines txt)
src/Hledger/Flow/DocHelpers.hs view
@@ -20,11 +20,17 @@ | GettingStarted | IncrementalImportFallback | InputFiles+ | StatementYearFolders | RulesFiles+ | PriceFiles+ | ManualJournals+ | Balances | PreprocessScript | SetLedgerFile | StatusCommand | Reports+ | ReviewUnknownTransactions+ | ReviewTransfers deriving (Bounded, Enum, Eq, Show) docSlug :: DocLink -> Text@@ -32,11 +38,17 @@ docSlug GettingStarted = "getting-started" docSlug IncrementalImportFallback = "incremental-import-fallback" docSlug InputFiles = "input-files"+docSlug StatementYearFolders = "statement-year-folders" docSlug RulesFiles = "rules-files"+docSlug PriceFiles = "price-files"+docSlug ManualJournals = "manual-journals"+docSlug Balances = "balances" docSlug PreprocessScript = "preprocess-script" docSlug SetLedgerFile = "set-ledger-file" docSlug StatusCommand = "status" docSlug Reports = "reports"+docSlug ReviewUnknownTransactions = "review-unknown-transactions"+docSlug ReviewTransfers = "review-transfers" siteURL :: Text siteURL = "https://hledger-flow.github.io/"
+ src/Hledger/Flow/HledgerConfig.hs view
@@ -0,0 +1,165 @@+{-# LANGUAGE OverloadedStrings #-}++module Hledger.Flow.HledgerConfig+ ( ConfigUse (..),+ ConfigConflict,+ ConfigLine (..),+ parseConfigLines,+ configConflicts,+ firstConfigConflict,+ findConfigConflict,+ findConfigConflicts,+ renderConfigConflict,+ requireSafeConfig,+ )+where++import qualified Data.ByteString as B+import Data.Char (isSpace)+import Data.List (find)+import Data.Maybe (maybeToList)+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import qualified Data.Text.Read as TR+import Hledger.Flow.Common (hledgerVersionNumber)+import qualified Hledger.Flow.RuntimeOptions as RT+import Hledger.Flow.Types (HledgerConf, HledgerInfo (..), hledgerConfPath)++data ConfigLine = ConfigLine+ { configSection :: T.Text,+ configLineNumber :: Int,+ configSource :: T.Text,+ configArguments :: [T.Text],+ configStartsSection :: Bool+ }+ deriving (Eq, Show)++parseConfigLines :: Bool -> [T.Text] -> [ConfigLine]+parseConfigLines allowIndented = parseConfigLinesWith allowIndented configWords++parseConfigLinesWith :: Bool -> (T.Text -> [T.Text]) -> [T.Text] -> [ConfigLine]+parseConfigLinesWith allowIndented splitWords = go "general" . zip [1 ..]+ where+ go _ [] = []+ go section ((n, raw) : more)+ | T.null body || "#" `T.isPrefixOf` body = go section more+ | allowIndented || raw == T.stripStart raw,+ Just start <- T.stripPrefix "[" body,+ (name, end) <- T.breakOn "]" start, not (T.null end) =+ let next = T.strip name+ in ConfigLine next n body (splitWords (T.drop 1 end)) True : go next more+ | otherwise = ConfigLine section n body (splitWords body) False : go section more+ where body = T.strip raw++data ConfigUse = ImportConfig | ReportConfig | BalanceConfig+ deriving (Eq, Show)++data ConfigConflict = ConfigConflict FilePath ConfigLine Bool+ deriving (Eq, Show)++configConflicts :: FilePath -> [ConfigLine] -> [ConfigConflict]+configConflicts file rows = concatMap fromSection sections+ where+ sections = ["general", "print", "balancesheet", "incomestatement", "balance", "accounts"]+ fromSection section = scan+ [ (row, token)+ | row <- reverse (lastDefinition (reverse (filter ((== section) . configSection) rows))),+ token <- configArguments row+ ]+ lastDefinition [] = []+ lastDefinition (row : more) =+ row : if configStartsSection row then [] else lastDefinition more+ scan [] = []+ scan ((row, token) : remaining)+ | token == "--" = []+ | token == "--file" || "--file=" `T.isPrefixOf` token =+ ConfigConflict file row True : scan (if token == "--file" then drop 1 remaining else remaining)+ | Just (option, value) <- shortValue token =+ [ConfigConflict file row True | option == 'f'] ++ scan (if T.null value then drop 1 remaining else remaining)+ | "@" `T.isPrefixOf` token = ConfigConflict file row False : scan remaining+ | token `elem` takesValue = scan (drop 1 remaining)+ | otherwise = scan remaining+ takesValue =+ [ "--alias", "--pivot", "--begin", "--end", "--period", "--depth",+ "--output-file", "--output-format", "--rules", "--rules-file",+ "--today", "--value", "--conf"+ ]+ -- hledger accepts bundles such as -If. The first short option that takes+ -- a value consumes the rest of the token, or the following token.+ shortValue token+ | "--" `T.isPrefixOf` token = Nothing+ | otherwise = T.stripPrefix "-" token >>= T.uncons . T.dropWhile (`notElem` ("fbepoOXcmw" :: String))++firstConfigConflict :: ConfigUse -> [ConfigConflict] -> Maybe ConfigConflict+firstConfigConflict use = find (\(ConfigConflict _ row _) -> configSection row `elem` relevant)+ where+ relevant = "general" : case use of+ ImportConfig -> ["print"]+ ReportConfig -> ["balancesheet", "incomestatement", "balance", "print", "accounts"]+ BalanceConfig -> ["balance"]++readConfigConflicts :: HledgerInfo -> HledgerConf -> IO [ConfigConflict]+readConfigConflicts info conf = do+ let file = hledgerConfPath conf+ indented = case TR.decimal (hledgerVersionNumber (hlVersion info)) of+ Right (major, rest)+ | Just suffix <- T.stripPrefix "." rest, Right (minor, _) <- TR.decimal suffix ->+ (major, minor) >= ((1, 99) :: (Int, Int))+ _ -> True+ splitWords = if indented then configWords else legacyConfigWords+ bytes <- B.readFile file+ case TE.decodeUtf8' bytes of+ Left err -> ioError (userError ("Cannot read hledger config " ++ file ++ ": " ++ show err))+ Right contents -> return (configConflicts file+ (parseConfigLinesWith indented splitWords (T.lines (T.dropWhile (== '\xFEFF') contents))))++findConfigConflict :: HledgerInfo -> ConfigUse -> Maybe HledgerConf -> IO (Maybe ConfigConflict)+findConfigConflict info use = fmap (firstConfigConflict use) . findConfigConflicts info++findConfigConflicts :: HledgerInfo -> Maybe HledgerConf -> IO [ConfigConflict]+findConfigConflicts info = maybe (pure []) (readConfigConflicts info)++renderConfigConflict :: ConfigConflict -> T.Text+renderConfigConflict (ConfigConflict file row addsJournal) =+ T.pack file <> " line " <> T.pack (show (configLineNumber row)) <> " [" <> configSection row <> "] "+ <> (if addsJournal then "adds a journal: " else "reads arguments from another file: ")+ <> configSource row <> "\n"+ <> (if addsJournal then "hledger-flow picks its own journals, so this would mix another journal into the ledger."+ else "hledger-flow cannot check which journals that argument file would add.")+ <> "\nRemove that line, or pass --hledger-conf with a config that doesn't have it."++requireSafeConfig :: ConfigUse -> RT.RuntimeOptions -> IO ()+requireSafeConfig use opts = case RT.hledgerConf opts of+ Nothing -> return ()+ conf -> do+ info <- RT.hledgerInfo opts+ conflict <- findConfigConflict info use conf+ maybe (return ()) (ioError . userError . T.unpack . renderConfigConflict) conflict++-- Config arguments can quote spaces or a #; neither quotes nor comments are+-- arguments. Leave malformed quoting for hledger to diagnose.+configWords :: T.Text -> [T.Text]+configWords = map T.pack . wordsOutside Nothing Nothing . T.unpack+ where+ wordsOutside _ word [] = map reverse (maybeToList word)+ wordsOutside quote word (c:cs)+ | Just c == quote = wordsOutside Nothing word cs+ | quote == Nothing, c `elem` ['\'', '"'] = wordsOutside (Just c) (Just (maybe "" id word)) cs+ | quote == Nothing, c == '#' = map reverse (maybeToList word)+ | quote == Nothing, isSpace c =+ map reverse (maybeToList word) ++ wordsOutside Nothing Nothing cs+ | otherwise = wordsOutside quote (Just (c : maybe "" id word)) cs++-- Older hledger config files start comments at any # and only interpret a+-- quote at the start of a word. Quotes within an unquoted word stay literal.+legacyConfigWords :: T.Text -> [T.Text]+legacyConfigWords = go . T.stripStart . T.takeWhile (/= '#')+ where+ go input = case T.uncons input of+ Nothing -> []+ Just (quote, contents) | quote == '\'' || quote == '"' ->+ let (word, end) = T.break (== quote) contents+ in word : case T.uncons (T.drop 1 end) of+ Just (next, _) | isSpace next -> go (T.stripStart (T.drop 1 end))+ _ -> []+ _ -> let (word, rest) = T.break isSpace input in word : go (T.stripStart rest)
src/Hledger/Flow/Import/CSVImport.hs view
@@ -94,7 +94,7 @@ cached = if or [refresh | (_, _, refresh) <- outcomes] then Nothing else Just inventory ((refreshed, orphans, journalsOnDisk), journalFindTime) <- timeAction $ reconciliationInventory importRoot cached forM_ (filter selected orphans) $ \file -> do- target <- moveToTrash file+ target <- moveToTrash (baseDir opts) file channelOutLn ch $ "Moved orphaned output " <> relative file <> " to " <> relative target withLogBlock opts True ch "Include files" $ \_ -> do (_, writeIncludeTime1) <- timeAction $ writeIncludesFromInventory opts ch importRoot refreshed journalsOnDisk
src/Hledger/Flow/Import/Dependencies.hs view
@@ -4,8 +4,9 @@ module Hledger.Flow.Import.Dependencies ( Release (..), FallbackReason (..), ReuseDecision (..), Converter (..),- ruleIncludes, nestedRuleIncludes, resolveRuleInclude, discoverRules, selectRules, isSeparatedValues,- checkedIO, atPath, checkArguments, discoverConfig, discoverDirectives, checkConfig, checkDirectives,+ IncludeFile (..), IncludeNode (..), readIncludeTree, scanIncludes, resolveInclude,+ ruleIncludes, nestedRuleIncludes, discoverRules, selectRules, isSeparatedValues,+ checkedIO, atPath, checkArguments, discoverConfig, discoverDirectives, checkConfig, journalIncludes, newConverterProbe, converterProbeWith ) where @@ -14,7 +15,7 @@ import qualified Data.ByteString as B import Data.Char (isAlpha, isAlphaNum, isAscii, isControl, isDigit, isSpace, toLower, GeneralCategory(CurrencySymbol), generalCategory) import Data.List (isPrefixOf)-import Data.Maybe (mapMaybe)+import Data.Maybe (fromMaybe, mapMaybe, maybeToList) import qualified Data.Set as Set import qualified Data.Text as T import qualified Data.Text.Encoding as TE@@ -22,13 +23,14 @@ import Data.Time (getCurrentTimeZone, timeZoneMinutes) import GHC.IO.Encoding (getLocaleEncoding, getFileSystemEncoding, getForeignEncoding) import Hledger.Flow.Common (firstConfirmedFile, lookupHledgerConf, probeHledgerVersion, hledgerVersionNumber)+import Hledger.Flow.HledgerConfig (ConfigLine (..), parseConfigLines) import Hledger.Flow.Import.Fingerprint import Hledger.Flow.Import.ImportHelpers (rulesFileCandidates) import Hledger.Flow.Import.Types (ImportDirs) import Hledger.Flow.PathHelpers (AbsDir, AbsFile) import qualified Hledger.Flow.RuntimeOptions as RT import Hledger.Flow.Types (HledgerConf(..), hledgerConfPath)-import Path (Abs, parseAbsFile, toFilePath)+import Path (Abs, parent, parseAbsFile, parseRelDir, parseRelFile, toFilePath, (</>)) import qualified System.Directory as D import qualified System.FilePath as F import System.Exit (ExitCode(..))@@ -47,6 +49,8 @@ data FallbackReason = FallbackReason {reasonScope :: T.Text, reasonDetail :: T.Text} | FileReason {reasonScope :: T.Text, reasonFile :: FilePath, reasonDetail :: T.Text}+ -- The notice quotes the entry, so people see exactly what to change.+ | ConfigEntry {reasonScope :: T.Text, reasonFile :: FilePath, reasonDetail :: T.Text} | OrdinaryInputError {reasonScope :: T.Text, reasonDetail :: T.Text} | CyclicRules {reasonScope :: T.Text, reasonDetail :: T.Text} deriving (Eq, Ord, Show)@@ -108,60 +112,130 @@ dropUtf8Bom :: B.ByteString -> B.ByteString dropUtf8Bom b = maybe b id (B.stripPrefix (B.pack [0xef,0xbb,0xbf]) b) --- Walk leading .. components relative to the including file's spelled path.--- Verify the result against the filesystem's resolution, which can differ when a--- symlink precedes .. . An internal symlink is mirrored under its original name.-resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)-resolveRuleInclude base parent child = checkedIO "uncertain rules include" raw $+data IncludeFile = CsvRules Release | JournalDirectives+ deriving (Eq, Show)++scanIncludes :: IncludeFile -> Bool -> AbsFile -> B.ByteString -> ReuseDecision [FilePath]+scanIncludes (CsvRules release) isRoot = if isRoot then ruleIncludes release else nestedRuleIncludes release+scanIncludes JournalDirectives _ = journalIncludes++fileKind :: IncludeFile -> T.Text+fileKind (CsvRules _) = "rules"+fileKind JournalDirectives = "journal"++resolveInclude :: IncludeFile -> AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)+resolveInclude kind base including child = checkedIO ("uncertain " <> label) raw $ if F.hasDrive child || not (F.isRelative child) || any F.isPathSeparator (take 1 child)- then pure (atPath "absolute rules include" child (T.pack (toFilePath parent)))- else do+ then pure (atPath ("absolute " <> label) child (T.pack (toFilePath including)))+ else case kind of+ CsvRules _ -> resolveRules+ JournalDirectives -> resolveJournal+ where+ -- Keep the filename as opened: a file symlink's nested includes are+ -- relative to its containing directory. A path without ".." is kept as+ -- spelled. Otherwise the OS resolves the parent, so a directory link+ -- before ".." leads where hledger's open would.+ resolveRules = case parseAbsFile raw of+ Just file -> pure (Reusable file)+ Nothing -> do+ exists <- D.doesFileExist raw+ if exists+ then do+ dir <- D.canonicalizePath (F.takeDirectory raw)+ pure (maybe outside Reusable (parseAbsFile (dir F.</> F.takeFileName raw)))+ else pure (atPath "could not read rules file" raw (T.pack (toFilePath including)))+ resolveJournal = do let (ups, rest) = span (== "..") (filter (/= ".") (F.splitDirectories child))- lexical = foldr (const F.takeDirectory) (F.takeDirectory (toFilePath parent)) ups F.</> F.joinPath rest+ lexical = foldr (const F.takeDirectory) directory ups F.</> F.joinPath rest if ".." `elem` rest || null rest || not (inside (toFilePath base) lexical) then pure outside else do actual <- D.canonicalizePath raw mirror <- D.canonicalizePath lexical root <- D.canonicalizePath (toFilePath base)- if F.equalFilePath actual mirror && inside root actual+ -- hledger resolves journal includes from the canonical including file.+ -- The snapshot preserves its spelled directory and journal reader.+ source <- D.canonicalizePath (toFilePath including)+ dir <- D.canonicalizePath directory+ let compatible = F.equalFilePath dir (F.takeDirectory source) &&+ F.equalFilePath actual (root F.</> F.makeRelative (toFilePath base) lexical)+ if compatible && F.equalFilePath actual mirror && inside root actual then pure (maybe outside Reusable (parseAbsFile lexical)) else pure outside- where- raw = F.takeDirectory (toFilePath parent) F.</> child- outside = atPath "rules include escapes the ledger or cannot be mirrored" raw (T.pack (toFilePath parent))+ directory = F.takeDirectory (toFilePath including)+ raw = directory F.</> child+ label = fileKind kind <> " include"+ outside = atPath (label <> " escapes the ledger or cannot be mirrored") raw (T.pack (toFilePath including)) inside root path = F.splitDirectories (F.dropTrailingPathSeparator root) `isPrefixOf` F.splitDirectories path discoverRules :: AbsDir -> Release -> AbsFile -> IO (ReuseDecision [(AbsFile,Digest)])-discoverRules base release = walk Set.empty+discoverRules base release = discoverTree base (CsvRules release)++discoverTree :: AbsDir -> IncludeFile -> AbsFile -> IO (ReuseDecision [(AbsFile,Digest)])+discoverTree base kind root = fmap (map dependency) <$> readIncludeTree base kind root+ where dependency node = (includeSource node, digestBytes (includeBytes node))++-- A rules file can be read through a directory link followed by "..". Its+-- source and its position in a link-free copy can then be different.+data IncludeNode = IncludeNode+ { includeSource :: AbsFile+ , includePosition :: AbsFile+ , includeDirectories :: [AbsDir]+ , includeBytes :: B.ByteString+ }++-- Walk the spelling that hledger will traverse in the private copy. Retain+-- directories visited before "..", even when no included file lives there.+rulesPosition :: AbsFile -> FilePath -> Maybe (AbsFile, [AbsDir])+rulesPosition including spelling = do+ name <- parseRelFile (F.takeFileName spelling)+ (dir, visited) <- foldM step (parent including, []) (F.splitDirectories (F.takeDirectory spelling))+ pure (dir </> name, visited) where- walk ancestors path = checkedIO "could not read rules file" (toFilePath path) $ do+ step acc "." = Just acc+ step (dir, visited) ".."+ | parent dir /= dir = Just (parent dir, dir : parent dir : visited)+ | otherwise = Nothing+ step (dir, visited) component = do+ child <- parseRelDir component+ pure (dir </> child, visited)++readIncludeTree :: AbsDir -> IncludeFile -> AbsFile -> IO (ReuseDecision [IncludeNode])+readIncludeTree base kind root = walk Set.empty root [] root+ where+ nativeRules = case kind of+ CsvRules release -> release /= V1996+ JournalDirectives -> False+ walk ancestors position dirs path = checkedIO ("could not read " <> fileKind kind <> " file") (toFilePath path) $ do canonical <- D.canonicalizePath (toFilePath path) directory <- D.canonicalizePath (F.takeDirectory (toFilePath path)) -- The same rules bytes can resolve includes in different directories. -- In 1.52.4 the first read also decodes line endings differently.- let node = (canonical, directory, release /= V1996 && Set.null ancestors)- if Set.member node ancestors && release /= V1996+ let node = (canonical, directory, nativeRules && Set.null ancestors)+ if Set.member node ancestors && nativeRules then pure $ Fallback (CyclicRules "cyclic rules include" (T.pack (toFilePath path))) else if Set.member node ancestors || Set.size ancestors >= 256- then pure (atPath "cyclic or excessively deep rules include" (toFilePath path) "")+ then pure (atPath ("cyclic or excessively deep " <> fileKind kind <> " include") (toFilePath path) "") else do bytes <- B.readFile (toFilePath path)- let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes- case scan release path bytes of+ case scanIncludes kind (Set.null ancestors) path bytes of Fallback r -> pure (Fallback r) Reusable children -> do- nested <- descend (Set.insert node ancestors) path children- pure (((path,digestBytes bytes):) <$> nested)- descend _ _ [] = pure (pure [])- descend ancestors path (c:cs) = do- selected <- resolveRuleInclude base path c+ nested <- descend (Set.insert node ancestors) path position children+ pure ((IncludeNode path position dirs bytes :) <$> nested)+ descend _ _ _ [] = pure (pure [])+ descend ancestors path position (c:cs) = do+ selected <- resolveInclude kind base path c tree <- case selected of Fallback r -> pure (Fallback r)- Reusable file -> walk ancestors file+ Reusable file -> case kind of+ JournalDirectives -> walk ancestors file [] file+ CsvRules _ -> case rulesPosition position c of+ Just (child, dirs) -> walk ancestors child dirs file+ Nothing -> pure (atPath "rules include cannot be mirrored" c (T.pack (toFilePath path))) case tree of Fallback r -> pure (Fallback r)- Reusable files -> fmap (files ++) <$> descend ancestors path cs+ Reusable files -> fmap (files ++) <$> descend ancestors path position cs selectRules :: AbsFile -> ImportDirs Abs -> IO (ReuseDecision (Maybe AbsFile)) selectRules csv dirs = checkedIO "could not select rules file" (toFilePath csv) $ do@@ -196,12 +270,12 @@ Nothing -> pure (atPath "uncertain config path" path "") Just absPath -> fmap Just <$> readDependency (checkConfig release) absPath -discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile,Digest)))+discoverDirectives :: AbsFile -> IO (ReuseDecision [(AbsFile,Digest)]) discoverDirectives file = checkedIO "could not discover directives" (toFilePath file) $ do exists <- firstConfirmedFile [toFilePath file] case exists of- Nothing -> pure (pure Nothing)- Just _ -> fmap Just <$> readDependency (checkDirectives V1996) file+ Nothing -> pure (pure [])+ Just _ -> discoverTree (parent file) JournalDirectives file readDependency :: (AbsFile -> B.ByteString -> ReuseDecision ()) -> AbsFile -> IO (ReuseDecision (AbsFile,Digest)) readDependency check path = checkedIO "could not read dependency" (toFilePath path) $ do@@ -210,41 +284,101 @@ checkConfig :: Release -> AbsFile -> B.ByteString -> ReuseDecision () checkConfig release path bytes = do- ls <- physicalLines path bytes- () <$ foldM line "general" (zip [1 :: Int ..] ls)+ ls <- physicalLines path (dropUtf8Bom bytes)+ let settings =+ [ (configLineNumber row, word)+ | row <- parseConfigLines (release == V1996) ls,+ configSection row `elem` ["general", "print"],+ word <- configArguments row+ ]+ case configDependency settings of+ Nothing -> pure ()+ Just n -> Fallback $ ConfigEntry "non-deterministic hledger config entry" (toFilePath path)+ ("line " <> T.pack (show n) <> ": " <> T.strip (ls !! (n - 1)))++-- A denylist, not a config validator: hledger validates unfamiliar syntax and+-- options on conversion. Add only known dependencies outside the cache inputs.+configDependency :: [(Int,T.Text)] -> Maybe Int+configDependency settings = go settings where- -- Only canonical built-ins other than print have an inert section. Addons,- -- aliases, abbreviations and names introduced in another release can have- -- different scope and must be passed through to the ordinary converter.- inactive = T.words $ "accounts activity add aregister balance balancesheet balancesheetequity cashflow check close codes commodities descriptions diff files help import incomestatement notes payees prices register repl rewrite roi run setup stats tags " <>- case release of- V1524 -> "commands demo test"- V1996 -> "get holdings transactions"- OtherSupported -> ""- line section (n,raw)- | T.null body || "#" `T.isPrefixOf` body = pure section- | release /= V1996, "[" `T.isPrefixOf` body, T.stripStart raw /= raw = unsupported n body- | Just start <- T.stripPrefix "[" body = case T.breakOn "]" start of- (s,end) | not (T.null end), not ("#" `T.isInfixOf` s), let name = T.strip s,- name `elem` ("general":"print":inactive) -> option n name (T.drop 1 end)- _ -> unsupported n body- | otherwise = option n section body- where body = T.strip raw- option n section arg- | section `elem` inactive = pure section- | T.null setting = pure section- | section == "print" && release /= V1524 && setting == "--layout=hledger1" = pure section- | otherwise = unsupported n setting- where setting = T.strip (T.takeWhile (/= '#') arg)- unsupported n value = atPath "unsupported hledger config" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)+ -- A fixed --today pins relative dates and valuation, but hledger ignores+ -- it for forecasting.+ pinned = or (zipWith fixedToday (map snd settings) (drop 1 (map snd settings) ++ [""]))+ fixedToday arg next = case T.stripPrefix "--today" arg of+ Just "" -> fixedDate next+ Just v | Just date <- T.stripPrefix "=" v -> fixedDate date+ _ -> False+ fixedDate date = not (T.null date || relativeDate date)+ go [] = Nothing+ go ((n,arg):rest)+ -- Untracked files, saved import state, extra output and temporary paths.+ | "@" `T.isPrefixOf` arg || flag ["--file", "--rules", "--rules-file", "--conf"] ["-f"] = found+ | flag ["--new", "--output-file", "--location", "--locations"] ["-o"] = found+ -- The rest depend on today's date.+ | flag ["--forecast"] [], T.null suffix || relativeDate value || openForecast value = found+ | pinned = go rest+ -- Without a report end date, hledger values amounts at today's prices.+ | flag ["--exchange"] ["-X", "-V"] = found+ | flag ["--value"] [], T.toLower (T.takeWhile (/= ',') value) `elem` ["end", "now"] = found+ | flag ["--begin", "--end", "--period", "--today"] ["-b", "-e", "-p"], relativeDate value = found+ | any relativeDate (queryDates arg) = found+ | otherwise = go rest+ where+ found = Just n+ (name,suffix) = T.breakOn "=" arg+ short = [s | s <- ["-f", "-o", "-X", "-b", "-e", "-p"], s `T.isPrefixOf` arg]+ flag longs shorts = name `elem` longs || any (`T.isPrefixOf` arg) shorts+ value | not (T.null suffix) = T.drop 1 suffix+ | s:_ <- short, T.length arg > T.length s = T.drop (T.length s) arg+ | (_,v):_ <- rest = v+ | otherwise = ""+ openForecast period = ".." `T.isSuffixOf` period+ || any (`T.isPrefixOf` period) ["from", "since"] && not ("to" `T.isInfixOf` period || ".." `T.isInfixOf` period)+ -- Dates in date: and date2: queries, including those inside expr:.+ queryDates token = case T.stripPrefix "expr:" (stripNot token) of+ Just expr -> mapMaybe queryDate (T.words (T.map (\c -> if c `elem` ("()" :: String) then ' ' else c) expr))+ Nothing -> maybeToList (queryDate token)+ queryDate token = case T.breakOn ":" (stripNot token) of+ (prefix,date) | prefix `elem` ["date", "date2"] -> Just (T.drop 1 date)+ _ -> Nothing+ stripNot token = maybe token stripNot (T.stripPrefix "not:" token) -checkDirectives :: Release -> AbsFile -> B.ByteString -> ReuseDecision ()-checkDirectives _ path bytes = do+-- Recognize relative words and dates with an implicit year. Unfamiliar date+-- syntax stays eligible, like unfamiliar options; this list can grow over time.+relativeDate :: T.Text -> Bool+relativeDate value = any relative parts+ where+ parts = T.words lowered ++ T.words (T.replace ".." " " (T.replace "to" " " lowered))+ lowered = T.toLower value+ relative part = any (`T.isPrefixOf` part) ["today", "tomorrow", "yesterday", "this", "last", "next"]+ || part `elem` ["ago", "ahead", "day", "days", "week", "weeks", "month", "months", "quarter", "quarters", "year", "years",+ "jan", "january", "feb", "february", "mar", "march", "apr", "april", "may",+ "jun", "june", "jul", "july", "aug", "august", "sep", "september",+ "oct", "october", "nov", "november", "dec", "december", "q1", "q2", "q3", "q4"]+ || let digits = T.takeWhile isDigit (T.dropWhile (`elem` ("+-" :: String)) part) in not (T.null digits) && T.length digits < 4++journalIncludes :: AbsFile -> B.ByteString -> ReuseDecision [FilePath]+journalIncludes path bytes = do ls <- physicalLines path (dropUtf8Bom bytes)- case [(n,T.strip l) | (n,l) <- zip [1 :: Int ..] ls, not (allowed l)] of- [] -> pure ()- (n,value):_ -> atPath "unsupported journal directive" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)+ sequence (mapMaybe classify (zip [1 :: Int ..] ls)) where+ classify (n,raw)+ | allowed raw = Nothing+ | ("include",rest) <- T.break horizontalSpace raw,+ let target = T.dropAround horizontalSpace (T.takeWhile (/= ';') rest)+ journalPath = T.stripPrefix "journal:" target+ name = fromMaybe target journalPath,+ not (T.null name),+ T.all (\c -> isAlphaNum c || c `elem` (" ./_-" :: String)) name =+ Just $ if journalPath == Nothing && map toLower (F.takeExtension (T.unpack name)) `elem` nonJournalExtensions+ then atPath "unsupported journal include reader" (toFilePath path) (lineDetail n raw)+ else pure (T.unpack name)+ | otherwise = Just (atPath "unsupported journal directive" (toFilePath path)+ (lineDetail n raw))+ -- hledger defaults to the journal reader for unrecognized extensions.+ nonJournalExtensions = [".csv", ".tsv", ".ssv", ".rules", ".timeclock", ".timedot"]+ lineDetail n raw = "line " <> T.pack (show n) <> ": " <> T.strip raw+ horizontalSpace c = c == ' ' || c == '\t' allowed raw | T.null stripped || T.isPrefixOf ";" stripped || T.isPrefixOf "#" stripped || T.isPrefixOf "*" stripped = True | Just (c,_) <- T.uncons raw, isSpace c = False
src/Hledger/Flow/Import/IncludeFiles.hs view
@@ -22,7 +22,7 @@ import qualified Data.Text as T import qualified Data.Text.IO as T import Hledger.Flow.BaseDir (relativeToBase')-import Hledger.Flow.Common (allYearsFileName, filterPaths, groupValuesBy, textFileNeedsUpdate, writeFiles')+import Hledger.Flow.Common (allYearsFileName, filterPaths, groupValuesBy, includePreamble, textFileNeedsUpdate, writeFiles') import Hledger.Flow.Import.ImportHelpers (groupIncludesUpTo) import Hledger.Flow.Import.Inventory (manualIncludeFiles, managedIncludeFiles) import Hledger.Flow.Import.Types@@ -74,9 +74,6 @@ let lns = map (toIncludeLine $ FilePath.takeDirectory outputFile) $ preFiles ++ files ++ postFiles T.intercalate "\n" $ lns ++ [""] -includePreamble :: T.Text-includePreamble = "### Generated by hledger-flow - DO NOT EDIT ###\n"- writeIncludesFromInventory :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> AbsDir -> [AbsFile] -> [AbsFile] -> IO [AbsFile] writeIncludesFromInventory opts ch stopAt inventory journals = do (files, roots) <- expectedIncludes (baseDir opts) (logVerbose opts ch) stopAt inventory journals@@ -84,10 +81,10 @@ return roots -- | Check the same contents import would write, including obsolete generated indexes.-includesNeedUpdate :: BaseDir -> AbsDir -> [AbsFile] -> [AbsFile] -> IO Bool-includesNeedUpdate base root inventory journals = do+includesNeedUpdate :: BaseDir -> AbsDir -> [AbsFile] -> [AbsFile] -> Bool -> IO Bool+includesNeedUpdate base root inventory journals compareRoot = do (includes, _) <- expectedIncludes base (const (return ())) root inventory journals- top <- toplevelAllYearsInclude base+ top <- if compareRoot then toplevelAllYearsInclude base else return Map.empty anyChanged (Map.toList (Map.union top includes)) where anyChanged [] = return False
src/Hledger/Flow/Import/Incremental.hs view
@@ -10,11 +10,11 @@ import Control.Monad (unless, when) import Data.IORef import qualified Data.Map.Strict as Map-import Data.Maybe (maybeToList)+import Data.Maybe (listToMaybe, maybeToList) import qualified Data.Set as Set import qualified Data.Text as T import Hledger.Flow.BaseDir (relativeToBase)-import Hledger.Flow.Common (directivesFile, writeStagedText)+import Hledger.Flow.Common (directivesFile, homeDirs, tildePath, writeStagedText) import Hledger.Flow.DocHelpers import Hledger.Flow.Cache (initializeCache) import Hledger.Flow.Import.Cache@@ -22,6 +22,7 @@ import Hledger.Flow.Import.Dependencies import Hledger.Flow.Import.Fingerprint import Hledger.Flow.Import.Inventory (generatedOutput)+import Hledger.Flow.Import.ShellSources import Hledger.Flow.Import.Snapshot import Hledger.Flow.Import.Types import Hledger.Flow.Logging@@ -36,16 +37,17 @@ -- Keep one path only when every occurrence of a reason names that file. data Notices = Notices !(Set.Set AbsFile) !(Map.Map T.Text (Maybe T.Text))-data IncrementalRun = IncrementalRun SnapshotRun (IO (ReuseDecision Converter)) (IORef Notices)+data IncrementalRun = IncrementalRun SnapshotRun SourceCache (IO (ReuseDecision Converter)) (IORef Notices) withIncrementalRun :: RuntimeOptions -> TChan LogMessage -> Int -> (IncrementalRun -> IO a) -> IO a withIncrementalRun opts ch count action = withSnapshotRun (baseDir opts) $ \snapshots -> do+ scripts <- newSourceCache reasons <- newIORef (Notices Set.empty Map.empty) probe <- newConverterProbe opts when (count > 0) $ do initialized <- tryIOError $ initializeCache (baseDir opts) either (channelErrLn ch . ("Could not initialize incremental cache: " <>) . T.pack . show) pure initialized- finally (action (IncrementalRun snapshots probe reasons)) $ do+ finally (action (IncrementalRun snapshots scripts probe reasons)) $ do Notices affected warnings <- readIORef reasons unless (Map.null warnings) $ do let scope | Set.size affected == count = "all selected statements"@@ -54,21 +56,27 @@ channelErrLn ch $ "Incremental reuse unavailable for " <> scope <> ": " <> T.intercalate ", " (map describe (Map.toAscList warnings)) <> ". See " <> docURL IncrementalImportFallback -data HookInputs = HookInputs AbsFile FileObservation AbsFile FileObservation+data HookInputs = HookInputs AbsFile FileObservation AbsFile FileObservation [SourceInput] deriving (Eq, Show) -observeHook :: AbsFile -> AbsFile -> IO (ReuseDecision HookInputs)-observeHook script source = checkedIO "could not check script inputs" (P.toFilePath script) $ do+observeHook :: SourceCache -> AbsFile -> AbsFile -> IO (ReuseDecision HookInputs)+observeHook scripts script source = checkedIO "could not check script inputs" (P.toFilePath script) $ do runnable <- if Sys.os == "mingw32" then pure True else D.executable <$> D.getPermissions (P.toFilePath script) if runnable- then Reusable <$> (HookInputs script <$> observeFile script <*> pure source <*> observeFile source)+ then do+ observed <- observeFile script+ input <- observeFile source+ deps <- checkedIO "unresolved shell source" (P.toFilePath script) $+ Reusable <$> hookSources scripts script observed+ pure (HookInputs script observed source input <$> deps) else pure $ Fallback (OrdinaryInputError "script is not executable" (pathText script)) hookInputsKey :: ImportDirs P.Abs -> FilePath -> T.Text -> HookInputs -> StageInputs-hookInputsKey dirs cwd stage (HookInputs script s input i) =+hookInputsKey dirs cwd stage (HookInputs script s input i deps) = [("stage",stage),("working directory",T.pack cwd),("hook",pathText script), ("hook digest",digestHex (observationDigest s)),("input",pathText input), ("input digest",digestHex (observationDigest i))] ++ map ("argument",) (hookDirArgs dirs)+ ++ concatMap sourceKey deps pathText :: AbsFile -> T.Text pathText = T.pack . P.toFilePath@@ -77,12 +85,12 @@ { engine :: Converter , copiedInputs :: SnapshotInput , recordedInputs :: StageInputs- , copiedDigests :: Map.Map AbsFile Digest+ , copiedDigests :: [(AbsFile, Digest)] , liveConstruct :: Maybe HookInputs } deriving (Eq, Show) importStatement :: IncrementalRun -> RuntimeOptions -> TChan LogMessage -> ImportDirs P.Abs -> AbsFile -> IO (AbsFile,Bool,Bool)-importStatement (IncrementalRun snapshots probe reasons) opts ch dirs source = do+importStatement (IncrementalRun snapshots scripts probe reasons) opts ch dirs source = do cwd <- D.getCurrentDirectory record <- recordPath (baseDir opts) source old <- readRecord record@@ -90,11 +98,16 @@ let hook name = findHook opts ch name (accountDir dirs P.</> (if name == "preprocess" then [P.relfile|preprocess|] else [P.relfile|construct|])) report OrdinaryInputError{} = pure () report problem = do+ homes <- homeDirs let reason = reasonScope problem+ located file = T.pack (tildePath homes (relativeToBase opts file)) path = case problem of- FileReason _ file _ -> Just (T.pack (relativeToBase opts file))+ FileReason _ file _ -> Just (located file)+ ConfigEntry _ file entry -> Just (located file <> ", " <> entry) _ -> Nothing- detail = reasonDetail problem+ detail = case problem of+ ConfigEntry{} -> ""+ _ -> reasonDetail problem common a b = if a == b then a else Nothing atomicModifyIORef' reasons $ \(Notices files summaries) -> (Notices (Set.insert source files) (Map.insertWith common reason path summaries), ())@@ -112,7 +125,7 @@ input <- savedInput opts ch source intermediate pure (input,Nothing,False) Just script -> do- before <- observeHook script source+ before <- observeHook scripts script source let key = hookInputsKey dirs cwd "preprocess" <$> before prior = old >>= preprocessRecord hit <- case key of@@ -127,7 +140,7 @@ previous <- digestIfPresent intermediate digest <- installOutput True intermediate (preprocess opts ch script dirs source intermediate) when (Just digest == previous) $ logVerbose opts ch "Preprocessing ran; the CSV is unchanged."- after <- observeHook script source+ after <- observeHook scripts script source selected <- hook "preprocess" usable <- case key of Fallback reason -> when skipScripts (report reason) >> pure Nothing@@ -149,10 +162,10 @@ ++ map ("converter",) (converterKey converter) configArgs = pinnedConfig ("<config>" <$ conf) mk inputs files args extra construct =- JournalInputs converter inputs (common ++ map ("hledger argument",) args ++ extra ++ [(pathText p,digestHex d) | (p,d) <- files]) (Map.fromList files) construct+ JournalInputs converter inputs (common ++ map ("hledger argument",) args ++ extra ++ [(pathText p,digestHex d) | (p,d) <- files]) files construct case candidate of Just script -> do- observation <- observeHook script csv+ observation <- observeHook scripts script csv pure $ (\h -> mk (ConstructInput (fst <$> conf)) (maybeToList conf) (constructArguments configArgs "<journal>") (hookInputsKey dirs cwd "construct" h) (Just h)) <$> observation Nothing | not (isSeparatedValues csv) -> pure $ atPath "unsupported input reader" (P.toFilePath csv) ""@@ -167,9 +180,10 @@ digest <- digestFile csv pure $ do rulesTree <- tree- ds <- directives- let files = (csv,digest) : rulesTree ++ maybeToList ds ++ maybeToList conf- pure $ mk (RulesInput csv rf (fst <$> ds) (fst <$> conf)) files+ journals <- directives+ let ds = fst <$> listToMaybe journals+ files = (csv,digest) : rulesTree ++ journals ++ maybeToList conf+ pure $ mk (RulesInput csv rf ds (fst <$> conf)) files (rulesArguments configArgs "<input>" "<rules>" ("<directives>" <$ ds)) [("rules",pathText rf)] Nothing ordinary = do script <- hook "construct"@@ -186,8 +200,8 @@ let agrees inputs = case preRecord of Nothing -> True Just entry -> Just (stageOutput entry) == case liveConstruct inputs of- Nothing -> Map.lookup csv (copiedDigests inputs)- Just (HookInputs _ _ _ observed) -> Just (observationDigest observed)+ Nothing -> lookup csv (copiedDigests inputs)+ Just (HookInputs _ _ _ observed _) -> Just (observationDigest observed) decision = rawDecision >>= \inputs -> if agrees inputs then pure inputs else Fallback changed (journal,generated) <- case decision of Fallback r -> fallback r@@ -209,7 +223,7 @@ contents <- convertRules opts ch executable (RulesPaths src rf ds) (RulesPaths src' rf' ds') (pin conf) translate target <- D.canonicalizePath (P.toFilePath output) >>= P.parseAbsFile installOutput False target (writeStagedText target contents)- (ConstructInput _,ConstructInput conf,Just (HookInputs s _ _ _)) ->+ (ConstructInput _,ConstructInput conf,Just (HookInputs s _ _ _ _)) -> installOutput False output (constructWith opts ch executable (pin conf) translate s dirs csv output) _ -> ioError (userError "Inconsistent incremental conversion inputs") again <- discover@@ -217,6 +231,8 @@ effective <- case preNow of Nothing -> P.doesFileExist intermediate >>= \exists -> pure (if exists then intermediate else source) Just _ -> pure intermediate+ -- Order-sensitive: copyInputs must copy in the order discovery+ -- lists inputs, or no rebuild ever counts as stable. let stable = again == Reusable inputs && snapshotManifest snapshot == copiedDigests inputs && preNow == pre && effective == csv if stable then pure $ Just (StageRecord (recordedInputs inputs) outDigest) else when (skipScripts || liveConstruct inputs == Nothing) (report changed) >> pure Nothing
src/Hledger/Flow/Import/Inventory.hs view
@@ -25,6 +25,7 @@ import qualified Data.List as List import qualified Data.Map.Strict as Map import qualified Data.Set as Set+import Hledger.Flow.Cache (initializeIgnoredDir, trashDir) import Hledger.Flow.Common (groupValuesBy, homeDirs, tildePath) import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir) import Path@@ -203,20 +204,19 @@ && (name == "all-years.journal" || (isYear (take 4 name) && drop 4 name == "-include.journal")) _ -> False -moveToTrash :: AbsFile -> IO AbsFile-moveToTrash file = do- let year = dirname (parent file)- stage = dirname (parent (parent file))- account = parent (parent (parent file))- trash = account </> [reldir|.trash|]+-- | Move a file under the ledger's trash, keeping its path from the ledger root.+moveToTrash :: AbsDir -> AbsFile -> IO AbsFile+moveToTrash base file = do+ rel <- stripProperPrefix base (parent file)+ let trash = trashDir base lock = trash </> [reldir|.move-lock|]- destination = trash </> year </> stage+ destination = trash </> rel acquire = Directory.createDirectory (toFilePath lock) `catchIOError` \err -> if isAlreadyExistsError err then homeDirs >>= \homes -> ioError (userError ("Trash is in use: " ++ tildePath homes (toFilePath lock) ++ ". If no import is running, remove this empty lock directory and retry.")) else ioError err- PathIO.createDirIfMissing True trash+ initializeIgnoredDir trash -- Serialize destination selection and rename so two imports cannot replace -- each other's recovery files. An interrupted move leaves a recoverable lock. bracket_ acquire (PathIO.removeDir lock) $ do
+ src/Hledger/Flow/Import/ShellSources.hs view
@@ -0,0 +1,196 @@+{-# LANGUAGE CPP #-}+{-# LANGUAGE OverloadedStrings #-}++module Hledger.Flow.Import.ShellSources+ ( SourceCache, newSourceCache, SourceInput, sourceKey, hookSources ) where++import Control.Monad (foldM, unless, when)+import Data.Char (isAlpha, isAlphaNum, isAscii, isControl, isSpace)+import qualified Data.ByteString as BS+import qualified Data.ByteString.Char8 as B8+import Data.Foldable (toList)+import Data.Functor.Identity (runIdentity)+import Data.IORef+import Data.List (isPrefixOf)+import Data.Maybe (isJust)+import qualified Data.Map.Strict as Map+import qualified Data.Set as Set+import qualified Data.Text as T+import qualified Data.Text.Encoding as TE+import Hledger.Flow.Import.Fingerprint+import Hledger.Flow.PathHelpers (AbsFile)+import qualified Path as P+import ShellCheck.AST+import ShellCheck.ASTLib (getLiteralString)+import ShellCheck.Interface+import ShellCheck.Parser (parseScript)+import qualified System.Directory as D+import System.Environment (lookupEnv)+import qualified System.FilePath as F+import System.IO (IOMode(ReadMode), withBinaryFile)+#ifndef mingw32_HOST_OS+import qualified System.Posix.Files as Posix+#endif++-- Cache only syntax, never path resolution or filesystem observations.+type SourceCache = IORef (Map.Map Digest [Step])+newSourceCache :: IO SourceCache+newSourceCache = newIORef Map.empty++data SourceInput = SourceInput FilePath FileObservation [(String, Digest)] deriving (Eq, Show)++sourceKey :: SourceInput -> [(T.Text, T.Text)]+sourceKey (SourceInput target obs variables) =+ (T.pack target, digestHex (observationDigest obs)) :+ [("source variable " <> T.pack name, digestHex digest) | (name,digest) <- variables]++data Piece = Literal String | Variable Bool String+data Step = Include [Piece] | ShellStateChanged | Uncertain++hookSources :: SourceCache -> AbsFile -> FileObservation -> IO [SourceInput]+hookSources cache hook observed = do+ let file = P.toFilePath hook+ header <- withBinaryFile file ReadMode (fmap (B8.takeWhile (/= '\n')) . flip BS.hGet 256)+ case interpreter file header of+ Other -> pure []+ UnrecognisedShell -> uncertain file+ ShellScript -> do+ startup <- lookupEnv "BASH_ENV"+ when (maybe False (not . null) startup) (uncertain file)+ files <- newIORef []+ visits <- newIORef (0 :: Int)+ let visit parents path observation = do+ canonical <- D.canonicalizePath path+ when (Set.member canonical parents) (uncertain path)+ n <- atomicModifyIORef' visits (\i -> (i+1,i))+ when (n >= 128) (uncertain path)+ plan <- scriptSteps cache path (observationDigest observation)+ foldM (step (Set.insert canonical parents) path) True plan+ step _ _ _ ShellStateChanged = pure False+ step _ path _ Uncertain = uncertain path+ step parents path pristine (Include parts) = do+ -- Earlier commands can change working directory, variable bindings+ -- or the meaning of the source builtin.+ unless pristine (uncertain path)+ expanded <- mapM (expand path) parts+ let name = concatMap fst expanded+ variables = concatMap snd expanded+ unless ('/' `elem` name && not ("-" `isPrefixOf` name) &&+ not (any isControl name)) (uncertain path)+ -- Test the original spelling before canonicalizing missing/../+ -- and symlink/../ components.+#ifdef mingw32_HOST_OS+ regular <- D.doesFileExist name+#else+ regular <- Posix.isRegularFile <$> Posix.getFileStatus name+#endif+ unless regular (uncertain path)+ target <- D.canonicalizePath name+ dep <- P.parseAbsFile target >>= observeFile+ modifyIORef' files (SourceInput target dep variables :)+ visit parents target dep+ _ <- visit Set.empty file observed+ reverse <$> readIORef files+ where+ uncertain f = ioError (userError ("cannot resolve shell source safely in " ++ f))+ expand _ (Literal s) = pure (s, [])+ expand f (Variable quoted name) = do+ value <- lookupEnv name+ case value of+ Just s | not (null s), quoted || not (any (\c -> isSpace c || c `elem` ("*?[" :: String)) s) ->+ pure (s, [(name, digestBytes (TE.encodeUtf8 (T.pack s)))])+ _ -> uncertain f++data Interpreter = ShellScript | Other | UnrecognisedShell++interpreter :: FilePath -> BS.ByteString -> Interpreter+#ifdef mingw32_HOST_OS+interpreter file _ | F.takeExtension file == ".ps1" = Other+#endif+interpreter _ header = case B8.stripPrefix "#!" header of+ Nothing -> if BS.all (\c -> c == 9 || c == 13 || c >= 32) header then UnrecognisedShell else Other+ Just line -> case words (B8.unpack line) of+ exe:args | F.takeFileName exe == "env" -> case args of+ [name] | shell name -> ShellScript+ _ | any shell args -> UnrecognisedShell+ _ -> Other+ | shell exe -> if null args then ShellScript else UnrecognisedShell+ _ -> Other+ where shell p = F.takeFileName p `elem` ["bash","sh","dash"]++scriptSteps :: SourceCache -> FilePath -> Digest -> IO [Step]+scriptSteps cache path digest = do+ existing <- Map.lookup digest <$> readIORef cache+ case existing of+ Just plan -> pure plan+ Nothing -> do+ bytes <- BS.readFile path+ unless (digestBytes bytes == digest) (ioError (userError ("script changed while reading " ++ path)))+ let plan = case TE.decodeUtf8' bytes of+ Left _ -> [Uncertain]+ Right body ->+ let result = runIdentity $ parseScript newSystemInterface newParseSpec+ { psScript = T.unpack body, psFilename = path, psShellTypeOverride = Just Bash, psIgnoreRC = True }+ in case prRoot result of+ Just root | all ((/= ErrorC) . cSeverity . pcComment) (prComments result) -> steps root+ _ -> [Uncertain]+ atomicModifyIORef' cache (\m -> (Map.insert digest plan m, ()))+ pure plan++-- A function or loop can execute later or repeatedly with different bindings.+steps :: Token -> [Step]+steps token = case token of+ T_Annotation _ _ t -> steps t+ T_Script _ _ ts -> concatMap steps ts+ T_Pipeline _ [] [t] -> steps t+ T_Redirecting _ [] t -> steps t+ T_AndIf _ a b | not (containsSource b) -> steps a ++ steps b+ T_OrIf _ a b | not (containsSource b) -> steps a ++ steps b+ T_SimpleCommand _ [] (cmd:args)+ | isSource cmd -> case args of+ path:rest | all sourceArgument rest, Just parts <- pieces False path -> [Include parts]+ _ -> [Uncertain]+ | Just name <- getLiteralString cmd, name `elem` ["exit","true","false",":"],+ all (isJust . getLiteralString) args -> []+ | getLiteralString cmd == Just "set",+ all (\a -> getLiteralString a `elem` map Just ["-e","-u","-eu","-euo","-o","pipefail"]) args -> []+ T_Function _ _ _ name body | name `notElem` ["source",".","set","exit","true","false",":","[","test"], not (containsSource body) -> []+ T_Condition _ _ (TC_Unary _ _ op arg) | op `elem` ["-z","-n"], isJust (pieces True arg) || positionalArgument arg -> []+ _ | containsSource token -> [Uncertain]+ | otherwise -> [ShellStateChanged]+ where+ -- Forwarded hook arguments are already captured by the hook key.+ sourceArgument t = positionalArgument t || maybe False (all literal) (pieces False t)+ literal (Literal _) = True+ literal _ = False+ positionalArgument (T_NormalWord _ [T_DoubleQuoted _ [T_DollarBraced _ _ name]]) =+ case getLiteralString name of+ Just "@" -> True+ Just n -> not (null n) && all (\c -> c >= '0' && c <= '9') n+ Nothing -> False+ positionalArgument _ = False++isSource :: Token -> Bool+isSource t = getLiteralString t `elem` [Just ".", Just "source"]++containsSource :: Token -> Bool+containsSource t@(OuterToken _ children) = case t of+ T_SimpleCommand _ _ (cmd:args)+ | isSource cmd || getLiteralString cmd == Just "eval" -> True+ | getLiteralString cmd `elem` [Just "builtin", Just "command"], any isSource args -> True+ _ -> any containsSource (toList children)++-- Parameter operators, command substitution, globs and tilde expansion need Bash.+pieces :: Bool -> Token -> Maybe [Piece]+pieces quoted token = case token of+ T_NormalWord _ ts -> concat <$> mapM (pieces quoted) ts+ T_DoubleQuoted _ ts -> concat <$> mapM (pieces True) ts+ T_SingleQuoted _ s -> Just [Literal s]+ T_Literal _ s | quoted || not (any (\c -> c `elem` ("~*?[]{}" :: String)) s) -> Just [Literal s]+ T_DollarBraced _ _ t | Just name@(c:_) <- getLiteralString t,+ isAscii c && (isAlpha c || c == '_'), all (\x -> isAscii x && (isAlphaNum x || x == '_')) name,+ not ("BASH" `isPrefixOf` name || name `elem` ["_", "PWD", "OLDPWD", "IFS", "LINENO", "RANDOM", "SRANDOM",+ "SECONDS", "SHLVL", "UID", "EUID", "PPID", "OPTIND", "OPTARG", "SHELLOPTS", "FUNCNAME",+ "DIRSTACK", "GROUPS", "PIPESTATUS", "EPOCHSECONDS", "EPOCHREALTIME"]) ->+ Just [Variable quoted name]+ _ -> Nothing
src/Hledger/Flow/Import/Snapshot.hs view
@@ -45,7 +45,7 @@ data PreparedSnapshot = PreparedSnapshot { snapshotInputs :: SnapshotInput,- snapshotManifest :: Map.Map AbsFile Digest,+ snapshotManifest :: [(AbsFile, Digest)], originalPaths :: Map.Map AbsFile AbsFile } @@ -196,9 +196,8 @@ copyInputs :: AbsDir -> AbsDir -> Release -> SnapshotInput -> IO PreparedSnapshot copyInputs base tree version input = do- manifest <- newIORef Map.empty+ manifest <- newIORef [] paths <- newIORef Map.empty- completed <- newIORef Set.empty let mirror file = case stripProperPrefix base file of Nothing -> reject "snapshot input outside ledger" (T.pack (toFilePath file)) Just rel -> pure (tree </> [reldir|l|] </> rel)@@ -212,15 +211,16 @@ -- equivalence rules. Never overwrite a copy made for another spelling. previous <- Map.lookup target <$> readIORef paths collision <- case previous of- Just original -> pure (toFilePath original /= toFilePath source)+ Just (original, _) -> pure (toFilePath original /= toFilePath source) Nothing -> D.doesPathExist (toFilePath target) when collision $ reject "snapshot filenames collide" (T.pack (toFilePath source)) record source target digest = do- previous <- Map.lookup source <$> readIORef manifest- when (maybe False (/= digest) previous) $+ previous <- Map.lookup target <$> readIORef paths+ when (maybe False ((/= digest) . snd) previous) $ reject "dependency changed during snapshot" (T.pack (toFilePath source))- modifyIORef' manifest (Map.insert source digest)- modifyIORef' paths (Map.insert target source)+ -- Include order matters if a directory link changes during copying.+ modifyIORef' manifest ((source, digest) :)+ modifyIORef' paths (Map.insert target (source, digest)) copyBytes source target = do destination source target bytes <- BS.readFile (toFilePath source)@@ -232,30 +232,38 @@ () <- fromDecision (check version source bytes) pure target copyConfig source = validated checkConfig source (tree </> [reldir|c|] </> filename source)- copyRule ancestors source = do- target <- mirror source- identity <- D.canonicalizePath (toFilePath source)- when (Set.member identity ancestors || Set.size ancestors >= 256) $ reject "cyclic or excessively deep rules include" (T.pack (toFilePath source))- done <- Set.member (toFilePath source) <$> readIORef completed- unless done $ do- bytes <- copyBytes source target- let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes- includes <- fromDecision (scan version source bytes)- forM_ includes $ \spelling -> do- child <- resolveRuleInclude base source spelling >>= fromDecision- void $ copyRule (Set.insert identity ancestors) child- modifyIORef' completed (Set.insert (toFilePath source))- pure target+ copyTree kind root = do+ nodes <- readIncludeTree base kind root >>= fromDecision+ let detours = Set.fromList (concatMap includeDirectories nodes)+ common = foldl' containing base (map (parent . includePosition) nodes ++ Set.toList detours)+ -- Keep ordinary layout short; use a separate tree when a rules+ -- include needs a sibling of the ledger or a different ancestor.+ folder = tree </> if common == base then [reldir|l|] else [reldir|r|]+ staged path = (folder </>) <$> stripProperPrefix common path+ forM_ detours $ \dir -> do+ target <- if dir == common then pure folder else staged dir+ P.createDirIfMissing True target+ forM_ nodes $ \node -> do+ let source = includeSource node+ bytes = includeBytes node+ target <- staged (includePosition node)+ destination source target+ BS.writeFile (toFilePath target) bytes+ record source target (digestBytes bytes)+ staged root+ containing dir path+ | dir == path || isProperPrefixOf dir path || dir == parent dir = dir+ | otherwise = containing (parent dir) path copied <- case input of ConstructInput conf -> ConstructInput <$> traverse copyConfig conf RulesInput csv rules directives conf -> do dataCopy <- mirror csv destination csv dataCopy copyDigest csv dataCopy >>= record csv dataCopy- rulesCopy <- copyRule Set.empty rules- directivesCopy <- traverse (\source -> mirror source >>= validated checkDirectives source) directives+ rulesCopy <- copyTree (CsvRules version) rules+ directivesCopy <- traverse (copyTree JournalDirectives) directives RulesInput dataCopy rulesCopy directivesCopy <$> traverse copyConfig conf- PreparedSnapshot copied <$> readIORef manifest <*> readIORef paths+ PreparedSnapshot copied <$> (reverse <$> readIORef manifest) <*> (Map.map fst <$> readIORef paths) -- hledger emits locations as "FILE:line:column:" (also "FILE:line:" and -- "FILE: explanation"). Restrict replacement to its location fields.
src/Hledger/Flow/Init.hs view
@@ -9,22 +9,25 @@ ) where -import Control.Monad (when)+import Control.Monad (forM, when) import qualified Data.Text as T import qualified Data.Text.IO as T import Hledger.Flow.BaseDir (findBaseDir) import Data.Maybe (fromMaybe)-import Hledger.Flow.Common (Shell (..), homeDirs, homeShellPath, nativeShell, shellArgument, tildePath)-import Hledger.Flow.PathHelpers (AbsDir)+import Hledger.Flow.Common (Shell (..), findHledgerConf, homeDirs, homeShellPath, nativeShell, shellArgument, tildePath)+import Hledger.Flow.DateTime (currentYear)+import Hledger.Flow.Init.Example (exampleAccount, exampleAccountPath, exampleBaseFiles, exampleImportFiles, starterOwner)+import Hledger.Flow.PathHelpers (AbsDir, RelDir, RelFile, pathOccupied, writeNewUtf8File) import Hledger.Flow.Status.Render (printNext) import Hledger.Flow.Types (BaseDir) import Path-import Path.IO (createDirIfMissing, doesDirExist)+import Path.IO (createDirIfMissing, doesDirExist, doesFileExist, renameDir, withTempDir)+import System.IO.Error (tryIOError) import qualified System.FilePath as FilePath data InitOutcome = -- | The new ledger, and an unrelated ledger above it, if any.- Created BaseDir (Maybe BaseDir)+ Created BaseDir (Maybe BaseDir) RelDir [(RelFile, Bool)] | -- | The folder already has @import/@. AlreadyLedger BaseDir | -- | The folder is inside this ledger's @import/@ tree.@@ -42,12 +45,34 @@ case above of Just b | let imp = b </> [reldir|import|], imp == dir || imp `isProperPrefixOf` dir -> return (InsideLedger b) _ -> do- createDirIfMissing True (dir </> [reldir|import|])- return (Created dir above)+ year <- currentYear+ owner <- starterOwner+ createDirIfMissing True dir+ baseFiles <- forM (exampleBaseFiles year) $ \(name, contents) -> do+ let target = dir </> name+ occupied <- pathOccupied target+ if occupied+ then do+ isFile <- doesFileExist target+ if isFile+ then return (name, False)+ else ioError (userError ("Cannot create starter file at " ++ toFilePath target))+ else do+ result <- tryIOError (writeNewUtf8File target contents)+ case result of+ Right () -> return (name, True)+ Left err -> do+ exists <- doesFileExist target+ if exists then return (name, False) else ioError err+ withTempDir dir ".hledger-flow-init-" $ \temp -> do+ let staged = temp </> [reldir|import|]+ mapM_ (\(name, contents) -> writeNewUtf8File (staged </> name) contents) (exampleImportFiles owner year)+ renameDir staged (dir </> [reldir|import|])+ return (Created dir above (exampleAccount owner) baseFiles) -- | Run @hledger-flow init@ for a folder, printing what happened and what's next.-runInit :: Maybe FilePath -> AbsDir -> AbsDir -> IO ()-runInit hledgerPath cwd dir = do+runInit :: Maybe FilePath -> Maybe FilePath -> AbsDir -> AbsDir -> IO ()+runInit hledgerPath confOption cwd dir = do homes <- homeDirs let absolute = FilePath.dropTrailingPathSeparator . toFilePath shown = T.pack . tildePath homes . absolute@@ -59,8 +84,10 @@ InsideLedger b -> ioError . userError . T.unpack $ "That folder is inside the ledger at " <> shown b <> ". Start a new ledger outside its import/ folder."- Created base parentLedger -> do+ Created base parentLedger account baseFiles -> do T.putStrLn ("Started a ledger in " <> shown base)+ T.putStrLn ("Created an example account (" <> exampleAccountPath account <> ") and starter journals for manual entries and balances.")+ mapM_ (\(name, created) -> T.putStrLn ((if created then "Created " else "Kept existing ") <> T.pack (toFilePath name))) baseFiles mapM_ (\p -> T.putStrLn ("Note: " <> shown p <> " also has an import/ folder. Commands run inside " <> shown base <> " use the new ledger.")) parentLedger when (base /= cwd) $ do let changeDir = case nativeShell of@@ -68,4 +95,5 @@ PowerShell -> "Set-Location -LiteralPath " path = absolute base T.putStrLn ("Change to it first: " <> changeDir <> fromMaybe (shellArgument nativeShell (T.pack path)) (homeShellPath nativeShell homes path))- printNext hledgerPath base+ conf <- findHledgerConf confOption+ printNext hledgerPath conf base
+ src/Hledger/Flow/Init/Example.hs view
@@ -0,0 +1,316 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module Hledger.Flow.Init.Example+ ( exampleImportFiles,+ isSampleStatement,+ exampleBaseFiles,+ starterOwner,+ exampleAccount,+ exampleAccountPath,+ sampleAccountDir,+ untouchedExample,+ directivesAreStarter,+ journalIsCommentsOnly,+ )+where++import qualified Data.ByteString as B+import Data.List (intercalate)+import Data.Maybe (fromMaybe, mapMaybe)+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import qualified Data.Text.IO as TIO+import Hledger.Flow.Common (Shell (..), nativeShell)+import Hledger.Flow.DocHelpers (DocLink (Balances, ManualJournals, RulesFiles), docURL)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile, pathOccupied)+import Hledger.Flow.Types (BaseDir)+import Path+import qualified Path.IO as PathIO+import System.Environment (lookupEnv)+import qualified System.FilePath as F+import qualified System.FilePath.Windows as Windows+import System.IO.Error (tryIOError)+import Text.Read (readMaybe)++starterOwner :: IO RelDir+starterOwner = do+ let variable = case nativeShell of+ Posix -> "USER"+ PowerShell -> "USERNAME"+ value <- either (const Nothing) id <$> tryIOError (lookupEnv variable)+ return (fromMaybe [reldir|me|] (value >>= validOwner))+ where+ validOwner name+ | Windows.isValid name,+ Windows.splitDirectories name == [name],+ not (any Windows.isPathSeparator name),+ T.strip text == text,+ not (T.isPrefixOf "." text || T.isPrefixOf "~" text),+ not (T.isSuffixOf "." text),+ ';' `notElem` name,+ not (T.isSuffixOf ".journal" folded),+ folded /= "_manual_",+ B.length (T.encodeUtf8 text) <= 255 = parseRelDir name+ | otherwise = Nothing+ where+ text = T.pack name+ folded = T.toCaseFold text++exampleAccount :: RelDir -> RelDir+exampleAccount owner = owner </> [reldir|mybank/checking|]++exampleAccountPath :: RelDir -> T.Text+exampleAccountPath account = "import/" <> T.pack (intercalate "/" parts)+ where+ parts = F.splitDirectories (F.dropTrailingPathSeparator (toFilePath account))++sampleName :: RelFile+sampleName = [relfile|hledger-flow-example.csv|]++isSampleStatement :: AbsFile -> Bool+isSampleStatement file = filename file == sampleName++rulesName :: RelFile+rulesName = [relfile|mybank-checking.rules|]++sampleCsv :: Integer -> T.Text+sampleCsv year = T.unlines $+ zipWith (<>) (sampleCsvRows year) [",Comment", ",\"" <> starterNotice <> "\"", ","]++sampleCsvRows :: Integer -> [T.Text]+sampleCsvRows year =+ ["Date,Description,Amount", date <> ",Example purchase,-5.00", date <> ",Example refund,5.00"]+ where+ date = T.pack (show year) <> "-01-02"++sampleRules :: T.Text+sampleRules = T.unlines+ [ "# Rules for the example statement.",+ "# Docs: " <> docURL RulesFiles,+ "skip 1",+ "fields date, description, amount",+ "currency USD",+ "account1 assets:mybank:checking",+ "account2 expenses:unknown",+ "",+ "if Example",+ " account2 expenses:example"+ ]++exampleImportFiles :: RelDir -> Integer -> [(RelFile, T.Text)]+exampleImportFiles starter year =+ (exampleAccount starter </> rulesName, starterComment "# " sampleRules)+ : paths ((samplePath, sampleCsv year) : manualFiles ++ balanceFiles)+ where+ account = F.dropTrailingPathSeparator (toFilePath (exampleAccount starter))+ bank = F.takeDirectory account+ owner = F.takeDirectory bank+ yearName = show year+ samplePath = account F.</> "1-in" F.</> yearName F.</> toFilePath sampleName+ date day = T.pack yearName <> "-" <> day+ entry day description postings = (date day <> " " <> description) : map (" " <>) postings+ manual level phase help example =+ (level F.</> "_manual_" F.</> yearName F.</> (phase ++ "-import.journal"),+ journalExample ManualJournals help example)+ balance suffix help example =+ (account F.</> (yearName ++ "-" ++ suffix ++ ".journal"), journalExample Balances help example)+ manualFiles =+ [ manual "" "pre" ["Record an entry shared across owners before their imported activity."]+ (entry "01-01" "Fund the household cash box from my cash"+ ["assets:household:cash 150.00 USD", "assets:me:cash"]),+ manual "" "post" ["Record a transfer between owners after reviewing both sets of statements."]+ (entry "01-02" "Reimburse partner for shared groceries"+ ["assets:partner:cash 35.00 USD", "assets:me:cash"]),+ manual owner "pre" ["Fill a gap in a transfer between this owner's banks before their first statements."]+ (entry "01-01" "Move savings to a new bank"+ ["assets:otherbank:savings 75.00 USD", "assets:mybank:checking"]),+ manual owner "post" ["Record a transfer between this owner's banks after reviewing their statements."]+ (entry "01-02" "Move emergency savings back to checking"+ ["assets:mybank:checking 50.00 USD", "assets:otherbank:savings"]),+ manual bank "pre" ["Record a transfer between accounts at this bank before their imported activity."]+ (entry "01-01" "Move money from checking to savings"+ ["assets:mybank:savings 25.00 USD", "assets:mybank:checking"]),+ manual bank "post" ["Fill in a transfer missing from both exports after reviewing this bank's statements."]+ (entry "01-02" "Sweep savings into checking"+ ["assets:mybank:checking 12.00 USD", "assets:mybank:savings"]),+ manual account "pre"+ ["Fill a gap before this account's first statement.", "Put opening balances in the dedicated yearly opening file."]+ (entry "01-01" "Replace debit card"+ ["expenses:bank-fees 5.00 USD", "assets:mybank:checking"]),+ manual account "post"+ ["Assert this account's balance after its imported entries for the same day.",+ "Use the balance shown by your bank. Put closing balances in the dedicated yearly closing file."]+ (entry "01-02" "Check balance after the sample statement"+ ["assets:mybank:checking 0 USD = 0 USD"])+ ]+ balanceFiles =+ [ balance "opening" ["Record what this account already held just before its first imported transaction."]+ (entry "01-01" "Checking opening balance"+ ["assets:mybank:checking 100.00 USD", "equity:opening/closing balances"]),+ balance "closing"+ ["Close this account's balance to zero when you deliberately close your books.",+ "This example assumes the starter opening balance and sample statement."]+ (entry "12-31" "Checking closing balance"+ ["assets:mybank:checking -100.00 USD = 0 USD", "equity:opening/closing balances"])+ ]++starterNotice :: T.Text+starterNotice = "Generated by hledger-flow init as a starter. This file is yours to edit. hledger-flow leaves it as you save it."++starterComment :: T.Text -> T.Text -> T.Text+starterComment prefix contents = prefix <> starterNotice <> "\n" <> contents++optionalIncludeNotice :: T.Text+optionalIncludeNotice = "; This file is optional. If absent, hledger-flow omits it from the include files it maintains."++optionalJournal :: T.Text -> T.Text+optionalJournal contents = starterComment "; " (optionalIncludeNotice <> "\n" <> contents)++journalExample :: DocLink -> [T.Text] -> [T.Text] -> T.Text+journalExample link help example = optionalJournal $ T.unlines $+ map ("; " <>) help ++ ["; Docs: " <> docURL link, "", "; Example. Remove its leading semicolons and adapt it."]+ ++ map ("; " <>) example++exampleBaseFiles :: Integer -> [(RelFile, T.Text)]+exampleBaseFiles year = paths $+ [("directives.journal", starterDirectives)]+ ++ [ (priceDir F.</> name, starterComment "; " (T.unlines+ [ "; This file is optional. Remove its include line from prices.journal to omit the pair.",+ "; Add market prices here.",+ "; P " <> date <> " " <> quote+ ]))+ | (name, quote) <- pairs+ ]+ ++ [(priceDir F.</> "prices.journal", optionalJournal (T.unlines ["include " <> T.pack name | (name, _) <- pairs]))]+ where+ priceDir = "prices" F.</> show year+ date = T.pack (show year) <> "-01-01"+ pairs = [("EUR-USD.prices", "EUR 1.10 USD"), ("BTC-USD.prices", "BTC 99000.00 USD")]++starterDirectives :: T.Text+starterDirectives = T.unlines+ [ "; Ledger-wide directives go here. https://hledger.org/dev/hledger.html#directives",+ "; " <> starterNotice,+ optionalIncludeNotice,+ "",+ "; Common commodities. https://hledger.org/dev/hledger.html#commodity-aliases",+ "commodity 1,000.00 USD ; alias: US$",+ "commodity 1,000.00 EUR ; alias: €",+ "commodity 1,000.00 GBP ; alias: £",+ "commodity 1,000.00 CAD ; alias: CA$",+ "commodity 1,000.00 AUD ; alias: A$",+ "commodity 1,000. JPY ; alias: JP¥",+ "commodity 1,000.00 CNY ; alias: CN¥, alias: RMB",+ "commodity 1,000.00 INR ; alias: ₹",+ "",+ "; Bitcoin and satoshis. https://hledger.org/dev/hledger.html#commodity-units",+ "commodity 1.00000000 BTC ; alias: 100000000 SAT, alias: ₿",+ "commodity 1. SAT",+ "",+ "; Account types. https://hledger.org/dev/hledger.html#account-types",+ "account assets ; type: A",+ "account liabilities ; type: L",+ "account equity ; type: E",+ "account revenues ; type: R",+ "account expenses ; type: X",+ "",+ "account assets:bank ; type: C",+ "account assets:cash ; type: C",+ "",+ "account equity:conversion ; type: V",+ "account equity:unrealised-gain ; type: U",+ "account revenues:gain ; type: G"+ ]++paths :: [(FilePath, a)] -> [(RelFile, a)]+paths = mapMaybe (\(p, a) -> fmap (\path -> (path, a)) (parseRelFile p))++untouchedExample :: BaseDir -> [AbsFile] -> IO (Maybe RelDir)+untouchedExample base [source]+ | Just rel <- stripProperPrefix (importRoot base) source,+ [ownerName, "mybank", "checking", "1-in", yearName, name] <- F.splitDirectories (toFilePath rel),+ name == toFilePath sampleName,+ Just starter <- parseRelDir ownerName,+ Just year <- readMaybe yearName,+ Just generated <- parseRelFile ("3-journal" F.</> yearName F.</> "hledger-flow-example.journal"),+ Just yearIndex <- parseRelFile (yearName ++ "-include.journal") = do+ let account = sampleAccountDir base starter+ result <- tryIOError $ do+ let rules = account </> rulesName+ journal = account </> generated+ bank = parent account+ owner = parent bank+ indexes dir = [dir </> yearIndex, dir </> [relfile|all-years.journal|]]+ otherStarters =+ [ (file, contents)+ | (relative, contents) <- exampleImportFiles starter year,+ let file = importRoot base </> relative,+ isProperPrefixOf owner file, file /= source, file /= rules+ ]+ allowed = [source, rules, journal] ++ map fst otherStarters ++ concatMap indexes [owner, bank, account]+ dirs = concatMap (takeWhile (isProperPrefixOf owner) . iterate parent . parent) allowed+ optionalStarter (file, contents) = do+ exists <- PathIO.doesFileExist file+ if exists then matches file contents else pure True+ cleanPath <- allM plainDir [importRoot base, owner]+ cleanContents <- if cleanPath then containsOnly dirs allowed owner else return False+ if cleanContents then do+ current <- (&&) <$> matches source (sampleCsv year)+ <*> matches rules (starterComment "# " sampleRules)+ sampleMatches <- if current then pure True+ else (&&) <$> matches source (T.unlines (sampleCsvRows year)) <*> matches rules sampleRules+ if sampleMatches then allM optionalStarter otherStarters else pure False+ else pure False+ return (case result of Right True -> Just (exampleAccount starter); _ -> Nothing)+ where+ matches file expected = (== T.encodeUtf8 expected) <$> B.readFile (toFilePath file)+untouchedExample _ _ = return Nothing++containsOnly :: [AbsDir] -> [AbsFile] -> AbsDir -> IO Bool+containsOnly allowedDirs allowedFiles dir = do+ (dirs, files) <- PathIO.listDir dir+ if all (`elem` allowedDirs) dirs && all (`elem` allowedFiles) files+ then do+ hasLinks <- (||) <$> (or <$> traverse PathIO.isSymlink dirs) <*> (or <$> traverse PathIO.isSymlink files)+ if hasLinks then return False else and <$> traverse (containsOnly allowedDirs allowedFiles) dirs+ else return False++importRoot :: BaseDir -> AbsDir+importRoot base = base </> [reldir|import|]++sampleAccountDir :: BaseDir -> RelDir -> AbsDir+sampleAccountDir base owner = importRoot base </> exampleAccount owner++-- Starter declarations and comments do not add transactions to the ledger.+directivesAreStarter :: BaseDir -> IO Bool+directivesAreStarter base = do+ let file = base </> [relfile|directives.journal|]+ exists <- pathOccupied file+ if exists+ then elem <$> readJournalBody file <*> pure [Just [], Just (journalBody starterDirectives)]+ else pure True++journalIsCommentsOnly :: AbsFile -> IO Bool+journalIsCommentsOnly file = (== Just []) <$> readJournalBody file++readJournalBody :: AbsFile -> IO (Maybe [T.Text])+readJournalBody file = do+ regular <- regularFile file+ if regular then Just . journalBody <$> TIO.readFile (toFilePath file) else pure Nothing++journalBody :: T.Text -> [T.Text]+journalBody = filter (\line -> not (T.null line || T.isPrefixOf ";" line)) . map T.strip . T.lines++plainDir :: AbsDir -> IO Bool+plainDir dir = do+ exists <- PathIO.doesDirExist dir+ if exists then not <$> PathIO.isSymlink dir else pure False++regularFile :: AbsFile -> IO Bool+regularFile file = do+ exists <- PathIO.doesFileExist file+ if exists then not <$> PathIO.isSymlink file else pure False++allM :: (a -> IO Bool) -> [a] -> IO Bool+allM check = fmap and . traverse check
+ src/Hledger/Flow/InputSnapshot.hs view
@@ -0,0 +1,138 @@+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}++module Hledger.Flow.InputSnapshot+ ( InputSnapshot,+ snapshotInputs,+ inputTargets,+ inputsMatch,+ inputsUnchanged,+ )+where++import Control.Exception (IOException, try)+import Control.Monad (filterM)+import Data.Aeson (FromJSON, ToJSON)+import Data.Char (isSpace)+import Data.List (nub)+import Data.Maybe (fromMaybe)+import qualified Data.Set as Set+import qualified Data.Text as T+import qualified Data.Text.IO as T+import Data.Time.Clock (UTCTime)+import GHC.Generics (Generic)+import Hledger.Flow.PathHelpers (AbsFile, isGlobMetacharacter)+import Path (toFilePath)+import qualified System.Directory as Directory+import qualified System.FilePath as F++data Entry = Entry FilePath FilePath UTCTime+ deriving (Eq, Generic, Show)++instance FromJSON Entry+instance ToJSON Entry++data InputSnapshot = InputSnapshot [Entry] [Entry]+ deriving (Eq, Generic, Show)++instance FromJSON InputSnapshot+instance ToJSON InputSnapshot++entry :: FilePath -> IO Entry+entry file = Entry file <$> Directory.canonicalizePath file <*> Directory.getModificationTime file++snapshotInputs :: [AbsFile] -> IO InputSnapshot+snapshotInputs inputs = do+ let files = nub (map toFilePath inputs)+ inputsWithTargets <- traverse entry files+ watched <- concat <$> traverse (\(Entry _ target _) -> globDirectories target) inputsWithTargets+ InputSnapshot inputsWithTargets <$> traverse entry (nub watched)++inputTargets :: InputSnapshot -> [FilePath]+inputTargets (InputSnapshot files _) = [target | Entry _ target _ <- files]++inputsMatch :: InputSnapshot -> IO Bool+inputsMatch (InputSnapshot files dirs) = do+ let entries = files ++ dirs+ result <- try (traverse (\(Entry file _ _) -> entry file) entries)+ return $ either (const False :: IOException -> Bool) (== entries) result++inputsUnchanged :: InputSnapshot -> UTCTime -> IO Bool+inputsUnchanged snapshot@(InputSnapshot files dirs) since =+ if all (\(Entry _ _ modified) -> modified < since) (files ++ dirs)+ then inputsMatch snapshot+ else return False++globDirectories :: FilePath -> IO [FilePath]+globDirectories file = do+ contents <- T.readFile file+ let includes =+ [ T.unpack (readerPath arg)+ | line <- T.lines (T.dropWhile (== '\xFEFF') contents),+ Just rest <- [T.stripPrefix "include" line],+ Just (first, _) <- [T.uncons rest],+ isSpace first,+ let arg = T.strip (T.takeWhile (/= ';') rest),+ not (T.null arg)+ ]+ concat <$> mapM watch includes+ where+ -- hledger-flow emits singleton character ranges for literal metacharacters.+ -- Track wildcard branches conservatively, including parents of missing branches.+ readerPath arg =+ let (prefix, path) = T.breakOn ":" arg+ in if prefix `elem` ["journal", "timeclock", "timedot", "csv", "tsv", "ssv"]+ then fromMaybe arg (T.stripPrefix ":" path)+ else arg+ watch ('~' : separator : path) | separator `elem` ("/\\" :: String) =+ Directory.getHomeDirectory >>= \home -> watchUnder home path+ watch path = watchUnder (F.takeDirectory file) path+ watchUnder parent path =+ let parts = concatMap expandRecursive (F.splitDirectories path)+ (fixed, patternParts) = span (not . snd . decodeLiteral) parts+ root = (if F.isAbsolute path then id else (parent F.</>))+ (F.joinPath (map (fst . decodeLiteral) fixed))+ in case patternParts of+ [] -> return []+ _ -> watchFrom Set.empty root patternParts++-- hledger also interprets ** with a filename suffix as a recursive glob.+expandRecursive :: String -> [String]+expandRecursive = go []+ where+ go before ('*' : '*' : suffix@(_ : _)) = (reverse before ++ "**") : go [] ('*' : suffix)+ go before (c : rest) = go (c : before) rest+ go before [] = [reverse before]++watchFrom :: Set.Set (FilePath, [FilePath]) -> FilePath -> [FilePath] -> IO [FilePath]+watchFrom _ _ [] = return []+watchFrom visited dir (part : rest) = do+ exists <- Directory.doesDirectoryExist dir+ if not exists+ then return []+ else do+ resolved <- Directory.canonicalizePath dir+ if Set.member (resolved, part : rest) visited+ then return [dir]+ else do+ let next = Set.insert (resolved, part : rest) visited+ (literal, glob) = decodeLiteral part+ descend <- case (part, glob, rest) of+ (_, _, []) | part /= "**" -> return []+ ("**", _, _) -> do+ children <- subdirs dir+ here <- watchFrom visited dir rest+ below <- concat <$> traverse (\child -> watchFrom next child (part : rest)) children+ return (here ++ below)+ (_, True, _) -> subdirs dir >>= fmap concat . traverse (\child -> watchFrom next child rest)+ _ -> watchFrom next (dir F.</> literal) rest+ return (dir : descend)+ where+ subdirs p = Directory.listDirectory p >>= filterM Directory.doesDirectoryExist . map (p F.</>)++decodeLiteral :: String -> (String, Bool)+decodeLiteral ('[' : x : '-' : y : ']' : rest) | x == y, isGlobMetacharacter x =+ let (decoded, glob) = decodeLiteral rest in (x : decoded, glob)+decodeLiteral (x : rest) =+ let (decoded, glob) = decodeLiteral rest in (x : decoded, isGlobMetacharacter x || glob)+decodeLiteral [] = ("", False)
+ src/Hledger/Flow/LedgerSelection.hs view
@@ -0,0 +1,184 @@+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}++module Hledger.Flow.LedgerSelection+ ( LedgerFile (..),+ Coverage (..),+ LedgerSelection (..),+ selectLedger,+ journalInputs,+ coverageFor,+ ledgerFileProblem,+ )+where++import Control.Exception (IOException, try)+import Control.Monad (when)+import Data.Aeson (FromJSON, ToJSON, decodeFileStrict', encodeFile)+import qualified Data.ByteString as B+import qualified Data.Text as T+import Data.Time.Clock (UTCTime)+import GHC.Generics (Generic)+import Hledger.Flow.Cache (cacheDir, initializeCache)+import Hledger.Flow.Common (configuredLedgerFile, hledgerFileArgument, runProcessCaptured, tildePath)+import Hledger.Flow.InputSnapshot+import Hledger.Flow.PathHelpers (AbsFile, withStagedOutput)+import Hledger.Flow.Types (BaseDir, HledgerInfo (..))+import Path+import qualified Path.IO as PathIO+import qualified System.Directory as Directory+import System.Exit (ExitCode (..))+import qualified System.FilePath as F++data Coverage = CoversLedger | UnrelatedLedger | UnreadableLedger T.Text+ deriving (Eq, Show)++data LedgerFile = LedgerFile+ { lfValue :: FilePath,+ lfExists :: Bool,+ lfCoverage :: Coverage+ }+ deriving (Eq, Show)++data LedgerSelection = LedgerSelection+ { selectedLedgerFile :: Maybe LedgerFile,+ aggregateJournal :: Maybe AbsFile,+ selectionError :: Maybe T.Text+ }+ deriving (Show)++data CachedInputs = CachedInputs FilePath FilePath T.Text UTCTime InputSnapshot+ deriving (Generic)++instance FromJSON CachedInputs+instance ToJSON CachedInputs++journalInputs :: HledgerInfo -> [AbsFile] -> IO (Either T.Text [AbsFile])+journalInputs info files = do+ let args = ["--no-conf", "files"] ++ concatMap (\file -> ["--file", hledgerFileArgument info (toFilePath file)]) files+ result <- try (runProcessCaptured (T.pack (toFilePath (hlPath info)), args, Nothing))+ case result of+ Left err -> return (Left (T.pack (show (err :: IOException))))+ Right (ExitFailure _, _, err) -> return (Left (T.strip err))+ Right (ExitSuccess, out, _) -> do+ let paths = filter (not . T.null) (map (T.dropWhileEnd (== '\r')) (T.lines out))+ case paths of+ [] -> return (Left "Could not read the paths printed by hledger files.")+ _ -> do+ resolved <- try (traverse (resolveLedgerPath . T.unpack) paths)+ return (either (Left . T.pack . show) Right (resolved :: Either IOException [AbsFile]))++coverageFor :: FilePath -> Maybe FilePath -> [FilePath] -> Coverage+coverageFor index directives files+ | index `elem` files && maybe True (`elem` files) directives = CoversLedger+ | otherwise = UnrelatedLedger++ledgerFileProblem :: [FilePath] -> LedgerFile -> T.Text+ledgerFileProblem homes file+ | not (lfExists file) = "LEDGER_FILE is set to " <> displayed <> ", which doesn't exist."+ | otherwise = case lfCoverage file of+ CoversLedger -> ""+ UnrelatedLedger -> "LEDGER_FILE (" <> displayed <> ") does not include this ledger."+ UnreadableLedger err -> "Could not read LEDGER_FILE (" <> displayed <> "): " <> err+ where+ displayed = T.pack (tildePath homes (lfValue file))++selectLedger :: [FilePath] -> Maybe HledgerInfo -> BaseDir -> IO LedgerSelection+selectLedger homes info base = do+ let root = base </> [relfile|all-years.journal|]+ rootExists <- Directory.doesFileExist (toFilePath root)+ configured <- configuredLedgerFile+ let missingRoot = "No all-years.journal in " <> T.pack (tildePath homes (toFilePath base)) <> " yet. Run `hledger-flow import` first."+ fallback value+ | rootExists = LedgerSelection value (Just root) Nothing+ | otherwise = LedgerSelection value Nothing (Just missingRoot)+ case configured of+ Nothing -> return (fallback Nothing)+ Just (value, path) -> do+ exists <- Directory.doesFileExist path+ if not exists+ then do+ let file = LedgerFile value False UnrelatedLedger+ return (LedgerSelection (Just file) Nothing (Just (ledgerFileProblem homes file <>+ " Point it at " <> T.pack (tildePath homes (toFilePath root)) <> ", or unset it.")))+ else do+ absolute <- resolveLedgerPath path+ canonical <- Directory.canonicalizePath (toFilePath absolute)+ rootPath <- Directory.canonicalizePath (toFilePath root)+ coverage <- if rootExists && canonical == rootPath+ then return CoversLedger+ else case info of+ Nothing -> return (UnreadableLedger "hledger is unavailable.")+ Just hl -> discoverCoverage hl base absolute+ let file = Just (LedgerFile value True coverage)+ return $ case coverage of+ CoversLedger -> LedgerSelection file (Just absolute) Nothing+ _ -> fallback file++-- Path excludes parent references. Resolve only the directory when needed+-- so hledger still reads a file symlink relative to its original location.+resolveLedgerPath :: FilePath -> IO AbsFile+resolveLedgerPath path = do+ absolute <- Directory.makeAbsolute path+ case parseAbsFile absolute of+ Just file -> return file+ Nothing -> do+ dir <- Directory.canonicalizePath (F.takeDirectory absolute)+ parseAbsFile (dir F.</> F.takeFileName absolute)++discoverCoverage :: HledgerInfo -> BaseDir -> AbsFile -> IO Coverage+discoverCoverage info base journal = do+ let cache = cacheDir base </> [relfile|ledger-coverage.json|]+ path = toFilePath journal+ executable = toFilePath (hlPath info)+ version = hlVersion info+ cached <- try (decodeFileStrict' (toFilePath cache))+ reusable <- case cached of+ Right (Just (CachedInputs oldPath oldExecutable oldVersion start snapshot))+ | (oldPath, oldExecutable, oldVersion) == (path, executable, version) -> do+ unchanged <- inputsUnchanged snapshot start+ return (if unchanged then Just (inputTargets snapshot) else Nothing)+ Left (_ :: IOException) -> return Nothing+ _ -> return Nothing+ inputs <- case reusable of+ Just paths -> return (Right paths)+ Nothing -> do+ scanned <- try $ do+ initializeCache base+ PathIO.withTempDir (parent cache) ".ledger-coverage-" $ \dir -> do+ let marker = dir </> [relfile|started|]+ B.writeFile (toFilePath marker) B.empty+ start <- PathIO.getModificationTime marker+ files <- journalInputs info [journal]+ return (Just start, files)+ (started, files) <- case scanned of+ Right result -> return result+ Left (_ :: IOException) -> do+ files <- journalInputs info [journal]+ return (Nothing, files)+ case files of+ Left err -> return (Left err)+ Right paths -> do+ snapshot <- try (snapshotInputs paths)+ case snapshot of+ Right saved -> do+ case started of+ Just start -> do+ unchanged <- inputsUnchanged saved start+ when unchanged $ do+ _ <- try (withStagedOutput cache (\stage ->+ encodeFile (toFilePath stage) (CachedInputs path executable version start saved))) :: IO (Either IOException ())+ return ()+ Nothing -> return ()+ return (Right (inputTargets saved))+ Left (_ :: IOException) -> Right <$> traverse (Directory.canonicalizePath . toFilePath) paths+ case inputs of+ Left err -> return (UnreadableLedger err)+ Right paths -> do+ index <- Directory.canonicalizePath (toFilePath (base </> [relfile|import/all-years.journal|]))+ let directives = toFilePath (base </> [relfile|directives.journal|])+ hasDirectives <- Directory.doesFileExist directives+ required <- if hasDirectives then Just <$> Directory.canonicalizePath directives else return Nothing+ return (coverageFor index required paths)
src/Hledger/Flow/PathHelpers.hs view
@@ -1,15 +1,25 @@+{-# LANGUAGE CPP #-} {-# LANGUAGE OverloadedStrings #-} module Hledger.Flow.PathHelpers where import Control.Monad.Catch (Exception, MonadThrow, throwM) import Control.Monad.IO.Class (MonadIO, liftIO)+import qualified Data.ByteString as B import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import Data.Time.Clock (UTCTime) import Hledger.Flow.DocHelpers (DocLink (..), docURL) import qualified Path import qualified Path.IO as Path-import System.Directory (doesDirectoryExist)+import System.Directory (doesDirectoryExist, doesPathExist, pathIsSymbolicLink)+import System.IO.Error (catchIOError, isDoesNotExistError) import qualified System.FilePath as FilePath+#ifdef mingw32_HOST_OS+import qualified System.Win32.File as Win32+#else+import qualified System.Posix.Files as Posix+#endif type AbsFile = Path.Path Path.Abs Path.File @@ -20,14 +30,63 @@ type RelDir = Path.Path Path.Rel Path.Dir -- | Publish a complete file by renaming it on the destination filesystem.-withStagedOutput :: AbsFile -> (AbsFile -> IO ()) -> IO ()+withStagedOutput :: AbsFile -> (AbsFile -> IO a) -> IO a withStagedOutput target writeOutput = do let dir = Path.parent target Path.createDirIfMissing True dir Path.withTempDir dir ".hledger-flow-" $ \tmp -> do let staged = tmp Path.</> Path.filename target- writeOutput staged+ result <- writeOutput staged Path.renameFile staged target+ return result++-- | Use this operation's own filesystem timestamp, even if another process+-- publishes a marker at the same destination.+writeTimestampMarker :: AbsFile -> IO UTCTime+writeTimestampMarker target = withStagedOutput target $ \stage -> do+ B.writeFile (Path.toFilePath stage) B.empty+ Path.getModificationTime stage++-- | Install complete UTF-8 bytes at an unused path. The final operation must+-- fail for an existing entry, including a dangling symlink.+writeNewUtf8File :: AbsFile -> T.Text -> IO ()+writeNewUtf8File target contents = do+ let dir = Path.parent target+ Path.createDirIfMissing True dir+ Path.withTempDir dir ".hledger-flow-" $ \tmp -> do+ let staged = tmp Path.</> Path.filename target+ B.writeFile (Path.toFilePath staged) (T.encodeUtf8 contents)+ publishNewFile staged target++-- | Publish a staged file exclusively, without replacing an existing target.+publishNewFile :: AbsFile -> AbsFile -> IO ()+publishNewFile staged target =+#ifdef mingw32_HOST_OS+ Win32.moveFile (extendedWindowsPath src) (extendedWindowsPath dest)+#else+ Posix.createLink src dest+#endif+ where+ src = Path.toFilePath staged+ dest = Path.toFilePath target++#ifdef mingw32_HOST_OS+-- The raw Win32 move needs the file namespace for paths beyond MAX_PATH.+-- GHC file writes already use that namespace for ordinary absolute paths.+extendedWindowsPath :: FilePath -> FilePath+extendedWindowsPath path = case FilePath.normalise path of+ absolute@('\\' : '\\' : marker : '\\' : _) | marker == '?' || marker == '.' -> absolute+ '\\' : '\\' : unc -> "\\\\?\\UNC\\" ++ unc+ absolute -> "\\\\?\\" ++ absolute+#endif++pathOccupied :: Path.Path Path.Abs t -> IO Bool+pathOccupied file = do+ let name = FilePath.dropTrailingPathSeparator (Path.toFilePath file)+ exists <- doesPathExist name+ if exists then return True else+ pathIsSymbolicLink name `catchIOError` \err ->+ if isDoesNotExistError err then return False else ioError err isGlobMetacharacter :: Char -> Bool isGlobMetacharacter c = c `elem` ("*?[{<" :: String)
+ src/Hledger/Flow/ReportReview.hs view
@@ -0,0 +1,199 @@+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}++module Hledger.Flow.ReportReview+ ( Review (..),+ ReviewCategory (..),+ ReviewItem (..),+ ReviewFile,+ ReviewKind (..),+ captureReportInputs,+ countTransferAccounts,+ countUnknownTransactions,+ invalidateReportReview,+ loadReportReview,+ reviewFile,+ reviewHasData,+ saveReportReview,+ startReportRun,+ )+where++import Control.Exception (IOException, try)+import Data.Aeson (FromJSON, ToJSON, decodeFileStrict', encodeFile)+import Data.Char (isDigit, isSpace)+import Data.List (nub, sortOn)+import Data.Maybe (mapMaybe)+import Data.Ord (Down (..))+import qualified Data.Text as T+import qualified Data.Text.IO as T+import Data.Time.Clock (UTCTime)+import GHC.Generics (Generic)+import Hledger.Flow.InputSnapshot (InputSnapshot, inputsUnchanged, snapshotInputs)+import Hledger.Flow.Cache (cacheDir, initializeCache)+import Hledger.Flow.LedgerSelection (journalInputs)+import Hledger.Flow.PathHelpers (AbsFile, RelFile, withStagedOutput, writeTimestampMarker)+import Hledger.Flow.Types (BaseDir, HledgerConf, HledgerInfo, hledgerConfPath)+import Path+import qualified Path.IO as PathIO+import qualified System.Directory as D+import qualified System.FilePath as F+import System.IO.Error (catchIOError, isDoesNotExistError)+import Text.Read (readMaybe)++data ReviewKind = UnknownTransactions | TransferBalances+ deriving (Eq, Generic, Show, FromJSON, ToJSON)++data ReviewFile = ReviewFile ReviewKind RelFile Integer+ deriving (Generic, FromJSON, ToJSON)++data ReviewItem = ReviewItem+ { itemKind :: ReviewKind,+ itemPath :: FilePath,+ itemYear :: Integer,+ itemCount :: Maybe Int+ }+ deriving (Show)++data ReviewCategory = ReviewCategory {reviewCount :: Maybe Int, reviewFiles :: [FilePath]}+ deriving (Show)++data Review = Review+ { reviewUnknowns :: ReviewCategory,+ reviewTransfers :: ReviewCategory,+ reviewItems :: [ReviewItem]+ }+ deriving (Show)++data ReportInputs = ReportInputs+ { runStarted :: UTCTime,+ runAggregate :: AbsFile,+ runConfig :: Maybe AbsFile,+ runInputs :: InputSnapshot+ }+ deriving (Generic, FromJSON, ToJSON)++data Manifest = Manifest ReportInputs [ReviewFile]+ deriving (Generic, FromJSON, ToJSON)++manifestFile :: BaseDir -> AbsFile+manifestFile base = cacheDir base </> [relfile|reports.json|]++invalidateReportReview :: BaseDir -> IO ()+invalidateReportReview base =+ PathIO.removeFile (manifestFile base) `catchIOError` \err ->+ if isDoesNotExistError err then return () else ioError err++startReportRun :: BaseDir -> IO UTCTime+startReportRun base = do+ initializeCache base+ invalidateReportReview base+ writeTimestampMarker (cacheDir base </> [relfile|report-started|])++configPath :: Maybe HledgerConf -> IO (Maybe AbsFile)+configPath = traverse ((>>= parseAbsFile) . D.canonicalizePath . hledgerConfPath)++captureReportInputs :: HledgerInfo -> UTCTime -> AbsFile -> Maybe HledgerConf -> [[AbsFile]] -> IO (Either T.Text ReportInputs)+captureReportInputs info started aggregate conf roots = do+ discovered <- traverse (journalInputs info) (nub roots)+ case sequence discovered of+ Left err -> return (Left err)+ Right paths -> do+ result <- try $ do+ config <- configPath conf+ snapshot <- snapshotInputs (maybe id (:) config (concat paths))+ return (ReportInputs started aggregate config snapshot)+ return (either (Left . T.pack . show) Right (result :: Either IOException ReportInputs))++reviewFile :: BaseDir -> AbsFile -> Maybe ReviewFile+reviewFile base file = do+ kind <- case toFilePath (filename file) of+ "unknown-transactions.txt" -> Just UnknownTransactions+ "transfer-balance.txt" -> Just TransferBalances+ _ -> Nothing+ rel <- stripProperPrefix base file+ year <- readMaybe (F.dropTrailingPathSeparator (toFilePath (dirname (parent file))))+ return (ReviewFile kind rel year)++saveReportReview :: BaseDir -> ReportInputs -> [ReviewFile] -> IO ()+saveReportReview base inputs files =+ withStagedOutput (manifestFile base) (\stage -> encodeFile (toFilePath stage) (Manifest inputs files))++loadReportReview :: BaseDir -> AbsFile -> Maybe HledgerConf -> IO (Maybe Review)+loadReportReview base aggregate conf = do+ result <- try $ do+ saved <- decodeFileStrict' (toFilePath (manifestFile base))+ case saved of+ Nothing -> return Nothing+ Just (Manifest inputs files) -> do+ currentConfig <- configPath conf+ if runConfig inputs /= currentConfig || runAggregate inputs /= aggregate+ then return Nothing+ else do+ current <- inputsUnchanged (runInputs inputs) (runStarted inputs)+ if current then Just . summarize <$> traverse readReview files else return Nothing+ return (either (const Nothing :: IOException -> Maybe Review) id result)+ where+ readReview (ReviewFile kind rel year) = do+ text <- T.readFile (toFilePath (base </> rel))+ let body = T.unlines (drop 2 (T.lines text))+ count = if T.null (T.strip body) then Nothing else countFor kind body+ return (ReviewItem kind (toFilePath rel) year count)+ summarize items =+ let ordered = sortOn (\item -> (Down (itemYear item), itemPath item)) items+ in Review (category UnknownTransactions ordered) (category TransferBalances ordered) ordered+ category kind items =+ let relevant = filter ((== kind) . itemKind) items+ in ReviewCategory (sum <$> traverse itemCount relevant) (map itemPath relevant)++countFor :: ReviewKind -> T.Text -> Maybe Int+countFor UnknownTransactions = countUnknownTransactions+countFor TransferBalances = countTransferAccounts++reviewHasData :: ReviewKind -> T.Text -> Bool+reviewHasData kind text = case filter (not . T.null) (map T.strip (T.lines text)) of+ [] -> False+ [line] | csvHeader kind line -> False+ _ -> countFor kind text /= Just 0++csvHeader :: ReviewKind -> T.Text -> Bool+csvHeader UnknownTransactions = T.isPrefixOf "\"txnidx\",\"date\","+csvHeader TransferBalances = T.isPrefixOf "\"account\","++countUnknownTransactions :: T.Text -> Maybe Int+countUnknownTransactions text+ | null rows = Just 0+ | not (null headers) && all known rows = Just (length headers)+ | otherwise = Nothing+ where+ rows = filter (not . T.null . T.strip) (T.lines text)+ headers = filter dateHeader rows+ known row = dateHeader row || maybe False (isSpace . fst) (T.uncons row) || ";" `T.isPrefixOf` row+ dateHeader row = case T.unpack (T.take 10 row) of+ [a,b,c,d,'-',e,f,'-',g,h] ->+ all isDigit [a,b,c,d,e,f,g,h]+ && maybe True (\(end, _) -> isSpace end || end == '=') (T.uncons (T.drop 10 row))+ _ -> False++countTransferAccounts :: T.Text -> Maybe Int+countTransferAccounts text = case filter (not . T.null) (map T.strip (T.lines text)) of+ [] -> Just 0+ title : table | "Balance changes in " `T.isPrefixOf` title ->+ case break divider table of+ (heading, _ : body)+ | any (\line -> "||" `T.isInfixOf` line || "║" `T.isInfixOf` line) heading ->+ let rows = filter (not . T.isPrefixOf "└") body+ in if all accountRow rows then Just (length rows) else Nothing+ _ -> Nothing+ _ -> Nothing+ where+ divider row = "╞" `T.isPrefixOf` row || ("=" `T.isPrefixOf` row && "++" `T.isInfixOf` row)+ accountRow row = case mapMaybe (\sep -> nonemptyField sep row) ["||", "║"] of+ [_] -> True+ _ -> False+ nonemptyField sep row =+ let (label, rest) = T.breakOn sep row+ in if T.null rest || T.null (T.strip (T.dropWhile (== '│') label)) then Nothing else Just label
src/Hledger/Flow/Reports.hs view
@@ -7,9 +7,10 @@ where import Control.Concurrent.STM-import Control.Monad (filterM, unless, when)+import Control.Monad (filterM, when) import Data.Char (toLower) import Data.Either+import Data.Maybe (mapMaybe) import qualified Data.List as List import qualified Data.Text as T import qualified Data.Text.IO as T@@ -17,14 +18,15 @@ import Hledger.Flow.Environment (renderSettings, reportSettings) import Hledger.Flow.BaseDir (relativeToBase) import Hledger.Flow.Common+import Hledger.Flow.LedgerSelection import Hledger.Flow.Logging import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile)+import Hledger.Flow.ReportReview import Hledger.Flow.RuntimeOptions import Hledger.Flow.Status (plural) import qualified Hledger.Flow.Types as FlowTypes-import Path ((</>), parent, reldir, relfile, parseRelDir, toFilePath)+import Path ((</>), parent, reldir, relfile, parseAbsFile, parseRelDir, toFilePath) import qualified Path.IO as PathIO-import qualified System.Directory as Directory import Prelude hiding (putStrLn, readFile, writeFile) data ReportParams = ReportParams@@ -51,15 +53,21 @@ generateReports' :: RuntimeOptions -> TChan FlowTypes.LogMessage -> IO [Either AbsFile AbsFile] generateReports' opts ch = do+ started <- startReportRun (baseDir opts) owners <- listOwners opts- configured <- configuredLedgerFile- let hledgerJournal = maybe (toFilePath (baseDir opts </> [relfile|all-years.journal|])) snd configured- hledgerJournalExists <- Directory.doesFileExist hledgerJournal+ info <- hledgerInfo opts homes <- homeDirs- unless hledgerJournalExists $ ioError $ userError $ case configured of- Just (v, _) -> "LEDGER_FILE is set to " ++ v ++ ", which doesn't exist. Point it at "- ++ tildePath homes (toFilePath (baseDir opts </> [relfile|all-years.journal|])) ++ ", or unset it."- Nothing -> "No all-years.journal in " ++ tildePath homes (toFilePath (baseDir opts)) ++ " yet. Run `hledger-flow import` first."+ selection <- selectLedger homes (Just info) (baseDir opts)+ journal <- case (aggregateJournal selection, selectionError selection) of+ (Just file, _) -> return file+ (_, Just problem) -> ioError (userError (T.unpack problem))+ _ -> ioError (userError "No journal selected for reporting.")+ let hledgerJournal = toFilePath journal+ case selectedLedgerFile selection of+ Just file | lfCoverage file == CoversLedger ->+ channelOutLn ch ("Reporting on " <> T.pack (tildePath homes hledgerJournal) <> " (from LEDGER_FILE).")+ Just file -> channelOutLn ch (ledgerFileProblem homes file <> " Reporting on all-years.journal.")+ Nothing -> return () let journalWithYears = baseDir opts </> [relfile|import/all-years.journal|] let aggregateReportDir | any ((== toFilePath [reldir|all|]) . map toLower . toFilePath) owners = baseDir opts </> [reldir|reports/_all_|]@@ -77,7 +85,13 @@ let sharedOptions = (if prettyReports opts then ["--pretty-tables"] else []) ++ ["--depth", "2"] let ownerWithAggregateReports = List.concat $ fmap (reportActions opts ch [incomeStatement sharedOptions, incomeMonthlyStatement sharedOptions, balanceSheet sharedOptions]) ownerWithAggregateParams let ownerOnlyReports = List.concat $ fmap (reportActions opts ch [accountList, unknownTransactions]) ownerParams- parAwareActions opts (aggregateOnlyReports ++ ownerWithAggregateReports ++ ownerOnlyReports)+ roots <- traverse (traverse parseAbsFile . ledgerFiles) ownerWithAggregateParams+ inputs <- captureReportInputs info started journal (hledgerConf opts) roots+ results <- parAwareActions opts (aggregateOnlyReports ++ ownerWithAggregateReports ++ ownerOnlyReports)+ case inputs of+ Left problem -> logVerbose opts ch ("Could not record report inputs: " <> problem)+ Right available -> saveReportReview (baseDir opts) available (mapMaybe (reviewFile (baseDir opts)) (rights results))+ return results reportActions :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [ReportGenerator] -> ReportParams -> [IO (Either AbsFile AbsFile)] reportActions opts ch reports (ReportParams journals years reportsDir) = do@@ -92,7 +106,7 @@ unknownTransactions :: ReportGenerator unknownTransactions opts ch journals baseOutDir year = do let reportArgs = ["print", "unknown"]- generateReport opts ch journals year baseOutDir [relfile|unknown-transactions.txt|] reportArgs (not . T.null)+ generateReport opts ch journals year baseOutDir [relfile|unknown-transactions.txt|] reportArgs (reviewHasData UnknownTransactions) incomeStatement :: [T.Text] -> ReportGenerator incomeStatement sharedOptions opts ch journals baseOutDir year = do@@ -112,7 +126,7 @@ transferBalance :: ReportGenerator transferBalance opts ch journals baseOutDir year = do let reportArgs = ["balance"] ++ (if prettyReports opts then ["--pretty-tables"] else []) ++ ["--quarterly", "--flat", "--no-total", "transfer"]- generateReport opts ch journals year baseOutDir [relfile|transfer-balance.txt|] reportArgs (\txt -> (length $ T.lines txt) > 4)+ generateReport opts ch journals year baseOutDir [relfile|transfer-balance.txt|] reportArgs (reviewHasData TransferBalances) generateReport :: RuntimeOptions -> TChan FlowTypes.LogMessage -> [FilePath] -> Integer -> AbsDir -> RelFile -> [T.Text] -> (T.Text -> Bool) -> IO (Either AbsFile AbsFile) generateReport opts logCh journals year reportsDir fileName args successCheck = do
src/Hledger/Flow/RulesTemplate.hs view
@@ -8,43 +8,32 @@ templateFields, rulesTemplate, rulesTemplateFor,+ rulesStarterFor,+ runRules, ) where -import Data.Char (isAlphaNum, isDigit, toLower)+import Data.Char (isAlpha, isAlphaNum, isDigit, toLower) import qualified Data.List as List import Data.Maybe (fromMaybe) import qualified Data.Set as Set import qualified Data.Text as T import Hledger.Flow.BaseDir (relativeToBase') import Hledger.Flow.DocHelpers (DocLink (..), docURL)+import Hledger.Flow.Common (homeDirs, nativeShell, tildePath)+import Hledger.Flow.Import.ImportHelpers (defaultRulesCandidates, extractImportDirs) import Hledger.Flow.Import.Inventory (findImportFiles, stageFiles)+import Hledger.Flow.Init.Example (isSampleStatement)+import Hledger.Flow.PathHelpers (AbsFile, writeNewUtf8File)+import Hledger.Flow.StatementScan+import Hledger.Flow.Status (flowCommand) import Hledger.Flow.Status.Gather (accountName, cachedStatement)-import Hledger.Flow.Types (BaseDir)+import Hledger.Flow.Types (BaseDir, HledgerConf) import Path-import qualified System.FilePath as FilePath-import System.IO (IOMode (ReadMode), hIsEOF, hSetEncoding, mkTextEncoding, withFile)+import qualified Path.IO as PathIO import qualified Data.Text.IO as T---- | The separator and the column names of a header line. Strips a UTF-8 BOM,--- surrounding quotes and spaces.--- ponytail: a quoted name containing the separator is split; use a CSV parser if that bites.-splitHeader :: T.Text -> (Char, [T.Text])-splitHeader line = (sep, map clean (T.splitOn (T.singleton sep) l))- where- l = T.dropWhileEnd (== '\r') (T.dropWhile (== '\xFEFF') line)- count c = T.count (T.singleton c) l- sep = List.foldl' (\best c -> if count c > count best then c else best) ',' [';', '\t']- clean = T.strip . unquote . T.strip- unquote t- | T.length t >= 2 && T.head t == '"' && T.last t == '"' = T.init (T.tail t)- | otherwise = t---- | False when any column looks like a date or an amount.-looksLikeHeader :: [T.Text] -> Bool-looksLikeHeader = not . any dataish- where- dataish f = not (T.null f) && T.all (`elem` ("0123456789-/.:,+$€£ " :: String)) f+import System.IO (Newline (..), nativeNewline)+import System.IO.Error (tryIOError, isAlreadyExistsError) -- | hledger field names for the columns: obvious date, description and amount -- columns are named for hledger; the rest keep a normalised version of their name.@@ -91,18 +80,24 @@ -- | The starter rules file for a statement, from its contents. rulesTemplate :: T.Text -> T.Text -> FilePath -> T.Text -> Either T.Text T.Text rulesTemplate bank account path contents- | map toLower (FilePath.takeExtension path) `notElem` [".csv", ".tsv", ".ssv"] = Left (notCsv path)- | T.null (T.strip line) || not (looksLikeHeader names) =- Left ("The first line of " <> T.pack path <> " looks like data, not column names. Write the fields line yourself: " <> docURL RulesFiles)- | otherwise =- Right . T.unlines $+ | not (isDelimitedFile path) = Left (notCsv path)+ | otherwise = renderTemplate bank account path (parseRuleSample path contents)++renderTemplate :: T.Text -> T.Text -> FilePath -> Either T.Text DelimitedStatement -> Either T.Text T.Text+renderTemplate bank account path result = case result of+ Left problem -> Left (T.pack path <> ": " <> problem <> " Write the fields line yourself: " <> docURL RulesFiles)+ Right statement -> case statementHeader statement of+ Nothing -> headerlessTemplate bank account path statement+ Just names ->+ let fields = templateFields names+ in Right . T.unlines $ [ "# Starter rules for " <> bank <> "/" <> account <> ", made by hledger-flow from the header row of", "# " <> T.pack path <> ". Check every line, then run: hledger-flow import", "# hledger's CSV rules reference: https://hledger.org/hledger.html#csv", "", "skip 1" ]- ++ ["separator " <> (if sep == '\t' then "TAB" else T.singleton sep) | sep /= ',']+ ++ separatorRule (statementSeparator statement) ++ [ "fields " <> T.intercalate ", " fields, "" ]@@ -120,38 +115,113 @@ "account1 assets:" <> bank <> ":" <> account, "account2 expenses:unknown" ]+separatorRule :: Char -> [T.Text]+separatorRule sep = ["separator " <> (if sep == '\t' then "TAB" else T.singleton sep) | sep /= ',']++headerlessTemplate :: T.Text -> T.Text -> FilePath -> DelimitedStatement -> Either T.Text T.Text+headerlessTemplate bank account path statement = case statementRows statement of+ [] -> Left ("No data rows in " <> T.pack path <> ".")+ sample : _ ->+ let columns = zip [1 :: Int ..] (List.transpose (statementRows statement))+ dateCandidates = [(n, cells) | (n, column) <- columns, Just cells <- [traverse parseDateCell column]]+ dateChoice = unique dateCandidates+ amountChoice = unique [n | (n, column) <- columns, all isSignedDecimal column]+ descriptionChoice = unique+ [ n | (n, column) <- columns, all (T.any isAlpha) column, Set.size (Set.fromList column) > 1+ ]+ field n+ | Just n == fmap fst dateChoice = "date"+ | Just n == amountChoice = "amount"+ | Just n == descriptionChoice = "description"+ | otherwise = columnName n+ dateRule = case dateChoice of+ Just (n, cells) -> case inferDateFormat cells of+ Just "%Y-%m-%d" -> []+ Just layout -> ["date-format " <> layout]+ Nothing -> ["# date-format (set the order and separator for " <> sampleValue n sample <> ")"]+ Nothing -> ["# Rename the transaction date column to date, then set date-format if needed."]+ in Right . T.unlines $+ [ "# Starter rules for " <> bank <> "/" <> account <> ", made by hledger-flow from",+ "# " <> T.pack path <> ", which has no header row.",+ "# The first row, by column:"+ ]+ ++ concat (zipWith commentValue [1 :: Int ..] sample)+ ++ ["# Check every line, then run: hledger-flow import", ""]+ ++ separatorRule (statementSeparator statement)+ ++ ["fields " <> T.intercalate ", " (map (field . fst) columns)]+ ++ dateRule+ ++ ["# Rename the amount column to amount." | amountChoice == Nothing]+ ++ ["# Rename the payee column to description." | descriptionChoice == Nothing]+ ++ ["", "account1 assets:" <> bank <> ":" <> account, "account2 expenses:unknown"] where- line = T.takeWhile (/= '\n') contents- (sep, names) = splitHeader line- fields = templateFields names+ unique [one] = Just one+ unique _ = Nothing+ columnName n = "column" <> T.pack (show n)+ sampleValue n = T.filter (/= '\n') . T.intercalate " " . take 1 . drop (n - 1)+ commentValue n value = case T.splitOn "\n" value of+ first : rest -> ("# " <> columnName n <> " =" <> if T.null first then "" else " " <> first)+ : map ("# " <>) rest+ [] -> [] --- | The starter rules file for an account (@owner/bank/account@), from its newest statement.-rulesTemplateFor :: BaseDir -> T.Text -> IO (Either T.Text T.Text)-rulesTemplateFor base requested = do- let account = T.dropWhileEnd (== '/') (maybe requested id (T.stripPrefix "import/" requested))+-- | The first default rules candidate and the starter for the newest statement,+-- skipping init's sample statement when the account also has real ones.+rulesStarterFor :: BaseDir -> T.Text -> IO (Either T.Text (AbsFile, T.Text))+rulesStarterFor base requested = do+ let account = T.dropWhileEnd (== '/') (fromMaybe requested (T.stripPrefix "import/" requested)) root = base </> [reldir|import|] files <- findImportFiles root let inputs = stageFiles root [reldir|1-in|] files- statements = filter ((== account) . accountName root) inputs+ found = filter ((== account) . accountName root) inputs+ statements = case filter (not . isSampleStatement) found of+ [] -> found+ real -> real known = List.nub (map (accountName root) inputs) case reverse statements of- [] ->- return . Left $- "No statements found for " <> account <> ". Accounts with statements: "- <> (if null known then "none" else T.intercalate ", " known) <> "."- newest : _ -> do- source <- fromMaybe newest <$> cachedStatement newest- let path = relativeToBase' (toFilePath base) (toFilePath source)- isCsv = map toLower (FilePath.takeExtension path) `elem` [".csv", ".tsv", ".ssv"]- (bank, acct) = case T.splitOn "/" account of- [_, b, a] -> (b, a)- _ -> (account, account)- header <- if isCsv then firstLine (toFilePath source) else return ""- return (rulesTemplate bank acct path header)+ [] -> return . Left $+ "No statements found for " <> account <> ". Accounts with statements: "+ <> (if null known then "none" else T.intercalate ", " known) <> "."+ newest : _ -> case extractImportDirs root newest of+ Left problem -> return (Left problem)+ Right dirs -> do+ candidates <- defaultRulesCandidates dirs+ case candidates of+ target : _ -> do+ source <- fromMaybe newest <$> cachedStatement newest+ let path = relativeToBase' (toFilePath base) (toFilePath source)+ (bank, acct) = case T.splitOn "/" account of+ [_, b, a] -> (b, a)+ _ -> (account, account)+ if isDelimitedFile path+ then fmap (fmap (\template -> (target, template)) . renderTemplate bank acct path) (readRuleSample source)+ else return (Left (notCsv path))+ [] -> return (Left ("No rules file target for " <> account <> ".")) --- | The first line of a file, tolerating encodings other than UTF-8.-firstLine :: FilePath -> IO T.Text-firstLine path = withFile path ReadMode $ \h -> do- hSetEncoding h =<< mkTextEncoding "UTF-8//ROUNDTRIP"- eof <- hIsEOF h- if eof then return "" else T.hGetLine h+rulesTemplateFor :: BaseDir -> T.Text -> IO (Either T.Text T.Text)+rulesTemplateFor base = fmap (fmap snd) . rulesStarterFor base++runRules :: Maybe HledgerConf -> BaseDir -> T.Text -> IO ()+runRules conf base account = do+ result <- rulesStarterFor base account+ (target, template) <- either (ioError . userError . T.unpack) return result+ cwd <- PathIO.getCurrentDir+ homes <- homeDirs+ let path = toFilePath target+ shown = T.pack (if cwd == base then relativeToBase' (toFilePath base) path else tildePath homes path)+ command sub args = flowCommand nativeShell homes cwd base conf sub args+ -- Match the native line endings of status --rules-template on stdout.+ fileTemplate = case nativeNewline of+ LF -> template+ CRLF -> T.replace "\n" "\r\n" template+ written <- tryIOError (writeNewUtf8File target fileTemplate)+ case written of+ Left err -> do+ occupied <- PathIO.doesFileExist target+ let problem+ | occupied || isAlreadyExistsError err =+ shown <> " already exists. Delete it first or preview a fresh starter with: "+ <> command "status" ["--rules-template", account]+ | otherwise = "Could not create " <> shown <> ": " <> T.pack (show err)+ ioError (userError (T.unpack problem))+ Right () -> do+ T.putStrLn ("Created " <> shown)+ T.putStrLn ("Edit it: check the fields line and the account names. Then run: " <> command "import" [])
+ src/Hledger/Flow/StatementScan.hs view
@@ -0,0 +1,285 @@+{-# LANGUAGE OverloadedStrings #-}++-- | Delimited statement records and the date evidence they contain.+module Hledger.Flow.StatementScan+ ( DelimitedStatement (..),+ parseDelimitedStatement,+ parseRuleSample,+ readRuleSample,+ readDelimitedStatement,+ splitHeader,+ looksLikeHeader,+ isDelimitedFile,+ DateCell,+ parseDateCell,+ dateCellYear,+ inferDateFormat,+ isSignedDecimal,+ StatementYear (..),+ inferStatementYear,+ )+where++import Data.Char (isDigit, isSpace, toLower)+import Data.List (intersect, nub, sort, transpose)+import Data.Maybe (catMaybes, isJust)+import qualified Data.Text as T+import qualified Data.Text.IO as T+import qualified Data.Text.Read as T+import Data.Time (fromGregorianValid)+import Hledger.Flow.PathHelpers (AbsFile)+import Path (toFilePath)+import qualified System.FilePath as F+import System.IO (IOMode (ReadMode), Handle, hIsEOF, hSetEncoding, mkTextEncoding, withFile)++data DelimitedStatement = DelimitedStatement+ { statementSeparator :: Char,+ statementHeader :: Maybe [T.Text],+ statementRows :: [[T.Text]]+ }+ deriving (Eq, Show)++isDelimitedFile :: FilePath -> Bool+isDelimitedFile path = map toLower (F.takeExtension path) `elem` [".csv", ".tsv", ".ssv"]++parseDelimitedStatement :: FilePath -> T.Text -> Either T.Text DelimitedStatement+parseDelimitedStatement = parseStatementSample (const Nothing)++parseRuleSample :: FilePath -> T.Text -> Either T.Text DelimitedStatement+parseRuleSample = parseStatementSample (Just . ruleRows)++parseStatementSample :: (DelimitedStatement -> Maybe Int) -> FilePath -> T.Text -> Either T.Text DelimitedStatement+parseStatementSample limit file contents =+ case logicalRecords (T.lines (T.dropWhile (== '\xFEFF') contents)) of+ [] -> Left noColumns+ first : rest -> do+ initial <- first >>= startStatement file+ rows <- traverse (>>= parseRow initial) (maybe rest (\n -> take (remaining n initial) rest) (limit initial))+ return initial {statementRows = statementRows initial ++ rows}++readDelimitedStatement :: Maybe Int -> AbsFile -> IO (Either T.Text DelimitedStatement)+readDelimitedStatement = readSample . const++readRuleSample :: AbsFile -> IO (Either T.Text DelimitedStatement)+readRuleSample = readSample (Just . ruleRows)++ruleRows :: DelimitedStatement -> Int+ruleRows initial = case statementHeader initial of+ Just _ -> 0+ Nothing -> 50++readSample :: (DelimitedStatement -> Maybe Int) -> AbsFile -> IO (Either T.Text DelimitedStatement)+readSample limit path = withFile (toFilePath path) ReadMode $ \handle -> do+ hSetEncoding handle =<< mkTextEncoding "UTF-8//ROUNDTRIP"+ first <- readRecord handle+ case first >>= traverse (startStatement (toFilePath path) . T.dropWhile (== '\xFEFF')) of+ Left err -> return (Left err)+ Right Nothing -> return (Left noColumns)+ Right (Just initial) -> do+ rows <- readRows handle initial (fmap (\n -> remaining n initial) (limit initial)) []+ return $ fmap (\xs -> initial {statementRows = statementRows initial ++ xs}) rows++remaining :: Int -> DelimitedStatement -> Int+remaining n initial = n - length (statementRows initial)++readRows :: Handle -> DelimitedStatement -> Maybe Int -> [[T.Text]] -> IO (Either T.Text [[T.Text]])+readRows _ _ (Just n) rows | n <= 0 = return (Right (reverse rows))+readRows handle initial limit rows = do+ next <- readRecord handle+ case next >>= traverse (parseRow initial) of+ Left err -> return (Left err)+ Right Nothing -> return (Right (reverse rows))+ Right (Just row) -> readRows handle initial (subtract 1 <$> limit) (row : rows)++readRecord :: Handle -> IO (Either T.Text (Maybe T.Text))+readRecord handle = do+ eof <- hIsEOF handle+ if eof then return (Right Nothing) else do+ line <- stripCR <$> T.hGetLine handle+ if T.null (T.strip line) then readRecord handle else finish (openQuote False line) [line]+ where+ finish False chunks = return (Right (Just (T.intercalate "\n" (reverse chunks))))+ finish True chunks = do+ eof <- hIsEOF handle+ if eof then return (Left "Unclosed quoted field.") else do+ line <- stripCR <$> T.hGetLine handle+ finish (openQuote True line) (line : chunks)++logicalRecords :: [T.Text] -> [Either T.Text T.Text]+logicalRecords = next . map stripCR+ where+ next [] = []+ next (line : rest)+ | T.null (T.strip line) = next rest+ | otherwise = collect (openQuote False line) [line] rest+ collect False chunks rest = Right (T.intercalate "\n" (reverse chunks)) : next rest+ collect True _ [] = [Left "Unclosed quoted field."]+ collect True chunks (line : rest) = collect (openQuote True line) (line : chunks) rest++stripCR :: T.Text -> T.Text+stripCR = T.dropWhileEnd (== '\r')++openQuote :: Bool -> T.Text -> Bool+openQuote = T.foldl' (\inside c -> if c == '"' then not inside else inside)++noColumns :: T.Text+noColumns = "Could not determine the statement's columns."++startStatement :: FilePath -> T.Text -> Either T.Text DelimitedStatement+startStatement file record = do+ let sep = chooseSeparator (map toLower (F.takeExtension file)) record+ fields <- splitFields sep record+ case fields of+ _ : _ : _ | looksLikeHeader fields -> return (DelimitedStatement sep (Just fields) [])+ | otherwise -> return (DelimitedStatement sep Nothing [fields])+ _ -> Left noColumns++parseRow :: DelimitedStatement -> T.Text -> Either T.Text [T.Text]+parseRow initial record = do+ row <- splitFields (statementSeparator initial) record+ let width = case (statementHeader initial, statementRows initial) of+ (Just header, _) -> length header+ (_, first : _) -> length first+ _ -> 0+ if length row == width then Right row else Left "Statement rows have different numbers of columns."++splitHeader :: T.Text -> (Char, [T.Text])+splitHeader line =+ let record = stripCR (T.dropWhile (== '\xFEFF') line)+ sep = chooseSeparator ".csv" record+ in (sep, either (const []) id (splitFields sep record))++chooseSeparator :: String -> T.Text -> Char+chooseSeparator ".tsv" _ = '\t'+chooseSeparator ".ssv" _ = ';'+chooseSeparator _ record = foldl' pick ',' [';', '\t']+ where+ pick best other | count other > count best = other+ | otherwise = best+ count sep = snd (T.foldl' (step sep) (False, 0 :: Int) record)+ step _ (quoted, n) '"' = (not quoted, n)+ step sep (quoted, n) c = (quoted, if not quoted && c == sep then n + 1 else n)++splitFields :: Char -> T.Text -> Either T.Text [T.Text]+splitFields sep = field [] . T.unpack+ where+ field fields rest+ | '"' : contents <- dropWhile (\c -> isSpace c && c /= sep) rest = quoted fields [] contents+ | otherwise =+ let (value, tail') = break (== sep) rest+ in if '"' `elem` value then Left "Unexpected quote in a delimited field."+ else end fields (T.strip (T.pack value)) tail'+ quoted _ _ [] = Left "Unclosed quoted field."+ quoted fields value ('"' : '"' : rest) = quoted fields ('"' : value) rest+ quoted fields value ('"' : rest) =+ end fields (T.strip (T.pack (reverse value))) (dropWhile (\c -> isSpace c && c /= sep) rest)+ quoted fields value (c : rest) = quoted fields (c : value) rest+ end fields value [] = Right (reverse (value : fields))+ end fields value (c : rest)+ | c == sep = field (value : fields) rest+ | otherwise = Left "Unexpected text after a quoted field."++looksLikeHeader :: [T.Text] -> Bool+looksLikeHeader = not . any dataish+ where+ dataish f = isJust (parseDateCell f) ||+ (not (T.null f) && T.all (`elem` ("0123456789-/.:,+$€£ " :: String)) f)++isSignedDecimal :: T.Text -> Bool+isSignedDecimal raw = case T.splitOn "." (dropSign raw) of+ [whole] -> digits whole+ [whole, fraction] -> digits whole && digits fraction+ _ -> False+ where+ dropSign t = case T.uncons t of+ Just (sign, rest) | sign `elem` ['+', '-'] -> rest+ _ -> t++digits :: T.Text -> Bool+digits t = not (T.null t) && T.all (\c -> c >= '0' && c <= '9') t++decimal :: T.Text -> Maybe Int+decimal t+ | digits t, Right (n, "") <- T.decimal t = Just n+ | otherwise = Nothing++data DateCell = DateCell {dateCellYear :: Integer, dateFormats :: [T.Text]}+ deriving (Eq, Show)++parseDateCell :: T.Text -> Maybe DateCell+parseDateCell value = do+ let (date, suffix) = T.break (\c -> isSpace c || c == 'T') (T.strip value)+ (sep, _) <- T.uncons (T.dropWhile isDigit date)+ if sep `elem` ['-', '/', '.'] then Just () else Nothing+ -- hledger accepts both padded and unpadded components with %-m and %-d.+ case T.splitOn (T.singleton sep) date of+ [a,b,c]+ | T.length a == 4 && short b && short c -> do+ year <- parseYear a+ m <- decimal b+ d <- decimal c+ -- Year-first inputs use conventional ISO order wherever it fits.+ let formats = if valid year m d then [fmt sep ["%Y","%-m","%-d"]]+ else [fmt sep ["%Y","%-d","%-m"] | valid year d m]+ cell year formats suffix+ | short a && short b && T.length c == 4 -> do+ year <- parseYear c+ lo <- decimal a+ hi <- decimal b+ cell year ([fmt sep ["%-m","%-d","%Y"] | valid year lo hi] +++ [fmt sep ["%-d","%-m","%Y"] | valid year hi lo]) suffix+ _ -> Nothing+ where+ short t = T.length t <= 2 && digits t+ valid y m d = isJust (fromGregorianValid y m d)+ parseYear t = do+ n <- decimal t+ if n >= 1900 && n <= 2099 then Just (toInteger n) else Nothing+ fmt sep = T.intercalate (T.singleton sep)+ cell _ [] _ = Nothing+ cell year formats suffix = case timeSuffix suffix of+ Nothing -> Nothing+ Just layout -> Just (DateCell year (maybe [] (\timeFormat -> map (<> timeFormat) formats) layout))++-- | An unfamiliar but time-like suffix supplies a year without a format guess.+timeSuffix :: T.Text -> Maybe (Maybe T.Text)+timeSuffix "" = Just (Just "")+timeSuffix suffix = do+ let (joiner, time) = case T.stripPrefix "T" suffix of+ Just t -> ("T", t)+ Nothing -> (" ", T.stripStart suffix)+ (hour, tail') <- T.uncons time+ if isDigit hour && ":" `T.isInfixOf` tail' then Just () else Nothing+ return $ case T.splitOn ":" time of+ [h,m] | within 23 h && within 59 m -> Just (joiner <> "%H:%M")+ [h,m,s] | within 23 h && within 59 m && within 59 s -> Just (joiner <> "%H:%M:%S")+ _ -> Nothing+ where+ within bound t = T.length t == 2 && maybe False (<= bound) (decimal t)++inferDateFormat :: [DateCell] -> Maybe T.Text+inferDateFormat cells = case map dateFormats cells of+ first : rest | [one] <- foldl' intersect first rest -> Just one+ _ -> Nothing++data StatementYear = UnknownYear | SingleYear Integer | SpanningYears [Integer]+ deriving (Eq, Show)++inferStatementYear :: DelimitedStatement -> StatementYear+inferStatementYear statement = case traverse agreed (transpose dateColumns) of+ Just years -> case sort (nub years) of+ [year] -> SingleYear year+ [] -> UnknownYear+ many -> SpanningYears many+ Nothing -> UnknownYear+ where+ dateColumns =+ [ dates+ | column <- transpose (statementRows statement),+ let dates = map parseDateCell column,+ any isJust dates,+ and (zipWith (\cell date -> T.null cell || isJust date) column dates)+ ]+ agreed cells = case nub (map dateCellYear (catMaybes cells)) of+ [year] -> Just year+ _ -> Nothing
src/Hledger/Flow/Status.hs view
@@ -12,28 +12,39 @@ statementExtensions, suggestPlacement, misplacedFiles,+ placementDestination,+ misplacedMoveCommand, nextStep,+ firstStatementStep,+ flowCommand,+ ledgerFileNote,+ balanceHint,+ withConfigCheck, hledgerVersionNumber, plural, ) where import Data.Char (isDigit, toLower)+import Data.List (nub) import Data.Maybe (fromMaybe, mapMaybe) import qualified Data.Set as Set import qualified Data.Text as T-import Hledger.Flow.Common (Shell (..), hledgerVersionNumber, homeShellPath, posixLiteral, powerShellLiteral, shellArgument, tildePath)+import Hledger.Flow.Common (Shell (..), createDirectoryCommand, hledgerFileArgument, hledgerVersionNumber, homeShellPath, moveFileCommand, posixLiteral, powerShellLiteral, shellArgument) import Hledger.Flow.DocHelpers (DocLink (..), docURL)+import Hledger.Flow.HledgerConfig (ConfigConflict, ConfigUse (..), firstConfigConflict, renderConfigConflict) import Hledger.Flow.Import.Inventory (stageFiles)-import Hledger.Flow.PathHelpers (AbsDir, AbsFile)-import Hledger.Flow.Types (BaseDir, HledgerInfo)-import Path (reldir, relfile, stripProperPrefix, toFilePath, (</>))+import Hledger.Flow.Init.Example (exampleAccountPath)+import Hledger.Flow.LedgerSelection (Coverage (..), LedgerFile (..), ledgerFileProblem)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile)+import Hledger.Flow.ReportReview+import Hledger.Flow.StatementScan (StatementYear (..))+import Hledger.Flow.Types (BaseDir, HledgerConf (..), HledgerInfo)+import Path (parseRelDir, parseRelFile, reldir, relfile, stripProperPrefix, toFilePath, (</>)) import qualified System.FilePath as FilePath data Situation = InLedger BaseDir | EmptyDir | Elsewhere deriving (Eq, Show) -data LedgerFile = LedgerFile {lfValue :: FilePath, lfExists :: Bool} deriving (Eq, Show)- data AccountStatus = AccountStatus { asAccount :: T.Text, asStatements :: Int,@@ -41,6 +52,8 @@ asRulesFile :: Maybe FilePath, asRulesCandidates :: [FilePath], asWithoutRules :: Int,+ asOtherRules :: [FilePath],+ asRulesTargetOccupied :: Bool, asPreprocess :: Bool, asConstruct :: Bool, asJournalsMissing :: Int,@@ -49,17 +62,31 @@ deriving (Eq, Show) -- | Both paths are relative to the base directory and start with @import@.-data Misplaced = Misplaced {mpPath :: FilePath, mpSuggestion :: FilePath} deriving (Eq, Show)+data Misplaced = Misplaced+ { mpPath :: FilePath,+ mpSuggestion :: FilePath,+ mpYear :: StatementYear,+ mpDestinationExists :: Bool+ }+ deriving (Eq, Show) data LedgerStatus = LedgerStatus { lsBaseDir :: BaseDir, lsHledger :: Maybe HledgerInfo, lsLedgerFile :: Maybe LedgerFile,+ lsAggregateJournal :: Maybe AbsFile,+ lsHledgerConf :: Maybe HledgerConf,+ lsConfigConflicts :: [ConfigConflict], lsAccounts :: [AccountStatus], lsMisplaced :: [Misplaced], lsManualJournals :: Int,+ lsHasManualWork :: Bool, lsAllYearsExists :: Bool,- lsNeedsReconciliation :: Bool+ lsNeedsReconciliation :: Bool,+ lsNeedsPrices :: Bool,+ lsReview :: Maybe Review,+ lsFirstStatementOwner :: RelDir,+ lsUntouchedExample :: Maybe RelDir } deriving (Show) @@ -101,69 +128,218 @@ misplaced f = do rel <- stripProperPrefix root f let parts = FilePath.splitDirectories (toFilePath rel)- return (Misplaced (FilePath.joinPath ("import" : parts)) (suggestPlacement parts))+ let evidence = case filter isYear (take (length parts - 1) parts) of+ first : _ -> SingleYear (read first)+ [] -> UnknownYear+ return (Misplaced (FilePath.joinPath ("import" : parts)) (suggestPlacement parts) evidence False) +placementDestination :: Misplaced -> Maybe RelFile+placementDestination item = case mpYear item of+ SingleYear _ | null (missingParts item) -> parseRelFile (mpSuggestion item)+ _ -> Nothing++missingParts :: Misplaced -> [T.Text]+missingParts item =+ [T.pack part | part <- ["owner", "bank", "account"],+ ("<" ++ part ++ ">") `elem` FilePath.splitDirectories (FilePath.takeDirectory (mpSuggestion item))]++misplacedMoveCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Misplaced -> Maybe T.Text+misplacedMoveCommand _ _ _ _ item | mpDestinationExists item = Nothing+misplacedMoveCommand shell homes cwd base item = do+ source <- parseRelFile (mpPath item)+ target <- placementDestination item+ return (moveFileCommand shell homes cwd base source target)++placementDetails :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Misplaced -> [T.Text]+placementDetails shell homes cwd base item =+ [T.pack (mpPath item) <> " → " <> T.pack (mpSuggestion item)]+ ++ ["Choose the " <> T.intercalate ", " parts <> "." | parts@(_ : _) <- [missingParts item]]+ ++ case mpYear item of+ SpanningYears years ->+ ["This statement contains transactions from " <> T.intercalate ", " (map (T.pack . show) years)+ <> ". Split it into one file per year."]+ UnknownYear -> ["Couldn't tell the year from the file. Use the year of its transactions."]+ SingleYear _ | mpDestinationExists item ->+ ["The destination already exists. Compare the two paths above before moving the statement."]+ SingleYear _ -> ["Move it with: " <> cmd | Just cmd <- [misplacedMoveCommand shell homes cwd base item]]+ plural :: Int -> T.Text -> T.Text plural 1 noun = "1 " <> noun plural n noun = T.pack (show n) <> " " <> noun <> "s" +firstStatementStep :: Integer -> Shell -> [FilePath] -> AbsDir -> BaseDir -> RelDir -> NextStep+firstStatementStep year shell homes cwd base owner =+ NextStep+ "Put your first statement in import/<owner>/<bank>/<account>/1-in/<year>/"+ Nothing+ ( [ "owner: whose money it is, for example " <> ownerName <> ", a partner or a business",+ "bank: the institution, for example mybank",+ "account: one account at that bank, for example checking"+ ]+ ++ ["Change the names, then run: " <> createDirectoryCommand shell homes cwd+ (base </> [reldir|import|] </> owner </> [reldir|mybank/checking/1-in|] </> folder)+ | Just folder <- [parseRelDir (show year)]]+ )+ (Just (docURL InputFiles))+ where+ ownerName = T.pack (FilePath.dropTrailingPathSeparator (toFilePath owner))+ -- | The single most important next step. The first matching rule wins. -- Paths under one of @homes@ show as @~@, and as @"$HOME/…"@ in commands. nextStep :: Integer -> Shell -> [FilePath] -> AbsDir -> LedgerStatus -> NextStep nextStep year shell homes cwd s | Nothing <- lsHledger s = NextStep "Install hledger, or pass its location with --hledger-path" Nothing [] (Just "https://hledger.org/install.html")- | statements == 0 && null (lsMisplaced s) && lsManualJournals s == 0 && not (lsNeedsReconciliation s) =- NextStep- "Put your first statement in import/<owner>/<bank>/<account>/1-in/<year>/"- Nothing- [ "For example: import/me/mybank/checking/1-in/" <> T.pack (show year) <> "/statement.csv",- "Then check with: " <> command "status"- ]- (Just (docURL InputFiles))+ | statements == 0 && null (lsMisplaced s) && not (lsHasManualWork s) && not (lsNeedsReconciliation s) =+ firstStatement | not (null (lsMisplaced s)) = NextStep ("Move " <> plural (length (lsMisplaced s)) "file" <> " to where hledger-flow looks") Nothing- [T.pack (mpPath m) <> " → " <> T.pack (mpSuggestion m) | m <- lsMisplaced s]- (Just (docURL InputFiles))+ (concatMap (placementDetails shell homes cwd (lsBaseDir s)) (lsMisplaced s))+ (Just (docURL (if any spansYears (lsMisplaced s) then StatementYearFolders else InputFiles)))+ | a : _ <- filter needsRules (lsAccounts s),+ source : other <- asOtherRules a =+ case other of+ [] | not (asRulesTargetOccupied a), Just commandLine <- renameRules a source ->+ NextStep ("Rename " <> T.pack source <> " to " <> T.pack (relativeCandidate a))+ (Just commandLine) ["Files it includes can keep their names."] (Just (docURL RulesFiles))+ _ ->+ NextStep ("Choose the rules to save as " <> T.pack (relativeCandidate a)) Nothing+ (map T.pack (source : other) +++ ["The target already exists. Check it before making changes." | asRulesTargetOccupied a] +++ ["Rename the rules for this account. Files it includes can keep their names."])+ (Just (docURL RulesFiles)) | a : _ <- filter needsRules (lsAccounts s) = NextStep- ("Create " <> T.pack (tildePath homes (firstCandidate a)) <> " (rules for " <> asAccount a <> ")")- (Just (command "status" <> " --rules-template " <> shellArgument shell (asAccount a)))- ["Save its output as that file, then edit it."]+ ("Create " <> T.pack (relativeCandidate a) <> " (rules for " <> asAccount a <> ")")+ (Just (commandArgs "rules" [asAccount a]))+ ["It creates that file from the statement\'s columns, for you to edit."] (Just (docURL RulesFiles)) | toImport > 0 =- NextStep ("Import " <> plural toImport "new or changed statement") (Just importCommand) [] Nothing- | not (lsAllYearsExists s) =- NextStep "Run import to create all-years.journal" (Just importCommand) [] Nothing+ withConfigCheck ImportConfig s $+ NextStep ("Import " <> plural toImport "new or changed statement") (Just importCommand) [] Nothing+ | Nothing <- lsAggregateJournal s, not (lsAllYearsExists s), not missingConfigured =+ withConfigCheck ImportConfig s $+ NextStep "Run import to create all-years.journal" (Just importCommand) [] Nothing | lsNeedsReconciliation s =- NextStep "Run import to reconcile generated files and journal includes" (Just importCommand) [] Nothing- | maybe True (not . lfExists) (lsLedgerFile s) =+ withConfigCheck ImportConfig s $+ NextStep "Run import to reconcile generated files and journal includes" (Just importCommand) [] Nothing+ | lsNeedsPrices s =+ withConfigCheck ImportConfig s $+ NextStep+ "Add market prices for your commodities"+ Nothing+ [ "Save each pair's P directives in prices/<year>/<commodity>-<quote>.prices, such as XAU-USD.prices.",+ "Include each pair file from prices/<year>/prices.journal, then run " <> importCommand <> "."+ ]+ (Just (docURL PriceFiles))+ | Just lf <- lsLedgerFile s, not (lfExists lf) = NextStep "Point LEDGER_FILE at this ledger so hledger finds it"- (Just exportLine)- [ "LEDGER_FILE is set to " <> T.pack (lfValue lf) <> ", which doesn't exist."- | Just lf <- [lsLedgerFile s]- ]+ (Just (setLedgerFileCommand shell homes s))+ [ledgerFileProblem homes lf] (Just (docURL SetLedgerFile))- | otherwise =- NextStep "Generate reports, or explore with hledger" (Just (command "report")) ["Or try: hledger balance"] Nothing+ | otherwise = case lsReview s of+ Nothing -> withConfigCheck ReportConfig s $+ NextStep "Generate reports, or explore with hledger" (Just (command "report"))+ ["Or try: " <> hint | Nothing <- [ledgerFileNote shell homes s], Just hint <- [balanceHint shell homes s]] Nothing+ Just review+ | Just account <- lsUntouchedExample s, null (reviewItems review) ->+ NextStep "Adapt the example account to your ledger" Nothing+ [ "Rename the owner, bank and account folders in " <> exampleAccountPath account <> " to match your ledger.",+ "Replace the sample CSV under 1-in/ with your statement in the folder for its year.",+ "Replace the sample rules with your own <bank>-<account>.rules in the account folder.",+ "Use your account names in any starter journals you fill in.",+ "Then run " <> importCommand <> " and " <> command "report" <> "."+ ]+ (Just (docURL InputFiles))+ | otherwise -> reviewStep importCommand firstStatement review where- command subcommand = "hledger-flow " <> subcommand <> baseArgument- baseArgument- | cwd == lsBaseDir s = ""- | otherwise = " " <> pasteable (shellArgument shell) (FilePath.dropTrailingPathSeparator (toFilePath (lsBaseDir s)))- pasteable literal path = fromMaybe (literal (T.pack path)) (homeShellPath shell homes path)+ commandArgs = flowCommand shell homes cwd (lsBaseDir s) (lsHledgerConf s)+ command subcommand = commandArgs subcommand []+ spansYears item = case mpYear item of SpanningYears _ -> True; _ -> False+ missingConfigured = maybe False (not . lfExists) (lsLedgerFile s)+ firstStatement = firstStatementStep year shell homes cwd (lsBaseDir s) (lsFirstStatementOwner s) importCommand = command "import" statements = sum (map asStatements (lsAccounts s)) toImport = sum [asJournalsMissing a + asJournalsOutdated a | a <- lsAccounts s] needsRules a = asWithoutRules a > 0 && not (asConstruct a)- firstCandidate a = case asRulesCandidates a of- c : _ | cwd /= lsBaseDir s -> toFilePath (lsBaseDir s) FilePath.</> c- | otherwise -> c- [] -> "a rules file"- allYears = toFilePath (lsBaseDir s </> [relfile|all-years.journal|])- exportLine = case shell of- Posix -> "export LEDGER_FILE=" <> pasteable posixLiteral allYears- PowerShell -> "$env:LEDGER_FILE = " <> pasteable powerShellLiteral allYears+ relativeCandidate a = fromMaybe "a rules file" (case asRulesCandidates a of+ target : _ -> Just target+ [] -> Nothing)+ renameRules a source = do+ old <- parseRelFile source+ new <- parseRelFile (relativeCandidate a)+ return (moveFileCommand shell homes cwd (lsBaseDir s) old new)++reviewStep :: T.Text -> NextStep -> Review -> NextStep+reviewStep importCommand firstStatement review+ | not (null unknowns) =+ NextStep (maybe "Review unknown transactions" (\n -> "Categorise " <> plural n "unknown transaction")+ (reviewCount (reviewUnknowns review))) Nothing+ (map (\path -> "Review " <> T.pack path <> ".") (take 1 unknowns) +++ ["Add if rules to the relevant account's rules file, then run: " <> importCommand] +++ ["Or start with transfers: " <> T.pack path | path : _ <- [transfers]])+ (Just (docURL ReviewUnknownTransactions))+ | not (null transfers) =+ NextStep ("Check transfers in " <> plural (length years) "year") Nothing+ (["Review " <> T.pack path <> "." | path : _ <- [transfers]] +++ ["A transfer may need review when the matching statement hasn't arrived yet."])+ (Just (docURL ReviewTransfers))+ | otherwise = firstStatement {nsSummary = "Everything is imported and reported"}+ where+ unknowns = reviewFiles (reviewUnknowns review)+ transfers = reviewFiles (reviewTransfers review)+ years = nub [itemYear item | item <- reviewItems review, itemKind item == TransferBalances]++flowCommand :: Shell -> [FilePath] -> AbsDir -> BaseDir -> Maybe HledgerConf -> T.Text -> [T.Text] -> T.Text+flowCommand shell homes cwd base conf subcommand arguments =+ T.unwords (["hledger-flow"] ++ config ++ [subcommand] ++ map (shellArgument shell) arguments ++ directory)+ where+ config = case conf of+ Just (GivenConf file) | subcommand `elem` ["import", "report", "rules"] ->+ ["--hledger-conf", shellPath shell homes file]+ _ -> []+ directory = [shellPath shell homes (FilePath.dropTrailingPathSeparator (toFilePath base)) | cwd /= base]++shellPath :: Shell -> [FilePath] -> FilePath -> T.Text+shellPath shell homes file =+ fromMaybe (shellArgument shell (T.pack file)) (homeShellPath shell homes file)++setLedgerFileCommand :: Shell -> [FilePath] -> LedgerStatus -> T.Text+setLedgerFileCommand shell homes s = case shell of+ Posix -> "export LEDGER_FILE=" <> setting posixLiteral+ PowerShell -> "$env:LEDGER_FILE = " <> setting powerShellLiteral+ where+ path = toFilePath (lsBaseDir s </> [relfile|all-years.journal|])+ setting literal = fromMaybe (literal (T.pack path)) (homeShellPath shell homes path)++balanceHint :: Shell -> [FilePath] -> LedgerStatus -> Maybe T.Text+balanceHint shell homes s = do+ case firstConfigConflict BalanceConfig (lsConfigConflicts s) of+ Just _ -> Nothing+ Nothing -> Just ()+ info <- lsHledger s+ journal <- lsAggregateJournal s+ let config = case lsHledgerConf s of+ Just (GivenConf configPath) -> ["--conf", shellPath shell homes configPath]+ _ -> []+ path = T.unpack (hledgerFileArgument info (toFilePath journal))+ return (T.unwords (["hledger"] ++ config ++ ["-f", shellPath shell homes path, "balance"]))++ledgerFileNote :: Shell -> [FilePath] -> LedgerStatus -> Maybe T.Text+ledgerFileNote shell homes s = do+ _ <- lsAggregateJournal s+ message <- case lsLedgerFile s of+ Nothing -> Just "LEDGER_FILE isn't set, so hledger won't find this ledger by default."+ Just file | lfExists file && lfCoverage file /= CoversLedger -> Just (ledgerFileProblem homes file)+ _ -> Nothing+ return (T.intercalate "\n" ([message, "hledger-flow report uses all-years.journal.", "For hledger, run: " <> setLedgerFileCommand shell homes s]+ ++ ["Or try: " <> hint | Just hint <- [balanceHint shell homes s]]))++withConfigCheck :: ConfigUse -> LedgerStatus -> NextStep -> NextStep+withConfigCheck use status next = case firstConfigConflict use (lsConfigConflicts status) of+ Just problem -> NextStep "Fix hledger.conf before continuing" Nothing+ (T.lines (renderConfigConflict problem)) (Just (docURL StatusCommand))+ Nothing -> next
src/Hledger/Flow/Status/Gather.hs view
@@ -5,14 +5,17 @@ module Hledger.Flow.Status.Gather ( classifyDir, gatherStatus,+ gatherStatusWithConfig, accountName, cachedStatement, hookExists,+ mainRulesCandidates, ) where import Control.Applicative ((<|>)) import Control.Monad (forM, when)+import Data.Char (toLower) import qualified Data.List as List import qualified Data.Map.Strict as Map import qualified Data.Set as Set@@ -21,17 +24,23 @@ import qualified Data.Text.IO as T import Data.Time.Clock (UTCTime) import Hledger.Flow.BaseDir (findBaseDir, relativeToBase')-import Hledger.Flow.Common (configuredLedgerFile, firstExistingFile, groupValuesBy)+import Hledger.Flow.Common (firstExistingFile, groupValuesBy, hledgerFileArgument, runProcessCaptured)+import Hledger.Flow.HledgerConfig (findConfigConflicts) import Hledger.Flow.Import.ImportHelpers (defaultRulesCandidates, extractImportDirs, rulesFileCandidates) import Hledger.Flow.Import.IncludeFiles (includesNeedUpdate) import Hledger.Flow.Import.Inventory (classifyGeneratedOutputs, findImportFiles, generatedOutput, isManualJournal, managedIncludeFiles, stageFiles) import Hledger.Flow.Import.Types (ImportDirs (..))-import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelFile)+import Hledger.Flow.Init.Example (directivesAreStarter, journalIsCommentsOnly, sampleAccountDir, starterOwner, untouchedExample)+import Hledger.Flow.LedgerSelection+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelFile, pathOccupied)+import Hledger.Flow.ReportReview (loadReportReview, reviewItems)+import Hledger.Flow.StatementScan (StatementYear (..), inferStatementYear, isDelimitedFile, readDelimitedStatement) import Hledger.Flow.Status-import Hledger.Flow.Types (BaseDir, HledgerInfo)+import Hledger.Flow.Types (BaseDir, HledgerConf, HledgerInfo (..)) import Path import qualified Path.IO as PathIO import qualified System.Directory as Directory+import System.Exit (ExitCode (..)) import qualified System.FilePath as FilePath import qualified System.Info as Sys import System.IO.Error (tryIOError)@@ -48,32 +57,107 @@ return (if all ("." `List.isPrefixOf`) names then EmptyDir else Elsewhere) gatherStatus :: Maybe HledgerInfo -> BaseDir -> IO LedgerStatus-gatherStatus hledger base = do+gatherStatus hledger = gatherStatusWithConfig hledger Nothing++gatherStatusWithConfig :: Maybe HledgerInfo -> Maybe HledgerConf -> BaseDir -> IO LedgerStatus+gatherStatusWithConfig hledger conf base = do let root = base </> [reldir|import|] files <- findImportFiles root let statements = stageFiles root [reldir|1-in|] files+ manualJournals = filter (isManualJournal root) files groups = Map.toAscList (groupValuesBy (accountName root) statements) accounts <- forM groups (uncurry (accountStatus base))- configured <- configuredLedgerFile- ledgerFile <- forM configured $ \(value, path) -> LedgerFile value <$> Directory.doesFileExist path- allYears <- PathIO.doesFileExist (base </> [relfile|all-years.journal|])+ conflicts <- maybe (pure []) (\hl -> findConfigConflicts hl conf) hledger+ selection <- selectLedger [] hledger base+ let defaultJournal = base </> [relfile|all-years.journal|]+ allYears <- PathIO.doesFileExist defaultJournal+ let selected = aggregateJournal selection (orphans, journals) <- classifyGeneratedOutputs root files statements -- A newly initialised, empty ledger has no indexes to reconcile yet.- reconciliation <- if allYears && null orphans- then includesNeedUpdate base root files journals+ reconciliation <- if (allYears || isJust selected) && null orphans+ then includesNeedUpdate base root files journals allYears else return (not (null orphans) || not (null (managedIncludeFiles root files)))+ needsPrices <- case (hledger, selected) of+ (Just info, Just journal) | not reconciliation -> journalNeedsPrices info journal+ _ -> return False+ review <- maybe (pure Nothing) (\journal -> loadReportReview base journal conf) selected+ owner <- starterOwner+ examplePresent <- either (const True) id <$> tryIOError (pathOccupied (sampleAccountDir base owner))+ let personal = [reldir|personal|]+ alternate = if FilePath.equalFilePath (toFilePath owner) (toFilePath personal) then [reldir|me|] else personal+ firstOwner = if examplePresent then alternate else owner+ example <- case review of+ Just current | null (reviewItems current) -> untouchedExample base statements+ _ -> return Nothing+ manualWork <- case manualJournals of+ [] -> pure False+ -- A wrapper can contribute journal entries of its own.+ _ | not (null statements) || maybe False (/= defaultJournal) selected -> pure True+ _ -> either (const True) not <$> tryIOError (onlyStarterJournals base manualJournals)+ misplaced <- traverse (inspectMisplaced base) (List.sortOn mpPath (misplacedFiles root files)) return LedgerStatus { lsBaseDir = base, lsHledger = hledger,- lsLedgerFile = ledgerFile,+ lsLedgerFile = selectedLedgerFile selection,+ lsAggregateJournal = selected,+ lsHledgerConf = conf,+ lsConfigConflicts = conflicts, lsAccounts = accounts,- lsMisplaced = List.sortOn mpPath (misplacedFiles root files),- lsManualJournals = length (filter (isManualJournal root) files),+ lsMisplaced = misplaced,+ lsManualJournals = length manualJournals,+ lsHasManualWork = manualWork, lsAllYearsExists = allYears,- lsNeedsReconciliation = reconciliation+ lsNeedsReconciliation = reconciliation,+ lsNeedsPrices = needsPrices,+ lsReview = review,+ lsFirstStatementOwner = firstOwner,+ lsUntouchedExample = example } +-- The starter ledger contains declarations and comments only.+onlyStarterJournals :: BaseDir -> [AbsFile] -> IO Bool+onlyStarterJournals base [] = directivesAreStarter base+onlyStarterJournals base (file : rest) = do+ commentOnly <- journalIsCommentsOnly file+ if commentOnly then onlyStarterJournals base rest else return False++inspectMisplaced :: BaseDir -> Misplaced -> IO Misplaced+inspectMisplaced base item = do+ evidence <- case mpYear item of+ UnknownYear | isDelimitedFile (mpPath item), Just path <- parseRelFile (mpPath item) -> do+ scanned <- tryIOError (readDelimitedStatement Nothing (base </> path))+ return $ case scanned of+ Right (Right statement) -> inferStatementYear statement+ _ -> UnknownYear+ known -> return known+ let suggestion = case evidence of+ SingleYear year -> FilePath.joinPath+ [if part == "<year>" then show year else part | part <- FilePath.splitDirectories (mpSuggestion item)]+ _ -> mpSuggestion item+ enriched = item {mpYear = evidence, mpSuggestion = suggestion}+ occupied <- case placementDestination enriched of+ Nothing -> return False+ Just path -> either (const True) id <$> tryIOError (pathOccupied (base </> path))+ return enriched {mpDestinationExists = occupied}++-- Ask hledger to resolve includes and parse the journal. If it cannot read+-- the journal, status can still report what it found in the import tree.+journalNeedsPrices :: HledgerInfo -> AbsFile -> IO Bool+journalNeedsPrices info journal = do+ commodities <- query ["commodities", "--used"]+ case T.lines <$> commodities of+ Just (_ : _ : _) -> (== Just "") . fmap T.strip <$> query ["prices"]+ _ -> return False+ where+ query args = do+ let executable = T.pack (toFilePath (hlPath info))+ fileArgs = ["--no-conf", "--file", hledgerFileArgument info (toFilePath journal)]+ result <- tryIOError (runProcessCaptured (executable, args ++ fileArgs, Nothing))+ return $ case result of+ Right (ExitSuccess, output, _) -> Just output+ _ -> Nothing+ accountName :: AbsDir -> AbsFile -> T.Text accountName root file = case stripProperPrefix root file of Just rel -> T.intercalate "/" (map T.pack (take 3 (FilePath.splitDirectories (toFilePath rel))))@@ -92,15 +176,25 @@ directives <- firstExistingFile [base </> [relfile|directives.journal|]] sources <- forM statements $ \s -> do cached <- cachedStatement s- rules <- rulesFileCandidates (fromMaybe s cached) dirs >>= firstExistingFile- return (s, cached, rules)- let resolved = [rules | (_, _, rules) <- sources]+ candidates <- rulesFileCandidates (fromMaybe s cached) dirs+ rules <- firstExistingFile candidates+ return (s, cached, rules, candidates)+ let resolved = [rules | (_, _, rules, _) <- sources]+ withoutRules = length (filter isNothing resolved)+ (otherRules, targetOccupied) <- if withoutRules > 0 && isNothing construct+ then do+ other <- tryIOError (mainRulesCandidates (accountDir dirs) (concat [xs | (_, _, _, xs) <- sources]))+ occupied <- case defaults of+ target : _ -> either (const True) id <$> tryIOError (pathOccupied target)+ _ -> return False+ return (either (const []) id other, occupied)+ else return ([], False) -- Statements in an account commonly share one rules file. ruleTimes <- case construct of Just _ -> return Map.empty Nothing -> traverse (tryIOError . latestRuleChange) $ Map.fromSet id (Set.fromList (catMaybes resolved))- journals <- forM sources $ \(s, cached, rules) -> do+ journals <- forM sources $ \(s, cached, rules, _) -> do let inputs = s : catMaybes [cached, preprocess, construct <|> directives] journal <- generatedOutput [reldir|3-journal|] ".journal" s journalState inputs (rules >>= (`Map.lookup` ruleTimes)) journal@@ -111,7 +205,9 @@ asYears = List.nub (List.sort (map yearOf statements)), asRulesFile = relative <$> rulesFile, asRulesCandidates = map relative defaults,- asWithoutRules = length (filter isNothing resolved),+ asWithoutRules = withoutRules,+ asOtherRules = map relative otherRules,+ asRulesTargetOccupied = targetOccupied, asPreprocess = isJust preprocess, asConstruct = isJust construct, asJournalsMissing = length (filter (== Missing) journals),@@ -130,12 +226,27 @@ key <- Directory.canonicalizePath file when (Set.member key ancestors) $ ioError (userError ("cyclic rules include: " ++ file)) modified <- Directory.getModificationTime file- contents <- T.readFile file- let children = [FilePath.takeDirectory file FilePath.</> T.unpack (T.strip path)- | line <- T.lines (T.dropWhile (== '\xFEFF') contents)- , Just path <- [T.stripPrefix "include " line]]+ children <- ruleIncludes file times <- mapM (visit (Set.insert key ancestors)) children return (maximum (modified : times))++ruleIncludes :: FilePath -> IO [FilePath]+ruleIncludes file = do+ contents <- T.readFile file+ return [FilePath.takeDirectory file FilePath.</> T.unpack (T.strip path)+ | line <- T.lines (T.dropWhile (== '\xFEFF') contents),+ Just path <- [T.stripPrefix "include " line]]++mainRulesCandidates :: AbsDir -> [AbsFile] -> IO [AbsFile]+mainRulesCandidates dir recognized = do+ (_, files) <- PathIO.listDir dir+ let local = List.sort [file | file <- files, map toLower (FilePath.takeExtension (toFilePath file)) == ".rules"]+ canonical = Directory.canonicalizePath . toFilePath+ recognizedTargets <- Set.fromList <$> traverse canonical recognized+ children <- concat <$> traverse (ruleIncludes . toFilePath) local+ childTargets <- Set.fromList <$> traverse Directory.canonicalizePath children+ pairs <- traverse (\file -> (,) file <$> canonical file) local+ return [file | (file, target) <- pairs, Set.notMember target (Set.union recognizedTargets childTargets)] journalState :: [AbsFile] -> Maybe (Either IOError UTCTime) -> AbsFile -> IO JournalState journalState inputs rulesTime journal = do
src/Hledger/Flow/Status/Render.hs view
@@ -22,11 +22,15 @@ import Hledger.Flow.DateTime (currentYear) import Hledger.Flow.Environment (coresSetting, environmentJSON, environmentSettings, renderSettings) import Hledger.Flow.Internals (SystemInfo, systemInfo)-import Hledger.Flow.Status.Gather (gatherStatus)+import Hledger.Flow.Status.Gather (gatherStatusWithConfig)+import Hledger.Flow.PathHelpers (AbsDir) import Hledger.Flow.Types (BaseDir) import Path.IO (getCurrentDir) import qualified Data.Text as T import Hledger.Flow.Status+import Hledger.Flow.ReportReview+import Hledger.Flow.StatementScan (StatementYear (..))+import Hledger.Flow.LedgerSelection (Coverage (..)) import Hledger.Flow.Types (HledgerConf, HledgerInfo (..)) import Path (toFilePath) import Hledger.Flow.Logging (bold, green, red, useColor, yellow)@@ -39,7 +43,7 @@ renderStatus :: Bool -> [FilePath] -> LedgerStatus -> NextStep -> T.Text renderStatus color homes s next =- T.unlines (header : accountBlock ++ misplacedBlock) <> renderNext color next+ T.unlines (header : accountBlock ++ misplacedBlock ++ noteBlock ++ reviewBlock) <> renderNext color next where header = bold color "Ledger: " <> T.pack (tildePath homes (baseText s)) <> " (" <> hledgerText <> ")" hledgerText = case lsHledger s of@@ -69,6 +73,21 @@ ++ [tshow outdated <> " outdated" | outdated > 0] txt = T.intercalate ", " parts in if missing + outdated > 0 then yellow color ("• " <> txt) else green color ("✓ " <> txt)+ noteBlock = case T.lines <$> ledgerFileNote nativeShell homes s of+ Just (first : rest) -> "" : (" Note: " <> first) : map (" " <>) rest+ _ -> []+ reviewBlock = case maybe [] reviewItems (lsReview s) of+ [] -> []+ items -> "" : " To review:" : map showReview (take 5 items)+ ++ [" and " <> tshow (length items - 5) <> " more" | length items > 5]+ showReview item =+ let label = case itemKind item of+ UnknownTransactions -> "unknown transaction"+ TransferBalances -> "transfer account to check"+ count = case itemKind item of+ TransferBalances -> fmap (\n -> plural n "transfer account" <> " to check") (itemCount item)+ UnknownTransactions -> fmap (\n -> plural n label) (itemCount item)+ in " " <> maybe "count unavailable" id count <> " " <> T.pack (itemPath item) misplacedBlock = case length (lsMisplaced s) of 0 -> [] n -> ["", " " <> yellow color ("⚠ " <> plural n "file" <> " not where hledger-flow looks")]@@ -83,16 +102,20 @@ tshow :: Show a => a -> T.Text tshow = T.pack . show -statusJSON :: SystemInfo -> Maybe HledgerConf -> LedgerStatus -> NextStep -> Value-statusJSON sys conf s next =+statusJSON :: SystemInfo -> Maybe HledgerConf -> AbsDir -> LedgerStatus -> NextStep -> Value+statusJSON sys conf cwd s next = object [ "schemaVersion" .= (1 :: Int), "environment" .= environmentJSON sys conf, "baseDir" .= baseText s, "hledger" .= fmap hledgerJSON (lsHledger s),- "ledgerFile" .= fmap (\lf -> object ["value" .= lfValue lf, "exists" .= lfExists lf]) (lsLedgerFile s),+ "ledgerFile" .= fmap (\lf -> object ["value" .= lfValue lf, "exists" .= lfExists lf, "coversLedger" .= (lfCoverage lf == CoversLedger)]) (lsLedgerFile s),+ "aggregateJournal" .= fmap toFilePath (lsAggregateJournal s),+ "ledgerFileNote" .= ledgerFileNote nativeShell [] s, "accounts" .= map accountJSON (lsAccounts s),- "misplaced" .= [object ["path" .= mpPath m, "suggestion" .= mpSuggestion m] | m <- lsMisplaced s],+ "misplaced" .= map misplacedJSON (lsMisplaced s),+ "reportsCurrent" .= maybe False (const True) (lsReview s),+ "review" .= fmap reviewJSON (lsReview s), "manualJournals" .= lsManualJournals s, "allYearsJournalExists" .= lsAllYearsExists s, "next"@@ -104,6 +127,17 @@ ] ] where+ reviewJSON review = object ["unknownTransactions" .= category (reviewUnknowns review),+ "transferAccounts" .= category (reviewTransfers review)]+ category review = object ["count" .= reviewCount review, "files" .= reviewFiles review]+ misplacedJSON item = object+ [ "path" .= mpPath item,+ "suggestion" .= mpSuggestion item,+ "year" .= (case mpYear item of SingleYear year -> Just year; _ -> Nothing),+ "years" .= (case mpYear item of SpanningYears years -> years; _ -> []),+ "destinationExists" .= mpDestinationExists item,+ "command" .= misplacedMoveCommand nativeShell [] cwd (lsBaseDir s) item+ ] hledgerJSON hl = object ["path" .= toFilePath (hlPath hl), "version" .= hledgerVersionNumber (hlVersion hl)] accountJSON a = object@@ -113,12 +147,14 @@ "rulesFile" .= asRulesFile a, "rulesCandidates" .= asRulesCandidates a, "statementsWithoutRules" .= asWithoutRules a,+ "otherRules" .= asOtherRules a,+ "rulesTargetOccupied" .= asRulesTargetOccupied a, "hooks" .= object ["preprocess" .= asPreprocess a, "construct" .= asConstruct a], "journalsMissing" .= asJournalsMissing a, "journalsOutdated" .= asJournalsOutdated a ] --- | The next step for people to read with @homes@, or for scripts with @[]@ (absolute paths).+-- | The next step for people to read with @homes@, or for scripts with @[]@ (no home abbreviation). nextStepHere :: [FilePath] -> LedgerStatus -> IO NextStep nextStepHere homes status = do year <- currentYear@@ -137,23 +173,23 @@ printStatus hledgerPath confOpt json base = do hledger <- findHledger hledgerPath conf <- findHledgerConf confOpt- status <- gatherStatus hledger base+ status <- gatherStatusWithConfig hledger conf base sys <- systemInfo+ cwd <- getCurrentDir+ year <- currentYear+ let next homes = nextStep year nativeShell homes cwd status if json- then do- next <- nextStepHere [] status- TL.putStrLn (encodeToLazyText (statusJSON sys conf status next))+ then TL.putStrLn (encodeToLazyText (statusJSON sys conf cwd status (next []))) else do color <- useColor stdout homes <- homeDirs- next <- nextStepHere homes status- T.putStr (renderStatus color homes status next <> renderEnvironment color homes sys conf status)+ T.putStr (renderStatus color homes status (next homes) <> renderEnvironment color homes sys conf status) -- | Print only the next step for a ledger.-printNext :: Maybe FilePath -> BaseDir -> IO ()-printNext hledgerPath base = do+printNext :: Maybe FilePath -> Maybe HledgerConf -> BaseDir -> IO ()+printNext hledgerPath conf base = do hledger <- findHledger hledgerPath- status <- gatherStatus hledger base+ status <- gatherStatusWithConfig hledger conf base homes <- homeDirs next <- nextStepHere homes status color <- useColor stdout
test/CSVImport/IncludeFilesTests.hs view
@@ -17,6 +17,7 @@ testIncludeYears = TestCase ( do+ assertEqual "An empty generated index has no report years" (Right []) (includeYears' (includePreamble <> "\n")) let txterr = "Some text without years" let expectederr = ["Unable to extract years from the following text:", txterr, "Errors:"] let actualerr = case lefts [includeYears' txterr] of
test/Common/Integration.hs view
@@ -13,6 +13,7 @@ import Data.Time.Clock (UTCTime(..), addUTCTime, secondsToDiffTime) import Data.Time.Calendar (fromGregorian) import Hledger.Flow.Common+import Hledger.Flow.PathHelpers (writeNewUtf8File) import qualified Hledger.Flow.RuntimeOptions as RT import Hledger.Flow.Types (HledgerConf (..), LogMessage (StdErr)) import qualified Path@@ -96,7 +97,8 @@ assertEqual "no existing files" Nothing absent testVerboseTestFile :: Test-testVerboseTestFile = TestCase $ withTestDir $ \base -> do+-- Pin colour off: stderr is a terminal when the suite runs interactively.+testVerboseTestFile = TestCase $ withEnv "NO_COLOR" "1" $ withTestDir $ \base -> do let opts = (defaultOpts base) {RT.verbose = True} file = base Path.</> [Path.relfile|statement.csv|] ch <- newTChanIO@@ -247,5 +249,15 @@ entries <- Directory.listDirectory dir assertBool ("no staged files left: " ++ show entries) (not (any (List.isPrefixOf ".") entries)) +testWriteNewFileAtLongPath :: Test+testWriteNewFileAtLongPath = TestLabel "create a file at a long path without replacing it" $ TestCase $ withTestDir $ \base -> do+ name <- Path.parseRelFile (replicate 100 'a' </> replicate 100 'b' </> replicate 100 'c' </> "starter.rules")+ let target = base Path.</> name+ writeNewUtf8File target "first\n"+ second <- tryIOError (writeNewUtf8File target "second\n")+ case second of+ Left _ -> T.readFile (Path.toFilePath target) >>= assertEqual "preserve the first write" "first\n"+ Right () -> assertFailure "an existing file was replaced"+ tests :: Test-tests = TestList [testWriteGeneratedText, testHledgerLookup, testFindHledgerConf, testHomeDirs, testFilterPaths, testFirstExistingFile, testVerboseTestFile, testNeedsRegeneration, testBatchOrder, testBatchFailure]+tests = TestList [testWriteGeneratedText, testWriteNewFileAtLongPath, testHledgerLookup, testFindHledgerConf, testHomeDirs, testFilterPaths, testFirstExistingFile, testVerboseTestFile, testNeedsRegeneration, testBatchOrder, testBatchFailure]
+ test/HledgerConfig/Unit.hs view
@@ -0,0 +1,30 @@+{-# LANGUAGE OverloadedStrings #-}++module HledgerConfig.Unit (tests) where++import Data.Maybe (isJust, isNothing)+import qualified Data.Text as T+import Hledger.Flow.HledgerConfig+import Test.HUnit++fromText :: [T.Text] -> [ConfigConflict]+fromText = configConflicts "hledger.conf" . parseConfigLines True++tests :: Test+tests = "hledger config sources" ~: TestList+ [ TestList [TestCase $ do+ let problem = firstConfigConflict ImportConfig (fromText ["# comment", "[print]", input])+ fmap (T.takeWhile (/= '\n') . renderConfigConflict) problem @?=+ Just ("hledger.conf line 3 [print] adds a journal: " <> input)+ | input <- ["-f other", "-fother", "--file other", "--file=other"]],+ TestList [isNothing (firstConfigConflict ReportConfig (fromText ["[general]", option <> " \"-f=value\""])) ~?= True+ | option <- ["--alias", "--pivot", "-b", "--begin", "-e", "--end", "-p", "--period",+ "--depth", "-o", "--output-file", "-O", "--output-format", "--rules",+ "--rules-file", "--today", "--value", "--conf"]],+ isJust (firstConfigConflict ReportConfig (fromText ["[balancesheet]", "--flat -f other"])) ~?= True,+ isNothing (firstConfigConflict ImportConfig (fromText ["[balance]", "--file other"])) ~?= True,+ isNothing (firstConfigConflict ReportConfig (fromText ["[files]", "--file other"])) ~?= True,+ isNothing (firstConfigConflict ReportConfig (fromText ["# -f file", "--pretty # -f file"])) ~?= True,+ isNothing (firstConfigConflict BalanceConfig (fromText ["-- --file other"])) ~?= True,+ isJust (firstConfigConflict ReportConfig (fromText [" [balance] @arguments"])) ~?= True+ ]
test/ImportHelpers/Integration.hs view
@@ -110,14 +110,16 @@ testTrashLock :: Test testTrashLock = TestCase $ withSystemTempDir "hlflow-trash" $ \root -> do let source = root </> [relfile|import/owner/bank/account/3-journal/2020/march.journal|]- lock = root </> [reldir|import/owner/bank/account/.trash/.move-lock|]- target = root </> [relfile|import/owner/bank/account/.trash/2020/3-journal/march.journal|]+ lock = root </> [reldir|.hledger-flow/trash/.move-lock|]+ target = root </> [relfile|.hledger-flow/trash/import/owner/bank/account/3-journal/2020/march.journal|]+ ignore = root </> [relfile|.hledger-flow/trash/.gitignore|] touchFile source createDirIfMissing True lock- result <- tryIOError (moveToTrash source)+ result <- tryIOError (moveToTrash root source) assertBool "A concurrent trash move must not race for a destination" (isLeft result) T.readFile (toFilePath source) >>= assertEqual "Busy trash leaves source intact" "x" doesFileExist target >>= assertBool "Busy trash writes no destination" . not removeDir lock- actual <- moveToTrash source+ actual <- moveToTrash root source assertEqual "Retry after the other move completes succeeds" target actual+ T.readFile (toFilePath ignore) >>= assertEqual "Git ignores the trash" "*\n"
test/Incremental/Dependencies.hs view
@@ -97,19 +97,28 @@ no release s = fallback (checkConfig release file s) forM_ [V1524,V1996,OtherSupported] $ \release -> do ok release "# comment\n[balance] --period=thismonth\n[register]\n@file\n[general]\n[print]\n"- forM_ ["--period=thismonth", "[general] --debug", "[print]\n@indirect", "[txns]", "[bal]", "[b]", "[alias]", "[missing", "[general", "[balance]\n[print] --period=thismonth"] (no release)- ok V1996 "[print] --layout=hledger1 # layout\n"- ok OtherSupported "[print] --layout=hledger1 # layout\n"- no OtherSupported "[print]\n [balance]\n--empty"- no OtherSupported "[holdings]"- no OtherSupported "[demo]"- no V1524 "[print]\n--layout=hledger1"- no V1996 "[general]\n--layout=hledger1"- ok V1996 "[holdings]"- no V1524 "[print]\n [balance]\n--empty"- no V1524 "[holdings]"- ok V1524 "[demo]"- no V1996 "[demo]"+ ok release "--invert\n[print] -x --round soft --layout=55 # literal values\n"+ forM_ ["not:archive", "--layout=hledger1 not:archive", "--unknown", "--round",+ "--alias /old/=new", "--begin=2025-01-01", "--period=2025", "date:2025..2026",+ "--forecast=2025", "--forecast=2025..2026", "--value=then", "--value=2025-06-30,EUR",+ "--today=2026-06-30\ndate:thismonth", "--today 2026-06-30 -V", "-B", "desc:\"in 30 days\"",+ "desc:today", "acct:next", "--period=\"monthly in 2025\"", "desc:\"a # b\"", "--round=soft # date:today"] $+ ok release . ("[print]\n" <>)+ forM_ ["[general] --debug", "[txns]", "[bal]", "[b]", "[alias]", "[missing", "[general",+ "[future-command] --period=thismonth", "[balance] date:today"] (ok release)+ forM_ ["--period=thismonth", "--begin\ntoday", "-p 'this month'", "-pthismonth", "--begin today", "-etomorrow",+ "date:today", "not:date:thisyear", "date2:'last month'", "--period=1/1..4/1",+ "--forecast", "--forecast=thismonth", "--forecast=2025..", "--value=now", "--exchange=EUR,now", "-XEUR,now",+ "-V", "-X USD", "--value=end", "--value=end,EUR", "date:'-7 days..'", "-b '2 weeks ago'",+ "expr:'tag:recurring and date:thismonth'", "expr:'not:(date:thisyear)'", "--today=2026-01-01\n--forecast"] $ \setting ->+ case checkConfig release file ("[print]\n" <> setting) of+ Fallback reason -> reasonScope reason @?= "non-deterministic hledger config entry"+ _ -> assertFailure ("expected clock dependency: " ++ show setting)+ forM_ ["--file=extra.journal", "-fextra.journal", "--rules-file extra.rules", "@indirect",+ "--new", "--output-file saved.txt", "-osaved.txt", "--locations"] $+ no release . ("[print]\n" <>)+ ok V1996 "[print]\n [balance]\n--period=thismonth"+ no V1524 "[print]\n [balance]\n--period=thismonth" forM_ ["[balance]\n--period=thismonth", "[balance]\n--period=thisyear"] $ \bytes -> do B.writeFile (toFilePath file) bytes discoverConfig V1996 (Just (GivenConf (toFilePath file))) >>= (@?= Reusable (Just (file,digestBytes bytes)))@@ -123,9 +132,35 @@ ) , "directives eligibility" ~: withTestDir (\base -> do let file = base </> [relfile|directives.journal|]- check = checkDirectives V1996 file . T.encodeUtf8- forM_ ["account assets:bank:current ; type: C","commodity 1,000.00000000 BTC ; lots:","commodity €1,000.00","P 2000-01-01 EUR €1","P 2000/01/01 BTC 12.30 EUR","; tags\n# prices\n* prices\n\n"] $ \s -> check s @?= Reusable ()+ check = journalIncludes file . T.encodeUtf8+ forM_ ["account assets:bank:current ; type: C","commodity 1,000.00000000 BTC ; lots:","commodity €1,000.00","P 2000-01-01 EUR €1","P 2000/01/01 BTC 12.30 EUR","; tags\n# prices\n* prices\n\n"] $ \s -> check s @?= Reusable []+ check "include accounts.journal ; declarations\ninclude\ttax/vat codes.journal" @?= Reusable ["accounts.journal","tax/vat codes.journal"]+ check "include vat-codes\ninclude vat.CUSTOM\ninclude journal:vat.CSV" @?= Reusable ["vat-codes","vat.CUSTOM","vat.CSV"]+ forM_ ["", "journal:", "vat.CsV", "vat.tsv", "vat.ssv", "vat.rules", "vat.timedot", "vat.timeclock"] $+ fallback . check . ("include " <>) forM_ ["account assets:any\n note", "comment\ncommodity €1\nend comment","include *.journal","P today EUR €1","P 2000-01-01 EUR €1 trailing","commodity 1,234 ABC extra", "2025/01/01 A"] (fallback . check))+ , "unmirrorable journal includes" ~: withTestDir (\tmp -> do+ let base = tmp </> [reldir|ledger|]+ root = base </> [relfile|directives.journal|]+ nested = base </> [relfile|sub/accounts.journal|]+ write path = B.writeFile (toFilePath path)+ P.createDirIfMissing True (parent nested)+ write root "include sub/accounts.journal\n"+ write nested "include ../directives.journal\n"+ discoverDirectives root >>= fallback+ write nested "include ../../outside.journal\n"+ write (tmp </> [relfile|outside.journal|]) "account expenses:cafe\n"+ discoverDirectives root >>= fallback+#ifndef mingw32_HOST_OS+ let link = base </> [relfile|alias.journal|]+ D.createFileLink (toFilePath nested) (toFilePath link)+ write root "include alias.journal\n"+ write nested "include prices.journal\n"+ write (base </> [relfile|prices.journal|]) "; different directory\n"+ write (parent nested </> [relfile|prices.journal|]) "; canonical directory\n"+ discoverDirectives root >>= fallback+#endif+ ) , "converter identity" ~: bracket getLocaleEncoding setLocaleEncoding (\_ -> withTestDir (\base -> do setLocaleEncoding utf8 let real = base </> [relfile|actual|]
test/Incremental/Snapshot.hs view
@@ -13,7 +13,6 @@ import GHC.IO.Encoding (getLocaleEncoding, setLocaleEncoding, utf8) import qualified Data.ByteString as BS import qualified Data.ByteString.Char8 as B8-import qualified Data.Map.Strict as Map import qualified Data.Text as T import Hledger.Flow.Import.Cache import Hledger.Flow.Import.Dependencies@@ -50,18 +49,24 @@ rules = base </> [relfile|account/bank.rules|] category = base </> [relfile|shared/coffee.rules|] directives = base </> [relfile|directives.journal|]+ accounts = base </> [relfile|shared/accounts.journal|] a = "account2 expenses:cafe\n" P.createDirIfMissing True (parent statement) P.createDir (parent category) BS.writeFile (toFilePath statement) "2026-01-01,Coffee,3\n" BS.writeFile (toFilePath rules) "include ../shared/coffee.rules\n" BS.writeFile (toFilePath category) a- BS.writeFile (toFilePath directives) "account expenses:cafe\n"+ BS.writeFile (toFilePath directives) "include shared/accounts.journal\n"+ BS.writeFile (toFilePath accounts) "account expenses:cafe\n" result <- withSnapshotRun base $ \run -> withSnapshot run V1524 (RulesInput statement rules (Just directives) Nothing) $ \snapshot -> do RulesInput dataCopy rulesCopy (Just directivesCopy) Nothing <- pure (snapshotInputs snapshot)- let deps = snapshotManifest snapshot- assertEqual "manifest includes nested rule" (Just (digestBytes a)) (Map.lookup category deps)- assertEqual "all four files" 4 (Map.size deps)+ let expected = [statement, rules, category, directives, accounts]+ digests <- mapM digestFile expected+ assertEqual "manifest follows the order of conversion inputs"+ (zip expected digests) (snapshotManifest snapshot)+ BS.writeFile (toFilePath accounts) "account expenses:food\n"+ BS.readFile (toFilePath (parent directivesCopy </> [relfile|shared/accounts.journal|]))+ >>= assertEqual "private copy retains original declarations" "account expenses:cafe\n" assertEqual "CSV basename" (filename statement) (filename dataCopy) canonicalCopy <- D.canonicalizePath (toFilePath rulesCopy) let location = ":1:2:\n"
test/Init/Integration.hs view
@@ -2,21 +2,28 @@ module Init.Integration (tests) where +import Hledger.Flow.DateTime (currentYear) import Hledger.Flow.Init (InitOutcome (..), initLedger)+import Hledger.Flow.Init.Example (exampleAccount, exampleBaseFiles, exampleImportFiles, starterOwner) import Path-import Path.IO (doesDirExist)+import Path.IO (doesDirExist, doesFileExist) import Test.HUnit hiding (Path) import TestHelpers (withTestDir) testInit :: Test testInit = TestCase $ withTestDir $ \tmp -> do let fresh = tmp </> [reldir|fin|]- (@?= Created fresh Nothing) =<< initLedger fresh+ year <- currentYear+ owner <- starterOwner+ let created = [(name, True) | (name, _) <- exampleBaseFiles year]+ (@?= Created fresh Nothing (exampleAccount owner) created) =<< initLedger fresh+ mapM_ (\(file, _) -> doesFileExist (fresh </> [reldir|import|] </> file) >>= assertBool (toFilePath file))+ (exampleImportFiles owner year) doesDirExist (fresh </> [reldir|import|]) >>= assertBool "import/ created" (@?= AlreadyLedger fresh) =<< initLedger fresh (@?= InsideLedger fresh) =<< initLedger (fresh </> [reldir|import/me|]) let nested = fresh </> [reldir|elsewhere|]- (@?= Created nested (Just fresh)) =<< initLedger nested+ (@?= Created nested (Just fresh) (exampleAccount owner) created) =<< initLedger nested tests :: Test tests = TestList [testInit]
+ test/LedgerSelection/Unit.hs view
@@ -0,0 +1,14 @@+{-# LANGUAGE OverloadedStrings #-}++module LedgerSelection.Unit (tests) where++import Hledger.Flow.LedgerSelection (Coverage (..), coverageFor)+import Test.HUnit++tests :: Test+tests = "ledger coverage" ~: TestList+ [ coverageFor "index" Nothing ["wrapper", "nested", "index"] ~?= CoversLedger,+ coverageFor "index" (Just "directives") ["wrapper", "index", "directives"] ~?= CoversLedger,+ coverageFor "index" (Just "directives") ["wrapper", "index"] ~?= UnrelatedLedger,+ coverageFor "index" Nothing ["wrapper", "other"] ~?= UnrelatedLedger+ ]
+ test/ReportReview/Unit.hs view
@@ -0,0 +1,33 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}++module ReportReview.Unit (tests) where++import qualified Data.Text as T+import Hledger.Flow.InputSnapshot (inputsUnchanged, snapshotInputs)+import Hledger.Flow.ReportReview+import Path+import qualified Path.IO as PathIO+import Test.HUnit hiding (Path)+import TestHelpers (withTestDir)++tests :: Test+tests = "report review" ~: TestList+ [ countUnknownTransactions "" ~?= Just 0,+ countUnknownTransactions "2016-01-02\n expenses:unknown 2\n" ~?= Just 1,+ countUnknownTransactions (T.unlines ["2016-01-01 Shop", " expenses:unknown 5", " assets:bank"]) ~?= Just 1,+ countUnknownTransactions (T.unlines ["2016-01-01 Shop", " expenses:unknown 5", "2017-01-01 Again", " expenses:unknown 7"]) ~?= Just 2,+ countUnknownTransactions "\"txnidx\",\"date\",\"account\"\n\"1\",\"2016-01-01\",\"assets:bank\"" ~?= Nothing,+ reviewHasData UnknownTransactions "\"txnidx\",\"date\",\"account\"\n" ~?= False,+ countTransferAccounts (T.unlines ["Balance changes in 2016:", " || 2016Q1 2016Q2", "==++==", "assets:transfer || 5 -5"]) ~?= Just 1,+ countTransferAccounts (T.unlines ["Balance changes in 2016:", "│ ║ 2016Q1 │", "╞═╬═══╡", "│ assets:transfer ║ 5 │", "└─╨───┘"]) ~?= Just 1,+ countTransferAccounts (T.unlines ["Balance changes in 2016:", " || 2016Q1", "==++=="]) ~?= Just 0,+ reviewHasData TransferBalances "\"account\",\"2016Q1\"\n\"assets:transfer\",\"5\"\n" ~?= True,+ reviewHasData TransferBalances "\"account\",\"2016Q1\"\n" ~?= False,+ "a timestamp equal to the marker is stale" ~: (withTestDir $ \base -> do+ let file = base </> [relfile|input.journal|]+ writeFile (toFilePath file) "; journal\n"+ recorded <- snapshotInputs [file]+ marker <- PathIO.getModificationTime file+ inputsUnchanged recorded marker >>= (@?= False))+ ]
test/RulesTemplate/Unit.hs view
@@ -1,15 +1,20 @@ {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-} module RulesTemplate.Unit (tests) where import qualified Data.Text as T import Hledger.Flow.RulesTemplate+import Path ((</>), parent, relfile, toFilePath)+import Path.IO (createDirIfMissing) import Test.HUnit+import TestHelpers (withTestDir) tests :: Test tests = TestList- [ "bom stripped" ~: templateFields (snd (splitHeader "\xFEFF\&Date,Desc,Amount")) ~?= ["date", "description", "amount"],+ [ "default target" ~: defaultTarget,+ "bom stripped" ~: templateFields (snd (splitHeader "\xFEFF\&Date,Desc,Amount")) ~?= ["date", "description", "amount"], "semicolons" ~: fst (splitHeader "\"Datum\";\"Omschrijving\";\"Bedrag\"") ~?= ';', "tabs" ~: fst (splitHeader "Date\tDesc\tAmount") ~?= '\t', "mapping" ~: templateFields ["Transaction Date", "Payee Name", "Amount", "Balance (ZAR)", "", "Amount"]@@ -41,8 +46,36 @@ "no separator line for commas" ~: fmap (T.isInfixOf "separator") (rulesTemplate "b" "a" "x.csv" "Date,Desc,Amount\n") ~?= Right False,- "no header" ~: either (T.isInfixOf "looks like data") (const False)- (rulesTemplate "b" "a" "x.csv" "2026-01-02,Coffee,-3.50\n") ~?= True,+ "no header" ~: fmap (T.isInfixOf "fields date, column2, amount")+ (rulesTemplate "b" "a" "x.csv" "2026-01-02,Coffee,-3.50\n") ~?= Right True,+ "Wells Fargo starter" ~:+ fmap (\t -> all (`T.isInfixOf` t)+ ["# column1 = 11/14/2026", "# column2 = -4.50", "# column3 = *", "# column4 =", "# column5 = COFFEE SHOP",+ "fields date, amount, column3, column4, description", "date-format %-m/%-d/%Y",+ "account1 assets:wf:bchecking", "account2 expenses:unknown"] && not ("skip " `T.isInfixOf` t))+ (rulesTemplate "wf" "bchecking" "x.csv" "11/14/2026,-4.50,*,,COFFEE SHOP\n11/15/2026,-6.75,*,,BOOK SHOP\n")+ ~?= Right True,+ "amount and balance stay neutral" ~: fmap (T.isInfixOf "fields date, column2, column3, description")+ (rulesTemplate "b" "a" "x.csv" "2026-01-02,-5,100,Coffee\n2026-01-03,10,110,Shop\n") ~?= Right True,+ "ambiguous day and month are commented" ~: fmap (\t -> "# date-format" `T.isInfixOf` t && not ("\ndate-format" `T.isInfixOf` t))+ (rulesTemplate "b" "a" "x.csv" "01/02/2026,-5,Coffee\n01/03/2026,-10,Shop\n") ~?= Right True,+ "mixed date separators are commented" ~: fmap (T.isInfixOf "# date-format")+ (rulesTemplate "b" "a" "x.csv" "12/31/2026,-5,Coffee\n30.12.2026,-10,Shop\n") ~?= Right True,+ "multiline sample stays commented" ~: fmap (T.isInfixOf "# account1 other")+ (rulesTemplate "b" "a" "x.csv" "2026-01-01,\"Coffee, shop\naccount1 other\",-5\n2026-01-02,Store,-2\n") ~?= Right True,+ "read only fifty data records" ~: fmap (T.isInfixOf "fields date, amount, column3")+ (rulesTemplate "b" "a" "x.csv" (T.replicate 50 "2026-01-01,-5,Store\n" <> "2026-01-02,bad,Diff\n")) ~?= Right True,+ "ragged sample refuses" ~: either (const True) (const False)+ (rulesTemplate "b" "a" "x.csv" "2026-01-01,-5,Store\n2026-01-02,-2\n") ~?= True, "not csv" ~: either (T.isInfixOf "isn't a CSV file") (const False) (rulesTemplate "b" "a" "x.pdf" "%PDF") ~?= True ]++defaultTarget :: Assertion+defaultTarget = withTestDir $ \base -> do+ let sample = base </> [relfile|import/sm/wf/bchecking/1-in/2026/latest_rfo-card.csv|]+ target = base </> [relfile|import/sm/wf/bchecking/wf-bchecking.rules|]+ createDirIfMissing True (parent sample)+ writeFile (toFilePath sample) "Date,Desc,Amount\n2026-01-02,Store,-2\n"+ result <- rulesStarterFor base "sm/wf/bchecking"+ fmap fst result @?= Right target
test/Spec.hs view
@@ -9,14 +9,17 @@ import qualified Common.Integration import qualified Common.Unit import qualified Environment.Unit+import qualified HledgerConfig.Unit import qualified ImportHelpers.Integration import qualified Incremental.Cache import qualified Incremental.Dependencies import qualified Incremental.Platform import qualified Incremental.Snapshot import qualified Init.Integration+import qualified LedgerSelection.Unit import qualified PathHelpers.Unit import qualified ReleaseNotice.Unit+import qualified ReportReview.Unit import qualified RulesTemplate.Unit import qualified RuntimeOptions.Unit import qualified Status.Integration@@ -43,9 +46,12 @@ Incremental.Platform.tests, Incremental.Snapshot.tests, Init.Integration.tests,+ LedgerSelection.Unit.tests, ReleaseNotice.Unit.tests,+ ReportReview.Unit.tests, RulesTemplate.Unit.tests, RuntimeOptions.Unit.tests,+ HledgerConfig.Unit.tests, Environment.Unit.tests ]
test/Status/Integration.hs view
@@ -8,7 +8,7 @@ import Hledger.Flow.Common (findHledger) import Hledger.Flow.PathHelpers (AbsFile) import Hledger.Flow.Status-import Hledger.Flow.Status.Gather (classifyDir, gatherStatus)+import Hledger.Flow.Status.Gather (classifyDir, gatherStatus, mainRulesCandidates) import Path import Path.IO import Control.Monad (unless)@@ -88,5 +88,23 @@ r <- findHledger (Just "/nonexistent/hledger") assertBool "Nothing for a missing path" (isNothing r) +testRulesRoots :: Test+testRulesRoots = TestCase $ withTestDir $ \dir -> do+ let mainRules = dir </> [relfile|wf-checking.rules|]+ included = dir </> [relfile|wf-common.rules|]+ override = dir </> [relfile|rfo-card.rules|]+ another = dir </> [relfile|z.rules|]+ roots = mainRulesCandidates dir [override]+ roots >>= (@?= [])+ writeFile (toFilePath mainRules) "include wf-common.rules\n"+ writeFile (toFilePath included) "# helper\n"+ writeFile (toFilePath override) "# statement-specific\n"+ roots >>= (@?= [mainRules])+ writeFile (toFilePath another) "# a second entry point\n"+ roots >>= (@?= [mainRules, another])+ removeFile another+ writeFile (toFilePath included) "include wf-checking.rules\n"+ roots >>= (@?= [])+ tests :: Test-tests = TestList [testClassify, testGather, testFindHledgerMissing, testLedgerFileTilde]+tests = TestList [testRulesRoots, testClassify, testGather, testFindHledgerMissing, testLedgerFileTilde]
test/Status/Unit.hs view
@@ -8,7 +8,9 @@ import qualified Data.Aeson.KeyMap as KeyMap import Data.List (sort) import qualified Data.Text as T-import Hledger.Flow.Common (Shell (..))+import Hledger.Flow.Common (Shell (..), createDirectoryCommand, shellArgument)+import Hledger.Flow.LedgerSelection (Coverage (..))+import Hledger.Flow.StatementScan (StatementYear (..), inferStatementYear, parseDelimitedStatement) import Hledger.Flow.Status.Render (renderStatus, statusJSON) import Hledger.Flow.Status import Path@@ -21,12 +23,20 @@ LedgerStatus { lsBaseDir = testRoot </> [reldir|fin|], lsHledger = Just defaultHlInfo,- lsLedgerFile = Just (LedgerFile "/x" True),+ lsLedgerFile = Just (LedgerFile "/x" True CoversLedger),+ lsAggregateJournal = Just (testRoot </> [relfile|fin/all-years.journal|]),+ lsHledgerConf = Nothing,+ lsConfigConflicts = [], lsAccounts = [], lsMisplaced = [], lsManualJournals = 0,+ lsHasManualWork = False, lsAllYearsExists = True,- lsNeedsReconciliation = False+ lsNeedsReconciliation = False,+ lsNeedsPrices = False,+ lsReview = Nothing,+ lsFirstStatementOwner = [reldir|me|],+ lsUntouchedExample = Nothing } acct :: AccountStatus@@ -38,6 +48,8 @@ asRulesFile = Just "import/me/mybank/checking/mybank-checking.rules", asRulesCandidates = ["import/me/mybank/checking/mybank-checking.rules", "import/mybank.rules"], asWithoutRules = 0,+ asOtherRules = [],+ asRulesTargetOccupied = False, asPreprocess = False, asConstruct = False, asJournalsMissing = 0,@@ -79,31 +91,30 @@ commandOf :: LedgerStatus -> Maybe T.Text commandOf = nsCommand . atRoot -allYears :: T.Text-allYears = T.pack (toFilePath (lsBaseDir blank </> [relfile|all-years.journal|]))- testRules :: Test testRules = TestList [ "1 hledger missing" ~: summaryOf blank {lsHledger = Nothing} ~?= "Install hledger, or pass its location with --hledger-path", "2 empty ledger" ~: summaryOf blank ~?= "Put your first statement in import/<owner>/<bank>/<account>/1-in/<year>/",- "2 example uses current year" ~: nsDetail (atRoot blank) ~?= ["For example: import/me/mybank/checking/1-in/2026/statement.csv", "Then check with: hledger-flow status"],- "2 skipped for manual-only ledger" ~: summaryOf blank {lsManualJournals = 1} ~?= "Generate reports, or explore with hledger",- "5 import when all-years.journal is missing" ~: commandOf blank {lsManualJournals = 1, lsAllYearsExists = False} ~?= Just "hledger-flow import",- "3 misplaced" ~: summaryOf blank {lsMisplaced = [Misplaced "import/a.csv" "x"]} ~?= "Move 1 file to where hledger-flow looks",- "3 misplaced plural" ~: summaryOf blank {lsMisplaced = [Misplaced "import/a.csv" "x", Misplaced "import/b.csv" "y"]} ~?= "Move 2 files to where hledger-flow looks",- "3 detail" ~: nsDetail (atRoot blank {lsMisplaced = [Misplaced "import/a.csv" "x"]}) ~?= ["import/a.csv → x"],- "4 rules" ~: commandOf blank {lsAccounts = [acct {asWithoutRules = 1, asRulesFile = Nothing}]} ~?= Just "hledger-flow status --rules-template me/mybank/checking",+ "2 example uses current year" ~:+ filter (T.isPrefixOf "Change the names, then run: ") (nsDetail (atRoot blank))+ ~?= ["Change the names, then run: " <> createDirectoryCommand Posix [] (lsBaseDir blank)+ (lsBaseDir blank </> [reldir|import/me/mybank/checking/1-in/2026|])],+ "2 skipped for manual-only ledger" ~: summaryOf blank {lsManualJournals = 1, lsHasManualWork = True} ~?= "Generate reports, or explore with hledger",+ "5 import when all-years.journal is missing" ~: commandOf blank {lsManualJournals = 1, lsHasManualWork = True, lsAllYearsExists = False, lsAggregateJournal = Nothing} ~?= Just "hledger-flow import",+ "3 misplaced" ~: summaryOf blank {lsMisplaced = [Misplaced "import/a.csv" "x" (SingleYear 2026) False]} ~?= "Move 1 file to where hledger-flow looks",+ "3 misplaced plural" ~: summaryOf blank {lsMisplaced = [Misplaced "import/a.csv" "x" (SingleYear 2026) False, Misplaced "import/b.csv" "y" (SingleYear 2026) False]} ~?= "Move 2 files to where hledger-flow looks",+ "3 detail" ~: take 1 (nsDetail (atRoot blank {lsMisplaced = [Misplaced "import/a.csv" "x" (SingleYear 2026) False]})) ~?= ["import/a.csv → x"],+ "4 rules" ~: commandOf blank {lsAccounts = [acct {asWithoutRules = 1, asRulesFile = Nothing}]} ~?= Just "hledger-flow rules me/mybank/checking", "4 summary" ~: summaryOf blank {lsAccounts = [acct {asWithoutRules = 1, asRulesFile = Nothing}]} ~?= "Create import/me/mybank/checking/mybank-checking.rules (rules for me/mybank/checking)", "4 powershell literal account" ~: nsCommand (nextStep 2026 PowerShell [] (lsBaseDir blank) blank {lsAccounts = [acct {asWithoutRules = 1, asAccount = "me/Bob's Bank/Lee’s $(example)"}]})- ~?= Just "hledger-flow status --rules-template 'me/Bob''s Bank/Lee’’s $(example)'",+ ~?= Just "hledger-flow rules 'me/Bob''s Bank/Lee’’s $(example)'", "4 skipped with construct" ~: commandOf blank {lsAccounts = [acct {asWithoutRules = 1, asConstruct = True}]} ~?= Just "hledger-flow report", "5 import" ~: summaryOf blank {lsAccounts = [acct {asJournalsMissing = 1, asJournalsOutdated = 1}]} ~?= "Import 2 new or changed statements", "5 import singular" ~: summaryOf blank {lsAccounts = [acct {asJournalsMissing = 1}]} ~?= "Import 1 new or changed statement",- "6 posix" ~: commandOf blank {lsAccounts = [acct], lsLedgerFile = Nothing} ~?= Just ("export LEDGER_FILE='" <> allYears <> "'"),- "6 windows" ~: nsCommand (nextStep 2026 PowerShell [] (lsBaseDir blank) blank {lsAccounts = [acct], lsLedgerFile = Nothing}) ~?= Just ("$env:LEDGER_FILE = '" <> allYears <> "'"),- "6 set but missing" ~: nsDetail (atRoot blank {lsAccounts = [acct], lsLedgerFile = Just (LedgerFile "/nope.journal" False)}) ~?= ["LEDGER_FILE is set to /nope.journal, which doesn't exist."],+ "6 unset" ~: commandOf blank {lsAccounts = [acct], lsLedgerFile = Nothing} ~?= Just "hledger-flow report",+ "6 set but missing" ~: nsDetail (atRoot blank {lsAccounts = [acct], lsLedgerFile = Just (LedgerFile "/nope.journal" False UnrelatedLedger)}) ~?= ["LEDGER_FILE is set to /nope.journal, which doesn't exist."], "7 done" ~: commandOf blank {lsAccounts = [acct]} ~?= Just "hledger-flow report" ] @@ -116,13 +127,18 @@ unset = blank {lsAccounts = [acct], lsLedgerFile = Nothing} quoted parts = "\"" <> T.pack (FilePath.joinPath ("$HOME" : parts)) <> "\"" dollarInPath = unset {lsBaseDir = testRoot </> [reldir|fin$|]}+ setting shell s = do+ note <- ledgerFileNote shell homes s+ case filter ("For hledger, run: " `T.isPrefixOf`) (T.lines note) of+ [line] -> T.stripPrefix "For hledger, run: " line+ _ -> Nothing in TestList- [ "posix export" ~: nsCommand (next Posix (lsBaseDir blank) unset) ~?= Just ("export LEDGER_FILE=" <> quoted ["fin", "all-years.journal"]),- "powershell export" ~: nsCommand (next PowerShell (lsBaseDir blank) unset) ~?= Just ("$env:LEDGER_FILE = " <> quoted ["fin", "all-years.journal"]),+ [ "posix export" ~: setting Posix unset ~?= Just ("export LEDGER_FILE=" <> quoted ["fin", "all-years.journal"]),+ "powershell export" ~: setting PowerShell unset ~?= Just ("$env:LEDGER_FILE = " <> quoted ["fin", "all-years.journal"]), "ledger argument" ~: nsCommand (next Posix elsewhere blank {lsAccounts = [acct]}) ~?= Just ("hledger-flow report " <> quoted ["fin"]), "summary" ~: nsSummary (next Posix elsewhere blank {lsAccounts = [acct {asWithoutRules = 1, asRulesFile = Nothing}]})- ~?= T.pack ("Create " <> FilePath.joinPath ["~", "fin", "import", "me", "mybank", "checking", "mybank-checking.rules"]) <> " (rules for me/mybank/checking)",- "special character falls back to the absolute path" ~: nsCommand (next Posix (lsBaseDir dollarInPath) dollarInPath)+ ~?= "Create import/me/mybank/checking/mybank-checking.rules (rules for me/mybank/checking)",+ "special character falls back to the absolute path" ~: setting Posix dollarInPath ~?= Just ("export LEDGER_FILE='" <> T.pack (toFilePath (lsBaseDir dollarInPath </> [relfile|all-years.journal|])) <> "'") ] @@ -150,10 +166,36 @@ assertBool "clean" ("✓ 1 journal" `T.isInfixOf` render acct) testJSONShape :: Test-testJSONShape = TestCase $ case statusJSON testSystemInfo Nothing blank (atRoot blank) of+testJSONShape = TestCase $ case statusJSON testSystemInfo Nothing (lsBaseDir blank) blank (atRoot blank) of Object o -> sort (map Key.toText (KeyMap.keys o))- @?= ["accounts", "allYearsJournalExists", "baseDir", "environment", "hledger", "ledgerFile", "manualJournals", "misplaced", "next", "schemaVersion"]+ @?= ["accounts", "aggregateJournal", "allYearsJournalExists", "baseDir", "environment", "hledger", "ledgerFile", "ledgerFileNote", "manualJournals", "misplaced", "next", "reportsCurrent", "review", "schemaVersion"] _ -> assertFailure "status JSON is an object" -tests = TestList [testSuggest, testMisplacedUpperCase, testRules, testHome, testVersion, testRenderNoColour, testRenderPending, testJSONShape]+tests = TestList [testFirstDirectory, testStatementYear, testSuggest, testMisplacedUpperCase, testRules, testHome, testVersion, testRenderNoColour, testRenderPending, testJSONShape]++testStatementYear :: Test+testStatementYear = "statement year" ~: TestList+ [ check "Date,Desc,Amount\n2025-12-31,\"Shop, City\",-5\n" (SingleYear 2025),+ check "2025-12-31,Shop,-5\n2026-01-01,Store,-7\n" (SpanningYears [2025,2026]),+ check "2025-12-31,2026-01-02,Shop,-5\n" UnknownYear,+ check "2025-12-31,Shop,-5\n,Store,-7\n" UnknownYear,+ check "20251231,Shop,-5\n31/12/25,Store,-7\n" UnknownYear,+ check "31.12.2025 12:30,Shop,-5\n" (SingleYear 2025),+ check ",,Shop,-5\n,,Store,-7\n" UnknownYear+ ]+ where+ check rows expected = fmap inferStatementYear (parseDelimitedStatement "2026-01-02.csv" rows) ~?= Right expected++testFirstDirectory :: Test+testFirstDirectory =+ let base = lsBaseDir blank+ dir = base </> [reldir|import/me/mybank/checking/1-in/2026|]+ elsewhere = testRoot </> [reldir|another place|]+ rootArgument = T.pack (native "import/me/mybank/checking/1-in/2026")+ in TestList+ [ createDirectoryCommand Posix [] base dir ~?= "mkdir -p " <> shellArgument Posix rootArgument,+ createDirectoryCommand PowerShell [] base dir ~?= "New-Item -ItemType Directory -Force -Path " <> shellArgument PowerShell rootArgument,+ T.isInfixOf (T.pack (FilePath.dropTrailingPathSeparator (toFilePath dir)))+ (createDirectoryCommand Posix [] elsewhere dir) ~?= True+ ]