hledger-lib 1.15.2 → 1.16

raw patch · 48 files changed

+4750/−2811 lines, 48 filesdep +tastydep +tasty-hunitdep −easytestdep −semigroupsdep ~basedep ~base-compat-batteriesPVP ok

version bump matches the API change (PVP)

Dependencies added: tasty, tasty-hunit

Dependencies removed: easytest, semigroups

Dependency ranges changed: base, base-compat-batteries

API changes (from Hackage documentation)

Files

• CHANGES.md +23/−0
• Hledger/Data/AccountName.hs +17/−21
• Hledger/Data/Amount.hs +72/−71
• Hledger/Data/Dates.hs +8/−31
• Hledger/Data/Journal.hs +108/−48
• Hledger/Data/Ledger.hs +6/−9
• Hledger/Data/PeriodicTransaction.hs +63/−27
• Hledger/Data/Posting.hs +27/−31
• Hledger/Data/RawOptions.hs +42/−8
• Hledger/Data/StringFormat.hs +13/−15
• Hledger/Data/Timeclock.hs +24/−25
• Hledger/Data/Transaction.hs +150/−215
• Hledger/Data/TransactionModifier.hs +2/−2
• Hledger/Data/Valuation.hs +9/−11
• Hledger/Query.hs +100/−108
• Hledger/Read.hs +2/−0
• Hledger/Read/Common.hs +131/−130
• Hledger/Read/CsvReader.hs +269/−171
• Hledger/Read/JournalReader.hs +94/−90
• Hledger/Reports/AccountTransactionsReport.hs +24/−6
• Hledger/Reports/BalanceReport.hs +68/−31
• Hledger/Reports/BudgetReport.hs +4/−4
• Hledger/Reports/EntriesReport.hs +2/−2
• Hledger/Reports/MultiBalanceReport.hs +48/−29
• Hledger/Reports/PostingsReport.hs +16/−22
• Hledger/Reports/ReportOptions.hs +45/−36
• Hledger/Utils.hs +2/−2
• Hledger/Utils/Parse.hs +5/−4
• Hledger/Utils/String.hs +5/−0
• Hledger/Utils/Test.hs +132/−169
• Hledger/Utils/Text.hs +8/−9
• Hledger/Utils/UTF8IOCompat.hs +6/−24
• hledger-lib.cabal +23/−125
• hledger_csv.5 +976/−361
• hledger_csv.info +939/−379
• hledger_csv.txt +641/−157
• hledger_journal.5 +185/−122
• hledger_journal.info +292/−207
• hledger_journal.txt +142/−90
• hledger_timeclock.5 +2/−2
• hledger_timeclock.info +2/−2
• hledger_timeclock.txt +2/−2
• hledger_timedot.5 +2/−2
• hledger_timedot.info +4/−4
• hledger_timedot.txt +2/−2
• test/doctests.hs +2/−0
• test/easytests.hs +0/−5
• test/unittest.hs +11/−0

@@ -1,6 +1,29 @@ Internal/api/developer-ish changes in the hledger-lib (and hledger) packages. For user-visible changes, see the hledger package changelog. +# 1.16 2019-12-01++- drop support for GHC 7.10, due to MonadFail hassles in JournalReader.hs++- add support for GHC 8.8, base-compat 0.11 (#1090)++  We are now using the new fail from the MonadFail class, which we+  always import qualified as Fail.fail, from base-compat-batteries+  Control.Monad.Fail.Compat to work with old GHC versions. If old fail+  is needed (shouldn't be) it should be imported qualified as+  Prelude.Fail, using imports such as:++      import Prelude hiding (fail)+      import qualified Prelude (fail)+      import Control.Monad.State.Strict hiding (fail)+      import "base-compat-batteries" Prelude.Compat hiding (fail)+      import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail++- hledger and hledger-lib unit tests have been ported to tasty.++- The doctest suite has been disabled for now since it doesn't run+  well with cabal (#1139)+   # 1.15.2 2019-09-05  Changes:

@@ -227,27 +227,23 @@ --isAccountRegex s = take 1 s == "^" && take 5 (reverse s) == ")$|:("  tests_AccountName = tests "AccountName" [-  tests "accountNameTreeFrom" [-     accountNameTreeFrom ["a"]       `is` Node "root" [Node "a" []]-    ,accountNameTreeFrom ["a","b"]   `is` Node "root" [Node "a" [], Node "b" []]-    ,accountNameTreeFrom ["a","a:b"] `is` Node "root" [Node "a" [Node "a:b" []]]-    ,accountNameTreeFrom ["a:b:c"]   `is` Node "root" [Node "a" [Node "a:b" [Node "a:b:c" []]]]-  ]-  ,tests "expandAccountNames" [-    expandAccountNames ["assets:cash","assets:checking","expenses:vacation"] `is`+   test "accountNameTreeFrom" $ do+    accountNameTreeFrom ["a"]       @?= Node "root" [Node "a" []]+    accountNameTreeFrom ["a","b"]   @?= Node "root" [Node "a" [], Node "b" []]+    accountNameTreeFrom ["a","a:b"] @?= Node "root" [Node "a" [Node "a:b" []]]+    accountNameTreeFrom ["a:b:c"]   @?= Node "root" [Node "a" [Node "a:b" [Node "a:b:c" []]]]+  ,test "expandAccountNames" $ do+    expandAccountNames ["assets:cash","assets:checking","expenses:vacation"] @?=      ["assets","assets:cash","assets:checking","expenses","expenses:vacation"]-  ]-  ,tests "isAccountNamePrefixOf" [-     "assets" `isAccountNamePrefixOf` "assets" `is` False-    ,"assets" `isAccountNamePrefixOf` "assets:bank" `is` True-    ,"assets" `isAccountNamePrefixOf` "assets:bank:checking" `is` True-    ,"my assets" `isAccountNamePrefixOf` "assets:bank" `is` False-  ]-  ,tests "isSubAccountNameOf" [-     "assets" `isSubAccountNameOf` "assets" `is` False-    ,"assets:bank" `isSubAccountNameOf` "assets" `is` True-    ,"assets:bank:checking" `isSubAccountNameOf` "assets" `is` False-    ,"assets:bank" `isSubAccountNameOf` "my assets" `is` False-  ]+  ,test "isAccountNamePrefixOf" $ do+    "assets" `isAccountNamePrefixOf` "assets" @?= False+    "assets" `isAccountNamePrefixOf` "assets:bank" @?= True+    "assets" `isAccountNamePrefixOf` "assets:bank:checking" @?= True+    "my assets" `isAccountNamePrefixOf` "assets:bank" @?= False+  ,test "isSubAccountNameOf" $ do+    "assets" `isSubAccountNameOf` "assets" @?= False+    "assets:bank" `isSubAccountNameOf` "assets" @?= True+    "assets:bank:checking" `isSubAccountNameOf` "assets" @?= False+    "assets:bank" `isSubAccountNameOf` "my assets" @?= False  ] 

@@ -51,6 +51,7 @@   usd,   eur,   gbp,+  per,   hrs,   at,   (@@),@@ -66,6 +67,7 @@   -- ** rendering   amountstyle,   styleAmount,+  styleAmountExceptPrecision,   showAmount,   cshowAmount,   showAmountWithZeroCommodity,@@ -180,6 +182,7 @@ usd n = amount{acommodity="$", aquantity=roundTo 2 n, astyle=amountstyle{asprecision=2}} eur n = amount{acommodity="€", aquantity=roundTo 2 n, astyle=amountstyle{asprecision=2}} gbp n = amount{acommodity="£", aquantity=roundTo 2 n, astyle=amountstyle{asprecision=2}}+per n = amount{acommodity="%", aquantity=n,           astyle=amountstyle{asprecision=1, ascommodityside=R, ascommodityspaced=True}} amt `at` priceamt = amt{aprice=Just $ UnitPrice priceamt} amt @@ priceamt = amt{aprice=Just $ TotalPrice priceamt} @@ -274,17 +277,17 @@ showAmountWithPrecision :: Int -> Amount -> String showAmountWithPrecision p = showAmount . setAmountPrecision p --- | Set an amount's display precision.-setAmountPrecision :: Int -> Amount -> Amount-setAmountPrecision p a@Amount{astyle=s} = a{astyle=s{asprecision=p}}- -- | Set an amount's display precision, flipped. withPrecision :: Amount -> Int -> Amount withPrecision = flip setAmountPrecision --- | Increase an amount's display precision, if necessary, enough so--- that it will be shown exactly, with all significant decimal places--- (excluding trailing zeros).+-- | Set an amount's display precision.+setAmountPrecision :: Int -> Amount -> Amount+setAmountPrecision p a@Amount{astyle=s} = a{astyle=s{asprecision=p}}++-- | Increase an amount's display precision, if needed, to enough+-- decimal places to show it exactly (showing all significant decimal+-- digits, excluding trailing zeros). setFullPrecision :: Amount -> Amount setFullPrecision a = setAmountPrecision p a   where@@ -292,16 +295,18 @@     displayprecision = asprecision $ astyle a     normalprecision  = fromIntegral $ decimalPlaces $ normalizeDecimal $ aquantity a --- | Set an amount's display precision to just enough so that it will--- be shown exactly, with all significant decimal places.+-- | Set an amount's display precision to just enough decimal places+-- to show it exactly (possibly less than the number specified by+-- the amount's display style). setNaturalPrecision :: Amount -> Amount setNaturalPrecision a = setAmountPrecision normalprecision a   where     normalprecision  = fromIntegral $ decimalPlaces $ normalizeDecimal $ aquantity a --- | Set an amount's display precision to just enough so that all--- significant decimal digits will be shown, but not more than the--- given maximum number of decimal digits.+-- | Set an amount's display precision to just enough decimal places+-- to show it exactly (possibly less than the number specified by the+-- amount's display style), but not more than the given maximum number+-- of decimal digits. setNaturalPrecisionUpTo :: Int -> Amount -> Amount setNaturalPrecisionUpTo n a = setAmountPrecision (min n normalprecision) a   where@@ -368,6 +373,13 @@     Just s  -> a{astyle=s}     Nothing -> a +-- | Like styleAmount, but keep the number of decimal places unchanged.+styleAmountExceptPrecision :: M.Map CommoditySymbol AmountStyle -> Amount -> Amount+styleAmountExceptPrecision styles a@Amount{astyle=AmountStyle{asprecision=origp}} =+  case M.lookup (acommodity a) styles of+    Just s  -> a{astyle=s{asprecision=origp}}+    Nothing -> a+ -- | Get the string representation of an amount, based on its -- commodity's display settings. String representations equivalent to -- zero are converted to just \"0\". The special "missing" amount is@@ -412,9 +424,9 @@         | p == maxprecision          = chopdotzero $ show q         | otherwise                  = show $ roundTo (fromIntegral p) q --- | Replace a number string's decimal point with the specified character,--- and add the specified digit group separators. The last digit group will--- be repeated as needed.+-- | Replace a number string's decimal mark with the specified+-- character, and add the specified digit group marks. The last digit+-- group will be repeated as needed. punctuatenumber :: Char -> Maybe DigitGroupStyle -> String -> String punctuatenumber dec mgrps s = sign ++ reverse (applyDigitGroupStyle mgrps (reverse int)) ++ frac''     where@@ -725,99 +737,88 @@ tests_Amount = tests "Amount" [    tests "Amount" [ -     tests "costOfAmount" [-       costOfAmount (eur 1) `is` eur 1-      ,costOfAmount (eur 2){aprice=Just $ UnitPrice $ usd 2} `is` usd 4-      ,costOfAmount (eur 1){aprice=Just $ TotalPrice $ usd 2} `is` usd 2-      ,costOfAmount (eur (-1)){aprice=Just $ TotalPrice $ usd 2} `is` usd (-2)-    ]+     test "costOfAmount" $ do+       costOfAmount (eur 1) @?= eur 1+       costOfAmount (eur 2){aprice=Just $ UnitPrice $ usd 2} @?= usd 4+       costOfAmount (eur 1){aprice=Just $ TotalPrice $ usd 2} @?= usd 2+       costOfAmount (eur (-1)){aprice=Just $ TotalPrice $ usd 2} @?= usd (-2) -    ,tests "isZeroAmount" [-       expect $ isZeroAmount amount-      ,expect $ isZeroAmount $ usd 0-    ]+    ,test "isZeroAmount" $ do+       assertBool "" $ isZeroAmount amount+       assertBool "" $ isZeroAmount $ usd 0 -    ,tests "negating amounts" [-       negate (usd 1) `is` (usd 1){aquantity= -1}-      ,let b = (usd 1){aprice=Just $ UnitPrice $ eur 2} in negate b `is` b{aquantity= -1}-    ]+    ,test "negating amounts" $ do+       negate (usd 1) @?= (usd 1){aquantity= -1}+       let b = (usd 1){aprice=Just $ UnitPrice $ eur 2} in negate b @?= b{aquantity= -1} -    ,tests "adding amounts without prices" [-       (usd 1.23 + usd (-1.23)) `is` usd 0-      ,(usd 1.23 + usd (-1.23)) `is` usd 0-      ,(usd (-1.23) + usd (-1.23)) `is` usd (-2.46)-      ,sum [usd 1.23,usd (-1.23),usd (-1.23),-(usd (-1.23))] `is` usd 0-      -- highest precision is preserved-      ,asprecision (astyle $ sum [usd 1 `withPrecision` 1, usd 1 `withPrecision` 3]) `is` 3-      ,asprecision (astyle $ sum [usd 1 `withPrecision` 3, usd 1 `withPrecision` 1]) `is` 3-      -- adding different commodities assumes conversion rate 1-      ,expect $ isZeroAmount (usd 1.23 - eur 1.23)-    ]+    ,test "adding amounts without prices" $ do+       (usd 1.23 + usd (-1.23)) @?= usd 0+       (usd 1.23 + usd (-1.23)) @?= usd 0+       (usd (-1.23) + usd (-1.23)) @?= usd (-2.46)+       sum [usd 1.23,usd (-1.23),usd (-1.23),-(usd (-1.23))] @?= usd 0+       -- highest precision is preserved+       asprecision (astyle $ sum [usd 1 `withPrecision` 1, usd 1 `withPrecision` 3]) @?= 3+       asprecision (astyle $ sum [usd 1 `withPrecision` 3, usd 1 `withPrecision` 1]) @?= 3+       -- adding different commodities assumes conversion rate 1+       assertBool "" $ isZeroAmount (usd 1.23 - eur 1.23) -    ,tests "showAmount" [-      showAmount (usd 0 + gbp 0) `is` "0"-    ]+    ,test "showAmount" $ do+      showAmount (usd 0 + gbp 0) @?= "0"    ]    ,tests "MixedAmount" [ -     tests "adding mixed amounts to zero, the commodity and amount style are preserved" [+     test "adding mixed amounts to zero, the commodity and amount style are preserved" $       sum (map (Mixed . (:[]))                [usd 1.25                ,usd (-1) `withPrecision` 3                ,usd (-0.25)                ])-        `is` Mixed [usd 0 `withPrecision` 3]-    ]+        @?= Mixed [usd 0 `withPrecision` 3] -    ,tests "adding mixed amounts with total prices" [+    ,test "adding mixed amounts with total prices" $ do       sum (map (Mixed . (:[]))        [usd 1 @@ eur 1        ,usd (-2) @@ eur 1        ])-        `is` Mixed [usd 1 @@ eur 1+        @?= Mixed [usd 1 @@ eur 1                    ,usd (-2) @@ eur 1                    ]-    ] -    ,tests "showMixedAmount" [-       showMixedAmount (Mixed [usd 1]) `is` "$1.00"-      ,showMixedAmount (Mixed [usd 1 `at` eur 2]) `is` "$1.00 @ €2.00"-      ,showMixedAmount (Mixed [usd 0]) `is` "0"-      ,showMixedAmount (Mixed []) `is` "0"-      ,showMixedAmount missingmixedamt `is` ""-    ]+    ,test "showMixedAmount" $ do+       showMixedAmount (Mixed [usd 1]) @?= "$1.00"+       showMixedAmount (Mixed [usd 1 `at` eur 2]) @?= "$1.00 @ €2.00"+       showMixedAmount (Mixed [usd 0]) @?= "0"+       showMixedAmount (Mixed []) @?= "0"+       showMixedAmount missingmixedamt @?= "" -    ,tests "showMixedAmountWithoutPrice" $-      let a = usd 1 `at` eur 2 in-    [-        showMixedAmountWithoutPrice (Mixed [a]) `is` "$1.00"-       ,showMixedAmountWithoutPrice (Mixed [a, -a]) `is` "0"-    ]+    ,test "showMixedAmountWithoutPrice" $ do+      let a = usd 1 `at` eur 2+      showMixedAmountWithoutPrice (Mixed [a]) @?= "$1.00"+      showMixedAmountWithoutPrice (Mixed [a, -a]) @?= "0"      ,tests "normaliseMixedAmount" [        test "a missing amount overrides any other amounts" $-        normaliseMixedAmount (Mixed [usd 1, missingamt]) `is` missingmixedamt+        normaliseMixedAmount (Mixed [usd 1, missingamt]) @?= missingmixedamt       ,test "unpriced same-commodity amounts are combined" $-        normaliseMixedAmount (Mixed [usd 0, usd 2]) `is` Mixed [usd 2]+        normaliseMixedAmount (Mixed [usd 0, usd 2]) @?= Mixed [usd 2]       ,test "amounts with same unit price are combined" $-        normaliseMixedAmount (Mixed [usd 1 `at` eur 1, usd 1 `at` eur 1]) `is` Mixed [usd 2 `at` eur 1]+        normaliseMixedAmount (Mixed [usd 1 `at` eur 1, usd 1 `at` eur 1]) @?= Mixed [usd 2 `at` eur 1]       ,test "amounts with different unit prices are not combined" $-        normaliseMixedAmount (Mixed [usd 1 `at` eur 1, usd 1 `at` eur 2]) `is` Mixed [usd 1 `at` eur 1, usd 1 `at` eur 2]+        normaliseMixedAmount (Mixed [usd 1 `at` eur 1, usd 1 `at` eur 2]) @?= Mixed [usd 1 `at` eur 1, usd 1 `at` eur 2]       ,test "amounts with total prices are not combined" $-        normaliseMixedAmount (Mixed  [usd 1 @@ eur 1, usd 1 @@ eur 1]) `is` Mixed [usd 1 @@ eur 1, usd 1 @@ eur 1]+        normaliseMixedAmount (Mixed  [usd 1 @@ eur 1, usd 1 @@ eur 1]) @?= Mixed [usd 1 @@ eur 1, usd 1 @@ eur 1]     ] -    ,tests "normaliseMixedAmountSquashPricesForDisplay" [-       normaliseMixedAmountSquashPricesForDisplay (Mixed []) `is` Mixed [nullamt]-      ,expect $ isZeroMixedAmount $ normaliseMixedAmountSquashPricesForDisplay+    ,test "normaliseMixedAmountSquashPricesForDisplay" $ do+       normaliseMixedAmountSquashPricesForDisplay (Mixed []) @?= Mixed [nullamt]+       assertBool "" $ isZeroMixedAmount $ normaliseMixedAmountSquashPricesForDisplay         (Mixed [usd 10                ,usd 10 @@ eur 7                ,usd (-10)                ,usd (-10) @@ eur 7                ])-    ]    ] 

@@ -60,7 +60,6 @@   spansSpan,   spanIntersect,   spansIntersect,-  spanIntervalIntersect,   spanDefaultsFrom,   spanUnion,   spansUnion,@@ -76,10 +75,9 @@ ) where -import Prelude ()-import "base-compat-batteries" Prelude.Compat+import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail (MonadFail, fail) import Control.Applicative.Permutations-import Control.Monad+import Control.Monad (unless) import "base-compat-batteries" Data.List.Compat import Data.Default import Data.Maybe@@ -262,27 +260,6 @@       b = latest b1 b2       e = earliest e1 e2 --- | Calculate the intersection of two DateSpans, adjusting the start date so--- the interval is preserved.------ >>> let intervalIntersect = spanIntervalIntersect (Days 3)--- >>> mkdatespan "2018-01-01" "2018-01-03" `intervalIntersect` mkdatespan "2018-01-01" "2018-01-05"--- DateSpan 2018/01/01-2018/01/02--- >>> mkdatespan "2018-01-01" "2018-01-05" `intervalIntersect` mkdatespan "2018-01-02" "2018-01-05"--- DateSpan 2018/01/04--- >>> mkdatespan "2018-01-01" "2018-01-05" `intervalIntersect` mkdatespan "2018-01-03" "2018-01-05"--- DateSpan 2018/01/04--- >>> mkdatespan "2018-01-01" "2018-01-05" `intervalIntersect` mkdatespan "2018-01-04" "2018-01-05"--- DateSpan 2018/01/04--- >>> mkdatespan "2018-01-01" "2018-01-05" `intervalIntersect` mkdatespan "2017-12-01" "2018-01-05"--- DateSpan 2018/01/01-2018/01/04-spanIntervalIntersect :: Interval -> DateSpan -> DateSpan -> DateSpan-spanIntervalIntersect (Days n) (DateSpan (Just b1) e1) sp2@(DateSpan (Just b2) _) =-      DateSpan (Just b) e1 `spanIntersect` sp2-    where-      b = if b1 < b2 then addDays (diffDays b1 b2 `mod` toInteger n) b2 else b1-spanIntervalIntersect _ sp1 sp2 = sp1 `spanIntersect` sp2- -- | Fill any unspecified dates in the first span with the dates from -- the second one. Sort of a one-way spanIntersect. spanDefaultsFrom (DateSpan a1 b1) (DateSpan a2 b2) = DateSpan a b@@ -771,10 +748,10 @@ validMonth s = maybe False (\n -> n>=1 && n<=12) $ readMay s validDay s = maybe False (\n -> n>=1 && n<=31) $ readMay s -failIfInvalidYear, failIfInvalidMonth, failIfInvalidDay :: (Monad m) => String -> m ()-failIfInvalidYear s  = unless (validYear s)  $ fail $ "bad year number: " ++ s-failIfInvalidMonth s = unless (validMonth s) $ fail $ "bad month number: " ++ s-failIfInvalidDay s   = unless (validDay s)   $ fail $ "bad day number: " ++ s+failIfInvalidYear, failIfInvalidMonth, failIfInvalidDay :: (Fail.MonadFail m) => String -> m ()+failIfInvalidYear s  = unless (validYear s)  $ Fail.fail $ "bad year number: " ++ s+failIfInvalidMonth s = unless (validMonth s) $ Fail.fail $ "bad month number: " ++ s+failIfInvalidDay s   = unless (validDay s)   $ Fail.fail $ "bad day number: " ++ s  yyyymmdd :: TextParser m SmartDate yyyymmdd = do@@ -864,8 +841,8 @@   wday <- T.toLower <$> (choice . map string' $ weekdays ++ weekdayabbrevs)   case catMaybes $ [wday `elemIndex` weekdays, wday `elemIndex` weekdayabbrevs] of     (i:_) -> return (i+1)-    []    -> fail  $ "weekday: should not happen: attempted to find " <>-                      show wday <> " in " <> show (weekdays ++ weekdayabbrevs)+    []    -> Fail.fail $ "weekday: should not happen: attempted to find " <>+                         show wday <> " in " <> show (weekdays ++ weekdayabbrevs)  today,yesterday,tomorrow :: TextParser m SmartDate today     = string' "today"     >> return ("","","today")

@@ -24,7 +24,7 @@   journalApplyCommodityStyles,   commodityStylesFromAmounts,   journalCommodityStyles,-  journalConvertAmountsToCost,+  journalToCost,   journalReverse,   journalSetLastReadTime,   journalPivot,@@ -905,18 +905,23 @@ -- | Choose and apply a consistent display format to the posting -- amounts in each commodity. Each commodity's format is specified by -- a commodity format directive, or otherwise inferred from posting--- amounts as in hledger < 0.28.-journalApplyCommodityStyles :: Journal -> Journal-journalApplyCommodityStyles j@Journal{jtxns=ts, jpricedirectives=pds} = j''-    where-      j' = journalInferCommodityStyles j-      styles = journalCommodityStyles j'-      j'' = j'{jtxns=map fixtransaction ts, jpricedirectives=map fixpricedirective pds}-      fixtransaction t@Transaction{tpostings=ps} = t{tpostings=map fixposting ps}-      fixposting p = p{pamount=styleMixedAmount styles $ pamount p-                      ,pbalanceassertion=fixbalanceassertion <$> pbalanceassertion p}-      fixbalanceassertion ba = ba{baamount=styleAmount styles $ baamount ba}-      fixpricedirective pd@PriceDirective{pdamount=a} = pd{pdamount=styleAmount styles a}+-- amounts as in hledger < 0.28. Can return an error message+-- eg if inconsistent number formats are found.+journalApplyCommodityStyles :: Journal -> Either String Journal+journalApplyCommodityStyles j@Journal{jtxns=ts, jpricedirectives=pds} =+  case journalInferCommodityStyles j of+    Left e   -> Left e+    Right j' -> Right j''+      where+        styles = journalCommodityStyles j'+        j'' = j'{jtxns=map fixtransaction ts+                ,jpricedirectives=map fixpricedirective pds+                }+        fixtransaction t@Transaction{tpostings=ps} = t{tpostings=map fixposting ps}+        fixposting p = p{pamount=styleMixedAmount styles $ pamount p+                        ,pbalanceassertion=fixbalanceassertion <$> pbalanceassertion p}+        fixbalanceassertion ba = ba{baamount=styleAmount styles $ baamount ba}+        fixpricedirective pd@PriceDirective{pdamount=a} = pd{pdamount=styleAmountExceptPrecision styles a}  -- | Get all the amount styles defined in this journal, either declared by -- a commodity directive or inferred from amounts, as a map from symbol to style.@@ -931,35 +936,60 @@ -- | Collect and save inferred amount styles for each commodity based on -- the posting amounts in that commodity (excluding price amounts), ie: -- "the format of the first amount, adjusted to the highest precision of all amounts".-journalInferCommodityStyles :: Journal -> Journal+-- Can return an error message eg if inconsistent number formats are found.+journalInferCommodityStyles :: Journal -> Either String Journal journalInferCommodityStyles j =-  j{jinferredcommodities =+  case     commodityStylesFromAmounts $-    dbg8 "journalInferCommmodityStyles using amounts" $ journalAmounts j}+    dbg8 "journalInferCommodityStyles using amounts" $+    journalAmounts j+  of+    Left e   -> Left e+    Right cs -> Right j{jinferredcommodities = cs} --- | Given a list of amounts in parse order, build a map from their commodity names--- to standard commodity display formats.-commodityStylesFromAmounts :: [Amount] -> M.Map CommoditySymbol AmountStyle-commodityStylesFromAmounts amts = M.fromList commstyles+-- | Given a list of parsed amounts, in parse order, build a map from+-- their commodity names to standard commodity display formats. Can+-- return an error message eg if inconsistent number formats are+-- found.+--+-- Though, these amounts may have come from multiple files, so we+-- shouldn't assume they use consistent number formats.+-- Currently we don't enforce that even within a single file,+-- and this function never reports an error.+--+commodityStylesFromAmounts :: [Amount] -> Either String (M.Map CommoditySymbol AmountStyle)+commodityStylesFromAmounts amts =+  Right $ M.fromList commstyles   where     commamts = groupSort [(acommodity as, as) | as <- amts]     commstyles = [(c, canonicalStyleFrom $ map astyle as) | (c,as) <- commamts] --- | Given an ordered list of amount styles, choose a canonical style.--- That is: the style of the first, and the maximum precision of all.+-- TODO: should probably detect and report inconsistencies here+-- | Given a list of amount styles (assumed to be from parsed amounts+-- in a single commodity), in parse order, choose a canonical style.+-- Traditionally it's "the style of the first, with the maximum precision of all".+-- canonicalStyleFrom :: [AmountStyle] -> AmountStyle canonicalStyleFrom [] = amountstyle-canonicalStyleFrom ss@(first:_) = first {asprecision = prec, asdecimalpoint = mdec, asdigitgroups = mgrps}+canonicalStyleFrom ss@(s:_) =+  s{asprecision=prec, asdecimalpoint=Just decmark, asdigitgroups=mgrps}   where-    mgrps = headMay $ mapMaybe asdigitgroups ss     -- precision is maximum of all precisions     prec = maximumStrict $ map asprecision ss-    mdec = Just $ headDef '.' $ mapMaybe asdecimalpoint ss-    -- precision is that of first amount with a decimal point-    -- (mdec, prec) =-    --   case filter (isJust . asdecimalpoint) ss of-    --   (s:_) -> (asdecimalpoint s, asprecision s)-    --   []    -> (Just '.', 0)+    -- identify the digit group mark (& group sizes)+    mgrps = headMay $ mapMaybe asdigitgroups ss+    -- if a digit group mark was identified above, we can rely on that;+    -- make sure the decimal mark is different. If not, default to period.+    defdecmark =+      case mgrps of+        Just (DigitGroups '.' _) -> ','+        _                        -> '.'+    -- identify the decimal mark: the first one used, or the above default,+    -- but never the same character as the digit group mark.+    -- urgh.. refactor..+    decmark = case mgrps of+                Just _ -> defdecmark+                _      -> headDef defdecmark $ mapMaybe asdecimalpoint ss  -- -- | Apply this journal's historical price records to unpriced amounts where possible. -- journalApplyPriceDirectives :: Journal -> Journal@@ -983,14 +1013,9 @@  -- | Convert all this journal's amounts to cost using the transaction prices, if any. -- The journal's commodity styles are applied to the resulting amounts.-journalConvertAmountsToCost :: Journal -> Journal-journalConvertAmountsToCost j@Journal{jtxns=ts} = j{jtxns=map fixtransaction ts}+journalToCost :: Journal -> Journal+journalToCost j@Journal{jtxns=ts} = j{jtxns=map (transactionToCost styles) ts}     where-      -- similar to journalApplyCommodityStyles-      fixtransaction t@Transaction{tpostings=ps} = t{tpostings=map fixposting ps}-      fixposting p@Posting{pamount=a} = p{pamount=fixmixedamount a}-      fixmixedamount (Mixed as) = Mixed $ map fixamount as-      fixamount = styleAmount styles . costOfAmount       styles = journalCommodityStyles j  -- -- | Get this journal's unique, display-preference-canonicalised commodities, by symbol.@@ -1281,7 +1306,7 @@                               }               ]       }-    `is` (DateSpan (Just $ fromGregorian 2014 1 10) (Just $ fromGregorian 2014 10 11))+    @?= (DateSpan (Just $ fromGregorian 2014 1 10) (Just $ fromGregorian 2014 10 11))    ,tests "standard account type queries" $     let@@ -1290,11 +1315,11 @@       journalAccountNamesMatching q = filter (q `matchesAccount`) . journalAccountNames       namesfrom qfunc = journalAccountNamesMatching (qfunc j) j     in [-       test "assets"      $ expectEq (namesfrom journalAssetAccountQuery)     ["assets","assets:bank","assets:bank:checking","assets:bank:saving","assets:cash"]-      ,test "liabilities" $ expectEq (namesfrom journalLiabilityAccountQuery) ["liabilities","liabilities:debts"]-      ,test "equity"      $ expectEq (namesfrom journalEquityAccountQuery)    []-      ,test "income"      $ expectEq (namesfrom journalRevenueAccountQuery)    ["income","income:gifts","income:salary"]-      ,test "expenses"    $ expectEq (namesfrom journalExpenseAccountQuery)   ["expenses","expenses:food","expenses:supplies"]+       test "assets"      $ assertEqual "" (namesfrom journalAssetAccountQuery)     ["assets","assets:bank","assets:bank:checking","assets:bank:saving","assets:cash"]+      ,test "liabilities" $ assertEqual "" (namesfrom journalLiabilityAccountQuery) ["liabilities","liabilities:debts"]+      ,test "equity"      $ assertEqual "" (namesfrom journalEquityAccountQuery)    []+      ,test "income"      $ assertEqual "" (namesfrom journalRevenueAccountQuery)    ["income","income:gifts","income:salary"]+      ,test "expenses"    $ assertEqual "" (namesfrom journalExpenseAccountQuery)   ["expenses","expenses:food","expenses:supplies"]     ]    ,tests "journalBalanceTransactions" [@@ -1306,12 +1331,12 @@             nulljournal{ jtxns = [               transaction "2019/01/01" [ vpost' "a" missingamt (balassert (num 1)) ]             ]}-      expectRight ej+      assertRight ej       let Right j = ej-      (jtxns j & head & tpostings & head & pamount) `is` Mixed [num 1]+      (jtxns j & head & tpostings & head & pamount) @?= Mixed [num 1]      ,test "same-day-1" $ do-      expectRight $ journalBalanceTransactions True $+      assertRight $ journalBalanceTransactions True $             --2019/01/01             --  (a)            = 1             --2019/01/01@@ -1322,7 +1347,7 @@             ]}      ,test "same-day-2" $ do-      expectRight $ journalBalanceTransactions True $+      assertRight $ journalBalanceTransactions True $             --2019/01/01             --    (a)                  2 = 2             --2019/01/01@@ -1340,7 +1365,7 @@             ]}      ,test "out-of-order" $ do-      expectRight $ journalBalanceTransactions True $+      assertRight $ journalBalanceTransactions True $             --2019/1/2             --  (a)    1 = 2             --2019/1/1@@ -1351,5 +1376,40 @@             ]}      ]++    ,tests "commodityStylesFromAmounts" $ [++      -- Journal similar to the one on #1091:+      -- 2019/09/24+      --     (a)            1,000.00+      -- +      -- 2019/09/26+      --     (a)             1000,000+      --+      test "1091a" $ do+        commodityStylesFromAmounts [+           nullamt{aquantity=1000, astyle=AmountStyle L False 3 (Just ',') Nothing}+          ,nullamt{aquantity=1000, astyle=AmountStyle L False 2 (Just '.') (Just (DigitGroups ',' [3]))}+          ]+         @?=+          -- The commodity style should have period as decimal mark+          -- and comma as digit group mark.+          Right (M.fromList [+            ("", AmountStyle L False 3 (Just '.') (Just (DigitGroups ',' [3])))+          ])+        -- same journal, entries in reverse order+      ,test "1091b" $ do+        commodityStylesFromAmounts [+           nullamt{aquantity=1000, astyle=AmountStyle L False 2 (Just '.') (Just (DigitGroups ',' [3]))}+          ,nullamt{aquantity=1000, astyle=AmountStyle L False 3 (Just ',') Nothing}+          ]+         @?=+          -- The commodity style should have period as decimal mark+          -- and comma as digit group mark.+          Right (M.fromList [+            ("", AmountStyle L False 3 (Just '.') (Just (DigitGroups ',' [3])))+          ])++     ]    ]

@@ -109,12 +109,9 @@ -- tests  tests_Ledger =-  tests-    "Ledger"-    [ tests-        "ledgerFromJournal"-        [ length (ledgerPostings $ ledgerFromJournal Any nulljournal) `is` 0-        , length (ledgerPostings $ ledgerFromJournal Any samplejournal) `is` 13-        , length (ledgerPostings $ ledgerFromJournal (Depth 2) samplejournal) `is` 7-        ]-    ]+  tests "Ledger" [+    test "ledgerFromJournal" $ do+        length (ledgerPostings $ ledgerFromJournal Any nulljournal) @?= 0+        length (ledgerPostings $ ledgerFromJournal Any samplejournal) @?= 13+        length (ledgerPostings $ ledgerFromJournal (Depth 2) samplejournal) @?= 7+  ]

@@ -24,7 +24,7 @@ import Hledger.Data.Posting (post, commentAddTagNextLine) import Hledger.Data.Transaction import Hledger.Utils.UTF8IOCompat (error')--- import Hledger.Utils.Debug+import Hledger.Utils.Debug  -- $setup -- >>> :set -XOverloadedStrings@@ -45,6 +45,17 @@           nullperiodictransaction{ ptperiodexpr=t , ptspan=s, ptinterval=i, ptpostings=["a" `post` usd 1] }           nulldatespan +_ptgenspan str span = do+  let+    t = T.pack str+    (i,s) = parsePeriodExpr' nulldate t+  case checkPeriodicTransactionStartDate i s t of+    Just e  -> error' e+    Nothing ->+      mapM_ (putStr . showTransaction) $+        runPeriodicTransaction+          nullperiodictransaction{ ptperiodexpr=t , ptspan=s, ptinterval=i, ptpostings=["a" `post` usd 1] }+          span  --deriving instance Show PeriodicTransaction -- for better pretty-printing:@@ -76,107 +87,107 @@ -- -- >>> _ptgen "monthly from 2017/1 to 2017/4" -- 2017/01/01---     ; generated-transaction:~ monthly from 2017/1 to 2017/4+--     ; generated-transaction: ~ monthly from 2017/1 to 2017/4 --     a           $1.00 -- <BLANKLINE> -- 2017/02/01---     ; generated-transaction:~ monthly from 2017/1 to 2017/4+--     ; generated-transaction: ~ monthly from 2017/1 to 2017/4 --     a           $1.00 -- <BLANKLINE> -- 2017/03/01---     ; generated-transaction:~ monthly from 2017/1 to 2017/4+--     ; generated-transaction: ~ monthly from 2017/1 to 2017/4 --     a           $1.00 -- <BLANKLINE> -- -- >>> _ptgen "monthly from 2017/1 to 2017/5" -- 2017/01/01---     ; generated-transaction:~ monthly from 2017/1 to 2017/5+--     ; generated-transaction: ~ monthly from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- 2017/02/01---     ; generated-transaction:~ monthly from 2017/1 to 2017/5+--     ; generated-transaction: ~ monthly from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- 2017/03/01---     ; generated-transaction:~ monthly from 2017/1 to 2017/5+--     ; generated-transaction: ~ monthly from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- 2017/04/01---     ; generated-transaction:~ monthly from 2017/1 to 2017/5+--     ; generated-transaction: ~ monthly from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- -- >>> _ptgen "every 2nd day of month from 2017/02 to 2017/04" -- 2017/01/02---     ; generated-transaction:~ every 2nd day of month from 2017/02 to 2017/04+--     ; generated-transaction: ~ every 2nd day of month from 2017/02 to 2017/04 --     a           $1.00 -- <BLANKLINE> -- 2017/02/02---     ; generated-transaction:~ every 2nd day of month from 2017/02 to 2017/04+--     ; generated-transaction: ~ every 2nd day of month from 2017/02 to 2017/04 --     a           $1.00 -- <BLANKLINE> -- 2017/03/02---     ; generated-transaction:~ every 2nd day of month from 2017/02 to 2017/04+--     ; generated-transaction: ~ every 2nd day of month from 2017/02 to 2017/04 --     a           $1.00 -- <BLANKLINE> -- -- >>> _ptgen "every 30th day of month from 2017/1 to 2017/5" -- 2016/12/30---     ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5+--     ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- 2017/01/30---     ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5+--     ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- 2017/02/28---     ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5+--     ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- 2017/03/30---     ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5+--     ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- 2017/04/30---     ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5+--     ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 --     a           $1.00 -- <BLANKLINE> -- -- >>> _ptgen "every 2nd Thursday of month from 2017/1 to 2017/4" -- 2016/12/08---     ; generated-transaction:~ every 2nd Thursday of month from 2017/1 to 2017/4+--     ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4 --     a           $1.00 -- <BLANKLINE> -- 2017/01/12---     ; generated-transaction:~ every 2nd Thursday of month from 2017/1 to 2017/4+--     ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4 --     a           $1.00 -- <BLANKLINE> -- 2017/02/09---     ; generated-transaction:~ every 2nd Thursday of month from 2017/1 to 2017/4+--     ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4 --     a           $1.00 -- <BLANKLINE> -- 2017/03/09---     ; generated-transaction:~ every 2nd Thursday of month from 2017/1 to 2017/4+--     ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4 --     a           $1.00 -- <BLANKLINE> -- -- >>> _ptgen "every nov 29th from 2017 to 2019" -- 2016/11/29---     ; generated-transaction:~ every nov 29th from 2017 to 2019+--     ; generated-transaction: ~ every nov 29th from 2017 to 2019 --     a           $1.00 -- <BLANKLINE> -- 2017/11/29---     ; generated-transaction:~ every nov 29th from 2017 to 2019+--     ; generated-transaction: ~ every nov 29th from 2017 to 2019 --     a           $1.00 -- <BLANKLINE> -- 2018/11/29---     ; generated-transaction:~ every nov 29th from 2017 to 2019+--     ; generated-transaction: ~ every nov 29th from 2017 to 2019 --     a           $1.00 -- <BLANKLINE> -- -- >>> _ptgen "2017/1" -- 2017/01/01---     ; generated-transaction:~ 2017/1+--     ; generated-transaction: ~ 2017/1 --     a           $1.00 -- <BLANKLINE> --@@ -199,11 +210,32 @@ -- >>> let reportperiod="daily from 2018/01/03" in let (i,s) = parsePeriodExpr' nulldate reportperiod in runPeriodicTransaction (nullperiodictransaction{ptperiodexpr=reportperiod, ptspan=s, ptinterval=i, ptpostings=["a" `post` usd 1]}) (DateSpan (Just $ parsedate "2018-01-01") (Just $ parsedate "2018-01-03")) -- [] --+-- >>> _ptgenspan "every 3 months from 2019-05" (mkdatespan "2020-01-01" "2020-02-01")+--  +-- >>> _ptgenspan "every 3 months from 2019-05" (mkdatespan "2020-02-01" "2020-03-01")+-- 2020/02/01+--     ; generated-transaction: ~ every 3 months from 2019-05+--     a           $1.00+-- <BLANKLINE>+-- >>> _ptgenspan "every 3 days from 2018" (mkdatespan "2018-01-01" "2018-01-05")+-- 2018/01/01+--     ; generated-transaction: ~ every 3 days from 2018+--     a           $1.00+-- <BLANKLINE>+-- 2018/01/04+--     ; generated-transaction: ~ every 3 days from 2018+--     a           $1.00+-- <BLANKLINE>+-- >>> _ptgenspan "every 3 days from 2018" (mkdatespan "2018-01-02" "2018-01-05")+-- 2018/01/04+--     ; generated-transaction: ~ every 3 days from 2018+--     a           $1.00+-- <BLANKLINE>+ runPeriodicTransaction :: PeriodicTransaction -> DateSpan -> [Transaction] runPeriodicTransaction PeriodicTransaction{..} requestedspan =-    [ t{tdate=d} | (DateSpan (Just d) _) <- ptinterval `splitSpan` spantofill ]+    [ t{tdate=d} | (DateSpan (Just d) _) <- alltxnspans, spanContainsDate requestedspan d ]   where-    spantofill = spanIntervalIntersect ptinterval ptspan requestedspan     t = nulltransaction{            tstatus      = ptstatus           ,tcode        = ptcode@@ -216,7 +248,11 @@           ,tpostings    = ptpostings           }     period = "~ " <> ptperiodexpr-+    -- All spans described by this periodic transaction, where spanStart is event date.+    -- If transaction does not have start/end date, we set them to start/end of requested span,+    -- to avoid generating (infinitely) many events. +    alltxnspans = dbg3 "alltxnspans" $ ptinterval `splitSpan` (spanDefaultsFrom ptspan requestedspan)+       -- | Check that this date span begins at a boundary of this interval, -- or return an explanatory error message including the provided period expression -- (from which the span and interval are derived).

@@ -372,58 +372,54 @@   | otherwise = c1 <> ", " <> c2  -- | Add a tag to a comment, comma-separated from any prior content.+-- A space is inserted following the colon, before the value. commentAddTag :: Text -> Tag -> Text commentAddTag c (t,v)   | T.null c' = tag   | otherwise = c' `commentJoin` tag   where     c'  = textchomp c-    tag = t <> ":" <> v+    tag = t <> ": " <> v  -- | Add a tag on its own line to a comment, preserving any prior content.+-- A space is inserted following the colon, before the value. commentAddTagNextLine :: Text -> Tag -> Text commentAddTagNextLine cmt (t,v) =-  cmt <> if "\n" `T.isSuffixOf` cmt then "" else "\n" <> t <> ":" <> v +  cmt <> if "\n" `T.isSuffixOf` cmt then "" else "\n" <> t <> ": " <> v    -- tests  tests_Posting = tests "Posting" [ -  tests "accountNamePostingType" [-    accountNamePostingType "a" `is` RegularPosting-    ,accountNamePostingType "(a)" `is` VirtualPosting-    ,accountNamePostingType "[a]" `is` BalancedVirtualPosting-  ]+  test "accountNamePostingType" $ do+    accountNamePostingType "a" @?= RegularPosting+    accountNamePostingType "(a)" @?= VirtualPosting+    accountNamePostingType "[a]" @?= BalancedVirtualPosting - ,tests "accountNameWithoutPostingType" [-    accountNameWithoutPostingType "(a)" `is` "a"-  ]+ ,test "accountNameWithoutPostingType" $ do+    accountNameWithoutPostingType "(a)" @?= "a" - ,tests "accountNameWithPostingType" [-    accountNameWithPostingType VirtualPosting "[a]" `is` "(a)"-  ]+ ,test "accountNameWithPostingType" $ do+    accountNameWithPostingType VirtualPosting "[a]" @?= "(a)" - ,tests "joinAccountNames" [-    "a" `joinAccountNames` "b:c" `is` "a:b:c"-    ,"a" `joinAccountNames` "(b:c)" `is` "(a:b:c)"-    ,"[a]" `joinAccountNames` "(b:c)" `is` "[a:b:c]"-    ,"" `joinAccountNames` "a" `is` "a"-  ]+ ,test "joinAccountNames" $ do+    "a" `joinAccountNames` "b:c" @?= "a:b:c"+    "a" `joinAccountNames` "(b:c)" @?= "(a:b:c)"+    "[a]" `joinAccountNames` "(b:c)" @?= "[a:b:c]"+    "" `joinAccountNames` "a" @?= "a" - ,tests "concatAccountNames" [-    concatAccountNames [] `is` ""-    ,concatAccountNames ["a","(b)","[c:d]"] `is` "(a:b:c:d)"-  ]+ ,test "concatAccountNames" $ do+    concatAccountNames [] @?= ""+    concatAccountNames ["a","(b)","[c:d]"] @?= "(a:b:c:d)" - ,tests "commentAddTag" [-    commentAddTag "" ("a","") `is` "a:"-   ,commentAddTag "[1/2]" ("a","") `is` "[1/2], a:"-  ]+ ,test "commentAddTag" $ do+    commentAddTag "" ("a","") @?= "a: "+    commentAddTag "[1/2]" ("a","") @?= "[1/2], a: " - ,tests "commentAddTagNextLine" [-    commentAddTagNextLine "" ("a","") `is` "\na:"-   ,commentAddTagNextLine "[1/2]" ("a","") `is` "[1/2]\na:"-  ]+ ,test "commentAddTagNextLine" $ do+    commentAddTagNextLine "" ("a","") @?= "\na: "+    commentAddTagNextLine "[1/2]" ("a","") @?= "[1/2]\na: "+  ] 

@@ -1,3 +1,5 @@+{-# LANGUAGE DeriveDataTypeable #-}+ {-|  hledger's cmdargs modes parse command-line arguments to an@@ -13,6 +15,8 @@   setboolopt,   inRawOpts,   boolopt,+  choiceopt,+  collectopts,   stringopt,   maybestringopt,   listofstringopt,@@ -23,39 +27,69 @@ where  import Data.Maybe-import qualified Data.Text as T+import Data.Data+import Data.Default import Safe  import Hledger.Utils   -- | The result of running cmdargs: an association list of option names to string values.-type RawOpts = [(String,String)]+newtype RawOpts = RawOpts { unRawOpts :: [(String,String)] }+    deriving (Show, Data, Typeable) +instance Default RawOpts where def = RawOpts []++overRawOpts f = RawOpts . f . unRawOpts+ setopt :: String -> String -> RawOpts -> RawOpts-setopt name val = (++ [(name, quoteIfNeeded $ val)])+setopt name val = overRawOpts (++ [(name, val)])  setboolopt :: String -> RawOpts -> RawOpts-setboolopt name = (++ [(name,"")])+setboolopt name = overRawOpts (++ [(name,"")])  -- | Is the named option present ? inRawOpts :: String -> RawOpts -> Bool-inRawOpts name = isJust . lookup name+inRawOpts name = isJust . lookup name . unRawOpts  boolopt :: String -> RawOpts -> Bool boolopt = inRawOpts +-- | From a list of RawOpts, get the last one (ie the right-most on the command line)+-- for which the given predicate returns a Just value.+-- Useful for exclusive choice flags like --daily|--weekly|--quarterly...+--+-- >>> choiceopt Just (RawOpts [("a",""), ("b",""), ("c","")])+-- Just "c"+-- >>> choiceopt (const Nothing) (RawOpts [("a","")])+-- Nothing+-- >>> choiceopt readMay (RawOpts [("LT",""),("EQ",""),("Neither","")]) :: Maybe Ordering+-- Just EQ+choiceopt :: (String -> Maybe a) -- ^ "parser" that returns 'Just' value for valid choice+          -> RawOpts             -- ^ actual options where to look for flag+          -> Maybe a             -- ^ exclusive choice among those returned as 'Just' from "parser"+choiceopt f = lastMay . collectopts (f . fst)++-- | Collects processed and filtered list of options preserving their order+--+-- >>> collectopts (const Nothing) (RawOpts [("x","")])+-- []+-- >>> collectopts Just (RawOpts [("a",""),("b","")])+-- [("a",""),("b","")]+collectopts :: ((String, String) -> Maybe a) -> RawOpts -> [a]+collectopts f = mapMaybe f . unRawOpts+ maybestringopt :: String -> RawOpts -> Maybe String-maybestringopt name = fmap (T.unpack . stripquotes . T.pack) . lookup name . reverse+maybestringopt name = lookup name . reverse . unRawOpts  stringopt :: String -> RawOpts -> String stringopt name = fromMaybe "" . maybestringopt name  maybecharopt :: String -> RawOpts -> Maybe Char-maybecharopt name rawopts = lookup name rawopts >>= headMay+maybecharopt name (RawOpts rawopts) = lookup name rawopts >>= headMay  listofstringopt :: String -> RawOpts -> [String]-listofstringopt name rawopts = [v | (k,v) <- rawopts, k==name]+listofstringopt name (RawOpts rawopts) = [v | (k,v) <- rawopts, k==name]  maybeintopt :: String -> RawOpts -> Maybe Int maybeintopt name rawopts =

@@ -18,7 +18,7 @@ import Numeric import Data.Char (isPrint) import Data.Maybe-import qualified Data.Text as T+-- import qualified Data.Text as T import Text.Megaparsec import Text.Megaparsec.Char @@ -137,7 +137,7 @@  ---------------------------------------------------------------------- -formatStringTester fs value expected = actual `is` expected+formatStringTester fs value expected = actual @?= expected   where     actual = case fs of       FormatLiteral l                   -> formatString False Nothing Nothing l@@ -145,20 +145,18 @@  tests_StringFormat = tests "StringFormat" [ -   tests "formatStringHelper" [+   test "formatStringHelper" $ do       formatStringTester (FormatLiteral " ")                                     ""            " "-    , formatStringTester (FormatField False Nothing Nothing DescriptionField)    "description" "description"-    , formatStringTester (FormatField False (Just 20) Nothing DescriptionField)  "description" "         description"-    , formatStringTester (FormatField False Nothing (Just 20) DescriptionField)  "description" "description"-    , formatStringTester (FormatField True Nothing (Just 20) DescriptionField)   "description" "description"-    , formatStringTester (FormatField True (Just 20) Nothing DescriptionField)   "description" "description         "-    , formatStringTester (FormatField True (Just 20) (Just 20) DescriptionField) "description" "description         "-    , formatStringTester (FormatField True Nothing (Just 3) DescriptionField)    "description" "des"-    ]+      formatStringTester (FormatField False Nothing Nothing DescriptionField)    "description" "description"+      formatStringTester (FormatField False (Just 20) Nothing DescriptionField)  "description" "         description"+      formatStringTester (FormatField False Nothing (Just 20) DescriptionField)  "description" "description"+      formatStringTester (FormatField True Nothing (Just 20) DescriptionField)   "description" "description"+      formatStringTester (FormatField True (Just 20) Nothing DescriptionField)   "description" "description         "+      formatStringTester (FormatField True (Just 20) (Just 20) DescriptionField) "description" "description         "+      formatStringTester (FormatField True Nothing (Just 3) DescriptionField)    "description" "des" -  ,tests "parseStringFormat" $-    let s `gives` expected = test (T.pack s) $ parseStringFormat s `is` Right expected-    in [+  ,let s `gives` expected = test s $ parseStringFormat s @?= Right expected+   in tests "parseStringFormat" [       ""                           `gives` (defaultStringFormatStyle [])     , "D"                          `gives` (defaultStringFormatStyle [FormatLiteral "D"])     , "%(date)"                    `gives` (defaultStringFormatStyle [FormatField False Nothing Nothing DescriptionField])@@ -176,6 +174,6 @@                                                                      ,FormatLiteral " "                                                                      ,FormatField False Nothing (Just 10) TotalField                                                                      ])-    , test "newline not parsed" $ expectLeft $ parseStringFormat "\n"+    , test "newline not parsed" $ assertLeft $ parseStringFormat "\n"     ]  ]

@@ -115,32 +115,31 @@ -- tests  tests_Timeclock = tests "Timeclock" [-  do-   today <- io getCurrentDay-   now' <- io getCurrentTime-   tz <- io getCurrentTimeZone-   let now = utcToLocalTime tz now'-       nowstr = showtime now-       yesterday = prevday today-       clockin = TimeclockEntry nullsourcepos In-       mktime d = LocalTime d . fromMaybe midnight .+  testCaseSteps "timeclockEntriesToTransactions tests" $ \step -> do+      step "gathering data"+      today <- getCurrentDay+      now' <- getCurrentTime+      tz <- getCurrentTimeZone+      let now = utcToLocalTime tz now'+          nowstr = showtime now+          yesterday = prevday today+          clockin = TimeclockEntry nullsourcepos In+          mktime d = LocalTime d . fromMaybe midnight . #if MIN_VERSION_time(1,5,0)-                  parseTimeM True defaultTimeLocale "%H:%M:%S"+                     parseTimeM True defaultTimeLocale "%H:%M:%S" #else-                  parseTime defaultTimeLocale "%H:%M:%S"+                     parseTime defaultTimeLocale "%H:%M:%S" #endif-       showtime = formatTime defaultTimeLocale "%H:%M"-       txndescs = map (T.unpack . tdescription) . timeclockEntriesToTransactions now-       future = utcToLocalTime tz $ addUTCTime 100 now'-       futurestr = showtime future-   tests "timeclockEntriesToTransactions" [-     test "started yesterday, split session at midnight" $-      txndescs [clockin (mktime yesterday "23:00:00") "" ""] `is` ["23:00-23:59","00:00-"++nowstr]-     ,test "split multi-day sessions at each midnight" $-      txndescs [clockin (mktime (addDays (-2) today) "23:00:00") "" ""] `is `["23:00-23:59","00:00-23:59","00:00-"++nowstr]-     ,test "auto-clock-out if needed" $-      txndescs [clockin (mktime today "00:00:00") "" ""] `is` ["00:00-"++nowstr]-     ,test "use the clockin time for auto-clockout if it's in the future" $-      txndescs [clockin future "" ""] `is` [printf "%s-%s" futurestr futurestr]-     ]+          showtime = formatTime defaultTimeLocale "%H:%M"+          txndescs = map (T.unpack . tdescription) . timeclockEntriesToTransactions now+          future = utcToLocalTime tz $ addUTCTime 100 now'+          futurestr = showtime future+      step "started yesterday, split session at midnight"+      txndescs [clockin (mktime yesterday "23:00:00") "" ""] @?= ["23:00-23:59","00:00-"++nowstr]+      step "split multi-day sessions at each midnight"+      txndescs [clockin (mktime (addDays (-2) today) "23:00:00") "" ""] @?= ["23:00-23:59","00:00-23:59","00:00-"++nowstr]+      step "auto-clock-out if needed"+      txndescs [clockin (mktime today "00:00:00") "" ""] @?= ["00:00-"++nowstr]+      step "use the clockin time for auto-clockout if it's in the future"+      txndescs [clockin future "" ""] @?= [printf "%s-%s" futurestr futurestr]  ]

@@ -30,6 +30,9 @@   isTransactionBalanced,   balanceTransaction,   balanceTransactionHelper,+  transactionTransformPostings,+  transactionApplyValuation,+  transactionToCost,   -- nonzerobalanceerror,   -- * date operations   transactionDate2,@@ -41,6 +44,7 @@   -- payeeAndNoteFromDescription,   -- * rendering   showTransaction,+  showTransactionOneLineAmounts,   showTransactionUnelided,   showTransactionUnelidedOneLineAmounts,   -- showPostingLine,@@ -60,13 +64,14 @@ import qualified Data.Text as T import Data.Time.Calendar import Text.Printf-import qualified Data.Map as Map+import qualified Data.Map as M  import Hledger.Utils import Hledger.Data.Types import Hledger.Data.Dates import Hledger.Data.Posting import Hledger.Data.Amount+import Hledger.Data.Valuation  sourceFilePath :: GenericSourcePos -> FilePath sourceFilePath = \case@@ -142,40 +147,29 @@ To facilitate this, postings with explicit multi-commodity amounts are displayed as multiple similar postings, one per commodity. (Normally does not happen with this function).--If there are multiple postings, all with explicit amounts,-and the transaction appears obviously balanced-(postings sum to 0, without needing to infer conversion prices),-the last posting's amount will not be shown. -}--- XXX why that logic ?--- XXX where is/should this be still used ?--- XXX rename these, after amount expressions/mixed posting amounts lands---     eg showTransactionSimpleAmountsElidingLast, showTransactionSimpleAmounts, showTransaction showTransaction :: Transaction -> String-showTransaction = showTransactionHelper True False+showTransaction = showTransactionHelper False --- | Like showTransaction, but does not change amounts' explicitness.--- Explicit amounts are shown and implicit amounts are not.--- The output will be parseable journal syntax.--- To facilitate this, postings with explicit multi-commodity amounts--- are displayed as multiple similar postings, one per commodity.--- Most often, this is the one you want to use.+-- | Deprecated alias for 'showTransaction' showTransactionUnelided :: Transaction -> String-showTransactionUnelided = showTransactionHelper False False+showTransactionUnelided = showTransaction  -- TODO: drop it --- | Like showTransactionUnelided, but explicit multi-commodity amounts+-- | Like showTransaction, but explicit multi-commodity amounts -- are shown on one line, comma-separated. In this case the output will -- not be parseable journal syntax.-showTransactionUnelidedOneLineAmounts :: Transaction -> String-showTransactionUnelidedOneLineAmounts = showTransactionHelper False True+showTransactionOneLineAmounts :: Transaction -> String+showTransactionOneLineAmounts = showTransactionHelper True +-- | Deprecated alias for 'showTransactionOneLineAmounts'+showTransactionUnelidedOneLineAmounts = showTransactionOneLineAmounts  -- TODO: drop it+ -- | Helper for showTransaction*.-showTransactionHelper :: Bool -> Bool -> Transaction -> String-showTransactionHelper elide onelineamounts t =+showTransactionHelper :: Bool -> Transaction -> String+showTransactionHelper onelineamounts t =     unlines $ [descriptionline]               ++ newlinecomments-              ++ (postingsAsLines elide onelineamounts t (tpostings t))+              ++ (postingsAsLines onelineamounts (tpostings t))               ++ [""]     where       descriptionline = rstrip $ concat [date, status, code, desc, samelinecomment]@@ -207,10 +201,6 @@ -- -- Explicit amounts are shown, any implicit amounts are not. ----- Setting elide to true forces the last posting's amount to be implicit, if:--- there are other postings, all with explicit amounts, and the transaction--- appears balanced.--- -- Postings with multicommodity explicit amounts are handled as follows: -- if onelineamounts is true, these amounts are shown on one line, -- comma-separated, and the output will not be valid journal syntax.@@ -223,11 +213,8 @@ -- Posting amounts will be aligned with each other, starting about 4 columns -- beyond the widest account name (see postingAsLines for details). ---postingsAsLines :: Bool -> Bool -> Transaction -> [Posting] -> [String]-postingsAsLines elide onelineamounts t ps-  | elide && length ps > 1 && all hasAmount ps && isTransactionBalanced Nothing t -- imprecise balanced check-   = concatMap (postingAsLines False onelineamounts ps) (init ps) ++ postingAsLines True onelineamounts ps (last ps)-  | otherwise = concatMap (postingAsLines False onelineamounts ps) ps+postingsAsLines :: Bool -> [Posting] -> [String]+postingsAsLines onelineamounts ps = concatMap (postingAsLines False onelineamounts ps) ps  -- | Render one posting, on one or more lines, suitable for `print` output. -- There will be an indented account name, plus one or more of status flag,@@ -356,7 +343,7 @@ -- and summing the real postings, and summing the balanced virtual postings; -- and applying the given display styles if any (maybe affecting decimal places); -- do both totals appear to be zero when rendered ?-isTransactionBalanced :: Maybe (Map.Map CommoditySymbol AmountStyle) -> Transaction -> Bool+isTransactionBalanced :: Maybe (M.Map CommoditySymbol AmountStyle) -> Transaction -> Bool isTransactionBalanced styles t =     -- isReallyZeroMixedAmountCost rsum && isReallyZeroMixedAmountCost bvsum     isZeroMixedAmount rsum' && isZeroMixedAmount bvsum'@@ -380,7 +367,7 @@ -- if provided, so that the result agrees with the numbers users can see. -- balanceTransaction ::-     Maybe (Map.Map CommoditySymbol AmountStyle)  -- ^ commodity display styles+     Maybe (M.Map CommoditySymbol AmountStyle)  -- ^ commodity display styles   -> Transaction   -> Either String Transaction balanceTransaction mstyles = fmap fst . balanceTransactionHelper mstyles@@ -389,12 +376,12 @@ -- use one of those instead. It also returns a list of accounts -- and amounts that were inferred. balanceTransactionHelper ::-     Maybe (Map.Map CommoditySymbol AmountStyle)  -- ^ commodity display styles+     Maybe (M.Map CommoditySymbol AmountStyle)  -- ^ commodity display styles   -> Transaction   -> Either String (Transaction, [(AccountName, MixedAmount)]) balanceTransactionHelper mstyles t = do   (t', inferredamtsandaccts) <--    inferBalancingAmount (fromMaybe Map.empty mstyles) $ inferBalancingPrices t+    inferBalancingAmount (fromMaybe M.empty mstyles) $ inferBalancingPrices t   if isTransactionBalanced mstyles t'   then Right (txnTieKnot t', inferredamtsandaccts)   else Left $ annotateErrorWithTransaction t' $ nonzerobalanceerror t'@@ -413,7 +400,7 @@           sep = if not (null rmsg) && not (null bvmsg) then "; " else "" :: String  annotateErrorWithTransaction :: Transaction -> String -> String-annotateErrorWithTransaction t s = intercalate "\n" [showGenericSourcePos $ tsourcepos t, s, showTransactionUnelided t]+annotateErrorWithTransaction t s = intercalate "\n" [showGenericSourcePos $ tsourcepos t, s, showTransaction t]  -- | Infer up to one missing amount for this transactions's real postings, and -- likewise for its balanced virtual postings, if needed; or return an error@@ -424,7 +411,7 @@ -- one of them is amountless. If the amounts had price(s) the inferred amount -- have the same price(s), and will be converted to the price commodity. inferBalancingAmount ::-     Map.Map CommoditySymbol AmountStyle -- ^ commodity display styles+     M.Map CommoditySymbol AmountStyle -- ^ commodity display styles   -> Transaction   -> Either String (Transaction, [(AccountName, MixedAmount)]) inferBalancingAmount styles t@Transaction{tpostings=ps}@@ -553,48 +540,31 @@ postingSetTransaction :: Transaction -> Posting -> Posting postingSetTransaction t p = p{ptransaction=Just t} +-- | Apply a transform function to this transaction's amounts.+transactionTransformPostings :: (Posting -> Posting) -> Transaction -> Transaction+transactionTransformPostings f t@Transaction{tpostings=ps} = t{tpostings=map f ps}++-- | Apply a specified valuation to this transaction's amounts, using+-- the provided price oracle, commodity styles, reference dates, and+-- whether this is for a multiperiod report or not. See+-- amountApplyValuation.+transactionApplyValuation :: PriceOracle -> M.Map CommoditySymbol AmountStyle -> Day -> Maybe Day -> Day -> Bool -> Transaction -> ValuationType -> Transaction+transactionApplyValuation priceoracle styles periodlast mreportlast today ismultiperiod t v =+  transactionTransformPostings (\p -> postingApplyValuation priceoracle styles periodlast mreportlast today ismultiperiod p v) t++-- | Convert this transaction's amounts to cost, and apply the appropriate amount styles.+transactionToCost :: M.Map CommoditySymbol AmountStyle -> Transaction -> Transaction+transactionToCost styles t@Transaction{tpostings=ps} = t{tpostings=map (postingToCost styles) ps}+ -- tests  tests_Transaction =-  tests-    "Transaction"-    [ tests-        "showTransactionUnelided"-        [ showTransactionUnelided nulltransaction `is` "0000/01/01\n\n"-        , showTransactionUnelided-            nulltransaction-              { tdate = parsedate "2012/05/14"-              , tdate2 = Just $ parsedate "2012/05/15"-              , tstatus = Unmarked-              , tcode = "code"-              , tdescription = "desc"-              , tcomment = "tcomment1\ntcomment2\n"-              , ttags = [("ttag1", "val1")]-              , tpostings =-                  [ nullposting-                      { pstatus = Cleared-                      , paccount = "a"-                      , pamount = Mixed [usd 1, hrs 2]-                      , pcomment = "\npcomment2\n"-                      , ptype = RegularPosting-                      , ptags = [("ptag1", "val1"), ("ptag2", "val2")]-                      }-                  ]-              } `is`-          unlines-            [ "2012/05/14=2012/05/15 (code) desc  ; tcomment1"-            , "    ; tcomment2"-            , "    * a         $1.00"-            , "    ; pcomment2"-            , "    * a         2.00h"-            , "    ; pcomment2"-            , ""-            ]-        ]-    , tests-        "postingAsLines"-        [ postingAsLines False False [posting] posting `is` [""]-        , let p =+  tests "Transaction" [++      tests "postingAsLines" [+          test "null posting" $ postingAsLines False False [posting] posting @?= [""]+        , test "non-null posting" $+           let p =                 posting                   { pstatus = Cleared                   , paccount = "a"@@ -603,7 +573,7 @@                   , ptype = RegularPosting                   , ptags = [("ptag1", "val1"), ("ptag2", "val2")]                   }-           in postingAsLines False False [p] p `is`+           in postingAsLines False False [p] p @?=               [ "    * a         $1.00  ; pcomment1"               , "    ; pcomment2"               , "    ;   tag3: val3  "@@ -612,106 +582,90 @@               , "    ;   tag3: val3  "               ]         ]-   -- postingsAsLines-    -- one implicit amount-    , let timp = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` missingamt]}-    -- explicit amounts, balanced-          texp = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` usd (-1)]}-    -- explicit amount, only one posting-          texp1 = nulltransaction {tpostings = ["(a)" `post` usd 1]}-    -- explicit amounts, two commodities, explicit balancing price-          texp2 = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` (hrs (-1) `at` usd 1)]}-    -- explicit amounts, two commodities, implicit balancing price-          texp2b = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` hrs (-1)]}-    -- one missing amount, not the last one-          t3 = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` missingamt, "c" `post` usd (-1)]}-    -- unbalanced amounts when precision is limited (#931)-          t4 = nulltransaction {tpostings = ["a" `post` usd (-0.01), "b" `post` usd (0.005), "c" `post` usd (0.005)]}-       in tests-            "postingsAsLines"-            [ test "null-transaction" $-              let t = nulltransaction-               in postingsAsLines True False t (tpostings t) `is` []-            , test "implicit-amount-elide-false" $-              let t = timp-               in postingsAsLines False False t (tpostings t) `is`-                  [ "    a           $1.00"-                  , "    b" -- implicit amount remains implicit-                  ]-            , test "implicit-amount-elide-true" $-              let t = timp-               in postingsAsLines True False t (tpostings t) `is`++    , let+        -- one implicit amount+        timp = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` missingamt]}+        -- explicit amounts, balanced+        texp = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` usd (-1)]}+        -- explicit amount, only one posting+        texp1 = nulltransaction {tpostings = ["(a)" `post` usd 1]}+        -- explicit amounts, two commodities, explicit balancing price+        texp2 = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` (hrs (-1) `at` usd 1)]}+        -- explicit amounts, two commodities, implicit balancing price+        texp2b = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` hrs (-1)]}+        -- one missing amount, not the last one+        t3 = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` missingamt, "c" `post` usd (-1)]}+        -- unbalanced amounts when precision is limited (#931)+        -- t4 = nulltransaction {tpostings = ["a" `post` usd (-0.01), "b" `post` usd (0.005), "c" `post` usd (0.005)]}+      in tests "postingsAsLines" [+              test "null-transaction" $ postingsAsLines False (tpostings nulltransaction) @?= []+            , test "implicit-amount" $ postingsAsLines False (tpostings timp) @?=                   [ "    a           $1.00"                   , "    b" -- implicit amount remains implicit                   ]-            , test "explicit-amounts-elide-false" $-              let t = texp-               in postingsAsLines False False t (tpostings t) `is`-                  [ "    a           $1.00"-                  , "    b          $-1.00" -- both amounts remain explicit-                  ]-            , test "explicit-amounts-elide-true" $-              let t = texp-               in postingsAsLines True False t (tpostings t) `is`+            , test "explicit-amounts" $ postingsAsLines False (tpostings texp) @?=                   [ "    a           $1.00"-                  , "    b" -- explicit amount is made implicit+                  , "    b          $-1.00"                   ]-            , test "one-explicit-amount-elide-true" $-              let t = texp1-               in postingsAsLines True False t (tpostings t) `is`-                  [ "    (a)           $1.00" -- explicit amount remains explicit since only one posting+            , test "one-explicit-amount" $ postingsAsLines False (tpostings texp1) @?=+                  [ "    (a)           $1.00"                   ]-            , test "explicit-amounts-two-commodities-elide-true" $-              let t = texp2-               in postingsAsLines True False t (tpostings t) `is`+            , test "explicit-amounts-two-commodities" $ postingsAsLines False (tpostings texp2) @?=                   [ "    a             $1.00"-                  , "    b" -- explicit amount is made implicit since txn is explicitly balanced+                  , "    b    -1.00h @ $1.00"                   ]-            , test "explicit-amounts-not-explicitly-balanced-elide-true" $-              let t = texp2b-               in postingsAsLines True False t (tpostings t) `is`+            , test "explicit-amounts-not-explicitly-balanced" $ postingsAsLines False (tpostings texp2b) @?=                   [ "    a           $1.00"-                  , "    b          -1.00h" -- explicit amount remains explicit since a conversion price would have be inferred to balance+                  , "    b          -1.00h"                   ]-            , test "implicit-amount-not-last" $-              let t = t3-               in postingsAsLines True False t (tpostings t) `is`+            , test "implicit-amount-not-last" $ postingsAsLines False (tpostings t3) @?=                   ["    a           $1.00", "    b", "    c          $-1.00"]-            , _test "ensure-visibly-balanced" $-              let t = t4-               in postingsAsLines False False t (tpostings t) `is`-                  ["    a          $-0.01", "    b           $0.005", "    c           $0.005"]+            -- , test "ensure-visibly-balanced" $+            --    in postingsAsLines False (tpostings t4) @?=+            --       ["    a          $-0.01", "    b           $0.005", "    c           $0.005"]+             ]-    , tests-         "inferBalancingAmount"-         [ (fst <$> inferBalancingAmount Map.empty nulltransaction) `is` Right nulltransaction-         , (fst <$> inferBalancingAmount Map.empty nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` missingamt]}) `is`++    , test "inferBalancingAmount" $ do+         (fst <$> inferBalancingAmount M.empty nulltransaction) @?= Right nulltransaction+         (fst <$> inferBalancingAmount M.empty nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` missingamt]}) @?=            Right nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` usd 5]}-         , (fst <$> inferBalancingAmount Map.empty nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` (eur 3 @@ usd 4), "c" `post` missingamt]}) `is`+         (fst <$> inferBalancingAmount M.empty nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` (eur 3 @@ usd 4), "c" `post` missingamt]}) @?=            Right nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` (eur 3 @@ usd 4), "c" `post` usd 1]}-         ]-    , tests-        "showTransaction"-        [ test "show a balanced transaction, eliding last amount" $-          let t =-                Transaction-                  0-                  ""-                  nullsourcepos-                  (parsedate "2007/01/28")-                  Nothing-                  Unmarked-                  ""-                  "coopportunity"-                  ""-                  []-                  [ posting {paccount = "expenses:food:groceries", pamount = Mixed [usd 47.18], ptransaction = Just t}-                  , posting {paccount = "assets:checking", pamount = Mixed [usd (-47.18)], ptransaction = Just t}+         +    , tests "showTransaction" [+          test "null transaction" $ showTransaction nulltransaction @?= "0000/01/01\n\n"+        , test "non-null transaction" $ showTransaction+            nulltransaction+              { tdate = parsedate "2012/05/14"+              , tdate2 = Just $ parsedate "2012/05/15"+              , tstatus = Unmarked+              , tcode = "code"+              , tdescription = "desc"+              , tcomment = "tcomment1\ntcomment2\n"+              , ttags = [("ttag1", "val1")]+              , tpostings =+                  [ nullposting+                      { pstatus = Cleared+                      , paccount = "a"+                      , pamount = Mixed [usd 1, hrs 2]+                      , pcomment = "\npcomment2\n"+                      , ptype = RegularPosting+                      , ptags = [("ptag1", "val1"), ("ptag2", "val2")]+                      }                   ]-           in showTransaction t `is`-              unlines-                ["2007/01/28 coopportunity", "    expenses:food:groceries          $47.18", "    assets:checking", ""]-        , test "show a balanced transaction, no eliding" $+              } @?=+          unlines+            [ "2012/05/14=2012/05/15 (code) desc  ; tcomment1"+            , "    ; tcomment2"+            , "    * a         $1.00"+            , "    ; pcomment2"+            , "    * a         2.00h"+            , "    ; pcomment2"+            , ""+            ]+        , test "show a balanced transaction" $           (let t =                  Transaction                    0@@ -727,14 +681,13 @@                    [ posting {paccount = "expenses:food:groceries", pamount = Mixed [usd 47.18], ptransaction = Just t}                    , posting {paccount = "assets:checking", pamount = Mixed [usd (-47.18)], ptransaction = Just t}                    ]-            in showTransactionUnelided t) `is`+            in showTransaction t) @?=           (unlines              [ "2007/01/28 coopportunity"              , "    expenses:food:groceries          $47.18"              , "    assets:checking                 $-47.18"              , ""              ])-     -- document some cases that arise in debug/testing:         , test "show an unbalanced transaction, should not elide" $           (showTransaction              (txnTieKnot $@@ -751,29 +704,13 @@                 []                 [ posting {paccount = "expenses:food:groceries", pamount = Mixed [usd 47.18]}                 , posting {paccount = "assets:checking", pamount = Mixed [usd (-47.19)]}-                ])) `is`+                ])) @?=           (unlines              [ "2007/01/28 coopportunity"              , "    expenses:food:groceries          $47.18"              , "    assets:checking                 $-47.19"              , ""              ])-        , test "show an unbalanced transaction with one posting, should not elide" $-          (showTransaction-             (txnTieKnot $-              Transaction-                0-                ""-                nullsourcepos-                (parsedate "2007/01/28")-                Nothing-                Unmarked-                ""-                "coopportunity"-                ""-                []-                [posting {paccount = "expenses:food:groceries", pamount = Mixed [usd 47.18]}])) `is`-          (unlines ["2007/01/28 coopportunity", "    expenses:food:groceries          $47.18", ""])         , test "show a transaction with one posting and a missing amount" $           (showTransaction              (txnTieKnot $@@ -788,7 +725,7 @@                 "coopportunity"                 ""                 []-                [posting {paccount = "expenses:food:groceries", pamount = missingmixedamt}])) `is`+                [posting {paccount = "expenses:food:groceries", pamount = missingmixedamt}])) @?=           (unlines ["2007/01/28 coopportunity", "    expenses:food:groceries", ""])         , test "show a transaction with a priced commodityless amount" $           (showTransaction@@ -806,13 +743,12 @@                 []                 [ posting {paccount = "a", pamount = Mixed [num 1 `at` (usd 2 `withPrecision` 0)]}                 , posting {paccount = "b", pamount = missingmixedamt}-                ])) `is`+                ])) @?=           (unlines ["2010/01/01 x", "    a          1 @ $2", "    b", ""])         ]-    , tests-        "balanceTransaction"-        [ test "detect unbalanced entry, sign error" $-          expectLeft+    , tests "balanceTransaction" [+         test "detect unbalanced entry, sign error" $+          assertLeft             (balanceTransaction                Nothing                (Transaction@@ -827,8 +763,8 @@                   ""                   []                   [posting {paccount = "a", pamount = Mixed [usd 1]}, posting {paccount = "b", pamount = Mixed [usd 1]}]))-        , test "detect unbalanced entry, multiple missing amounts" $-          expectLeft $+        ,test "detect unbalanced entry, multiple missing amounts" $+          assertLeft $              balanceTransaction                Nothing                (Transaction@@ -845,7 +781,7 @@                   [ posting {paccount = "a", pamount = missingmixedamt}                   , posting {paccount = "b", pamount = missingmixedamt}                   ])-        , test "one missing amount is inferred" $+        ,test "one missing amount is inferred" $           (pamount . last . tpostings <$>            balanceTransaction              Nothing@@ -860,9 +796,9 @@                 ""                 ""                 []-                [posting {paccount = "a", pamount = Mixed [usd 1]}, posting {paccount = "b", pamount = missingmixedamt}])) `is`+                [posting {paccount = "a", pamount = Mixed [usd 1]}, posting {paccount = "b", pamount = missingmixedamt}])) @?=           Right (Mixed [usd (-1)])-        , test "conversion price is inferred" $+        ,test "conversion price is inferred" $           (pamount . head . tpostings <$>            balanceTransaction              Nothing@@ -879,10 +815,10 @@                 []                 [ posting {paccount = "a", pamount = Mixed [usd 1.35]}                 , posting {paccount = "b", pamount = Mixed [eur (-1)]}-                ])) `is`+                ])) @?=           Right (Mixed [usd 1.35 @@ (eur 1 `withPrecision` maxprecision)])-        , test "balanceTransaction balances based on cost if there are unit prices" $-          expectRight $+        ,test "balanceTransaction balances based on cost if there are unit prices" $+          assertRight $           balanceTransaction             Nothing             (Transaction@@ -899,8 +835,8 @@                [ posting {paccount = "a", pamount = Mixed [usd 1 `at` eur 2]}                , posting {paccount = "a", pamount = Mixed [usd (-2) `at` eur 1]}                ])-        , test "balanceTransaction balances based on cost if there are total prices" $-          expectRight $+        ,test "balanceTransaction balances based on cost if there are total prices" $+          assertRight $           balanceTransaction             Nothing             (Transaction@@ -918,10 +854,9 @@                , posting {paccount = "a", pamount = Mixed [usd (-2) @@ eur 1]}                ])         ]-    , tests-        "isTransactionBalanced"-        [ test "detect balanced" $-          expect $+    , tests "isTransactionBalanced" [+         test "detect balanced" $+          assertBool "" $           isTransactionBalanced Nothing $           Transaction             0@@ -937,8 +872,8 @@             [ posting {paccount = "b", pamount = Mixed [usd 1.00]}             , posting {paccount = "c", pamount = Mixed [usd (-1.00)]}             ]-        , test "detect unbalanced" $-          expect $+        ,test "detect unbalanced" $+          assertBool "" $           not $           isTransactionBalanced Nothing $           Transaction@@ -955,8 +890,8 @@             [ posting {paccount = "b", pamount = Mixed [usd 1.00]}             , posting {paccount = "c", pamount = Mixed [usd (-1.01)]}             ]-        , test "detect unbalanced, one posting" $-          expect $+        ,test "detect unbalanced, one posting" $+          assertBool "" $           not $           isTransactionBalanced Nothing $           Transaction@@ -971,8 +906,8 @@             ""             []             [posting {paccount = "b", pamount = Mixed [usd 1.00]}]-        , test "one zero posting is considered balanced for now" $-          expect $+        ,test "one zero posting is considered balanced for now" $+          assertBool "" $           isTransactionBalanced Nothing $           Transaction             0@@ -986,8 +921,8 @@             ""             []             [posting {paccount = "b", pamount = Mixed [usd 0]}]-        , test "virtual postings don't need to balance" $-          expect $+        ,test "virtual postings don't need to balance" $+          assertBool "" $           isTransactionBalanced Nothing $           Transaction             0@@ -1004,8 +939,8 @@             , posting {paccount = "c", pamount = Mixed [usd (-1.00)]}             , posting {paccount = "d", pamount = Mixed [usd 100], ptype = VirtualPosting}             ]-        , test "balanced virtual postings need to balance among themselves" $-          expect $+        ,test "balanced virtual postings need to balance among themselves" $+          assertBool "" $           not $           isTransactionBalanced Nothing $           Transaction@@ -1023,8 +958,8 @@             , posting {paccount = "c", pamount = Mixed [usd (-1.00)]}             , posting {paccount = "d", pamount = Mixed [usd 100], ptype = BalancedVirtualPosting}             ]-        , test "balanced virtual postings need to balance among themselves (2)" $-          expect $+        ,test "balanced virtual postings need to balance among themselves (2)" $+          assertBool "" $           isTransactionBalanced Nothing $           Transaction             0

@@ -57,7 +57,7 @@ -- >>> putStr $ showTransaction $ transactionModifierToFunction (TransactionModifier "" ["pong" `post` usd 2]) nulltransaction{tpostings=["ping" `post` usd 1]} -- 0000/01/01 --     ping           $1.00---     pong           $2.00  ; generated-posting:=+--     pong           $2.00  ; generated-posting: = -- <BLANKLINE> -- >>> putStr $ showTransaction $ transactionModifierToFunction (TransactionModifier "miss" ["pong" `post` usd 2]) nulltransaction{tpostings=["ping" `post` usd 1]} -- 0000/01/01@@ -66,7 +66,7 @@ -- >>> putStr $ showTransaction $ transactionModifierToFunction (TransactionModifier "ping" ["pong" `post` amount{aismultiplier=True, aquantity=3}]) nulltransaction{tpostings=["ping" `post` usd 2]} -- 0000/01/01 --     ping           $2.00---     pong           $6.00  ; generated-posting:= ping+--     pong           $6.00  ; generated-posting: = ping -- <BLANKLINE> -- transactionModifierToFunction :: TransactionModifier -> (Transaction -> Transaction)

@@ -48,11 +48,6 @@ import Hledger.Data.Dates (parsedate)  -tests_Valuation = tests "Valuation" [-   tests_priceLookup-  ]-- ------------------------------------------------------------------------------ -- Types @@ -278,12 +273,11 @@       ,p "2001/01/01" "A" 11 "B"       ]     pricesatdate = pricesAtDate ps1-  in tests "priceLookup" [-     priceLookup pricesatdate (d "1999/01/01") "A" Nothing    `is` Nothing-    ,priceLookup pricesatdate (d "2000/01/01") "A" Nothing    `is` Just ("B",10)-    ,priceLookup pricesatdate (d "2000/01/01") "B" (Just "A") `is` Just ("A",0.1)-    ,priceLookup pricesatdate (d "2000/01/01") "A" (Just "E") `is` Just ("E",500)-    ]+  in test "priceLookup" $ do+    priceLookup pricesatdate (d "1999/01/01") "A" Nothing    @?= Nothing+    priceLookup pricesatdate (d "2000/01/01") "A" Nothing    @?= Just ("B",10)+    priceLookup pricesatdate (d "2000/01/01") "B" (Just "A") @?= Just ("A",0.1)+    priceLookup pricesatdate (d "2000/01/01") "A" (Just "E") @?= Just ("E",500)  ------------------------------------------------------------------------------ -- Building the price graph (network of commodity conversions) on a given day.@@ -365,3 +359,7 @@ nodesEdgeLabel g (from,to) = headMay $ sort [l | (_,t,l) <- out g from, t==to]  ------------------------------------------------------------------------------++tests_Valuation = tests "Valuation" [+   tests_priceLookup+  ]

@@ -653,130 +653,122 @@ -- tests  tests_Query = tests "Query" [-   tests "simplifyQuery" [--     (simplifyQuery $ Or [Acct "a"])      `is` (Acct "a")-    ,(simplifyQuery $ Or [Any,None])      `is` (Any)-    ,(simplifyQuery $ And [Any,None])     `is` (None)-    ,(simplifyQuery $ And [Any,Any])      `is` (Any)-    ,(simplifyQuery $ And [Acct "b",Any]) `is` (Acct "b")-    ,(simplifyQuery $ And [Any,And [Date (DateSpan Nothing Nothing)]]) `is` (Any)-    ,(simplifyQuery $ And [Date (DateSpan Nothing (Just $ parsedate "2013-01-01")), Date (DateSpan (Just $ parsedate "2012-01-01") Nothing)])-      `is` (Date (DateSpan (Just $ parsedate "2012-01-01") (Just $ parsedate "2013-01-01")))-    ,(simplifyQuery $ And [Or [],Or [Desc "b b"]]) `is` (Desc "b b")-   ]+   test "simplifyQuery" $ do+     (simplifyQuery $ Or [Acct "a"])      @?= (Acct "a")+     (simplifyQuery $ Or [Any,None])      @?= (Any)+     (simplifyQuery $ And [Any,None])     @?= (None)+     (simplifyQuery $ And [Any,Any])      @?= (Any)+     (simplifyQuery $ And [Acct "b",Any]) @?= (Acct "b")+     (simplifyQuery $ And [Any,And [Date (DateSpan Nothing Nothing)]]) @?= (Any)+     (simplifyQuery $ And [Date (DateSpan Nothing (Just $ parsedate "2013-01-01")), Date (DateSpan (Just $ parsedate "2012-01-01") Nothing)])+       @?= (Date (DateSpan (Just $ parsedate "2012-01-01") (Just $ parsedate "2013-01-01")))+     (simplifyQuery $ And [Or [],Or [Desc "b b"]]) @?= (Desc "b b") -  ,tests "parseQuery" [-     (parseQuery nulldate "acct:'expenses:autres d\233penses' desc:b") `is` (And [Acct "expenses:autres d\233penses", Desc "b"], [])-    ,parseQuery nulldate "inacct:a desc:\"b b\""                     `is` (Desc "b b", [QueryOptInAcct "a"])-    ,parseQuery nulldate "inacct:a inacct:b"                         `is` (Any, [QueryOptInAcct "a", QueryOptInAcct "b"])-    ,parseQuery nulldate "desc:'x x'"                                `is` (Desc "x x", [])-    ,parseQuery nulldate "'a a' 'b"                                  `is` (Or [Acct "a a",Acct "'b"], [])-    ,parseQuery nulldate "\""                                        `is` (Acct "\"", [])-   ]+  ,test "parseQuery" $ do+     (parseQuery nulldate "acct:'expenses:autres d\233penses' desc:b") @?= (And [Acct "expenses:autres d\233penses", Desc "b"], [])+     parseQuery nulldate "inacct:a desc:\"b b\""                     @?= (Desc "b b", [QueryOptInAcct "a"])+     parseQuery nulldate "inacct:a inacct:b"                         @?= (Any, [QueryOptInAcct "a", QueryOptInAcct "b"])+     parseQuery nulldate "desc:'x x'"                                @?= (Desc "x x", [])+     parseQuery nulldate "'a a' 'b"                                  @?= (Or [Acct "a a",Acct "'b"], [])+     parseQuery nulldate "\""                                        @?= (Acct "\"", []) -  ,tests "words''" [-      (words'' [] "a b")                   `is` ["a","b"]-    , (words'' [] "'a b'")                 `is` ["a b"]-    , (words'' [] "not:a b")               `is` ["not:a","b"]-    , (words'' [] "not:'a b'")             `is` ["not:a b"]-    , (words'' [] "'not:a b'")             `is` ["not:a b"]-    , (words'' ["desc:"] "not:desc:'a b'") `is` ["not:desc:a b"]-    , (words'' prefixes "\"acct:expenses:autres d\233penses\"") `is` ["acct:expenses:autres d\233penses"]-    , (words'' prefixes "\"")              `is` ["\""]-    ]+  ,test "words''" $ do+      (words'' [] "a b")                   @?= ["a","b"]+      (words'' [] "'a b'")                 @?= ["a b"]+      (words'' [] "not:a b")               @?= ["not:a","b"]+      (words'' [] "not:'a b'")             @?= ["not:a b"]+      (words'' [] "'not:a b'")             @?= ["not:a b"]+      (words'' ["desc:"] "not:desc:'a b'") @?= ["not:desc:a b"]+      (words'' prefixes "\"acct:expenses:autres d\233penses\"") @?= ["acct:expenses:autres d\233penses"]+      (words'' prefixes "\"")              @?= ["\""] -  ,tests "filterQuery" [-     filterQuery queryIsDepth Any       `is` Any-    ,filterQuery queryIsDepth (Depth 1) `is` Depth 1-    ,filterQuery (not.queryIsDepth) (And [And [StatusQ Cleared,Depth 1]]) `is` StatusQ Cleared-    ,filterQuery queryIsDepth (And [Date nulldatespan, Not (Or [Any, Depth 1])]) `is` Any   -- XXX unclear-   ]+  ,test "filterQuery" $ do+     filterQuery queryIsDepth Any       @?= Any+     filterQuery queryIsDepth (Depth 1) @?= Depth 1+     filterQuery (not.queryIsDepth) (And [And [StatusQ Cleared,Depth 1]]) @?= StatusQ Cleared+     filterQuery queryIsDepth (And [Date nulldatespan, Not (Or [Any, Depth 1])]) @?= Any   -- XXX unclear -  ,tests "parseQueryTerm" [-     parseQueryTerm nulldate "a"                                `is` (Left $ Acct "a")-    ,parseQueryTerm nulldate "acct:expenses:autres d\233penses" `is` (Left $ Acct "expenses:autres d\233penses")-    ,parseQueryTerm nulldate "not:desc:a b"                     `is` (Left $ Not $ Desc "a b")-    ,parseQueryTerm nulldate "status:1"                         `is` (Left $ StatusQ Cleared)-    ,parseQueryTerm nulldate "status:*"                         `is` (Left $ StatusQ Cleared)-    ,parseQueryTerm nulldate "status:!"                         `is` (Left $ StatusQ Pending)-    ,parseQueryTerm nulldate "status:0"                         `is` (Left $ StatusQ Unmarked)-    ,parseQueryTerm nulldate "status:"                          `is` (Left $ StatusQ Unmarked)-    ,parseQueryTerm nulldate "payee:x"                          `is` (Left $ Tag "payee" (Just "x"))-    ,parseQueryTerm nulldate "note:x"                           `is` (Left $ Tag "note" (Just "x"))-    ,parseQueryTerm nulldate "real:1"                           `is` (Left $ Real True)-    ,parseQueryTerm nulldate "date:2008"                        `is` (Left $ Date $ DateSpan (Just $ parsedate "2008/01/01") (Just $ parsedate "2009/01/01"))-    ,parseQueryTerm nulldate "date:from 2012/5/17"              `is` (Left $ Date $ DateSpan (Just $ parsedate "2012/05/17") Nothing)-    ,parseQueryTerm nulldate "date:20180101-201804"             `is` (Left $ Date $ DateSpan (Just $ parsedate "2018/01/01") (Just $ parsedate "2018/04/01"))-    ,parseQueryTerm nulldate "inacct:a"                         `is` (Right $ QueryOptInAcct "a")-    ,parseQueryTerm nulldate "tag:a"                            `is` (Left $ Tag "a" Nothing)-    ,parseQueryTerm nulldate "tag:a=some value"                 `is` (Left $ Tag "a" (Just "some value"))-    ,parseQueryTerm nulldate "amt:<0"                           `is` (Left $ Amt Lt 0)-    ,parseQueryTerm nulldate "amt:>10000.10"                    `is` (Left $ Amt AbsGt 10000.1)-   ]+  ,test "parseQueryTerm" $ do+     parseQueryTerm nulldate "a"                                @?= (Left $ Acct "a")+     parseQueryTerm nulldate "acct:expenses:autres d\233penses" @?= (Left $ Acct "expenses:autres d\233penses")+     parseQueryTerm nulldate "not:desc:a b"                     @?= (Left $ Not $ Desc "a b")+     parseQueryTerm nulldate "status:1"                         @?= (Left $ StatusQ Cleared)+     parseQueryTerm nulldate "status:*"                         @?= (Left $ StatusQ Cleared)+     parseQueryTerm nulldate "status:!"                         @?= (Left $ StatusQ Pending)+     parseQueryTerm nulldate "status:0"                         @?= (Left $ StatusQ Unmarked)+     parseQueryTerm nulldate "status:"                          @?= (Left $ StatusQ Unmarked)+     parseQueryTerm nulldate "payee:x"                          @?= (Left $ Tag "payee" (Just "x"))+     parseQueryTerm nulldate "note:x"                           @?= (Left $ Tag "note" (Just "x"))+     parseQueryTerm nulldate "real:1"                           @?= (Left $ Real True)+     parseQueryTerm nulldate "date:2008"                        @?= (Left $ Date $ DateSpan (Just $ parsedate "2008/01/01") (Just $ parsedate "2009/01/01"))+     parseQueryTerm nulldate "date:from 2012/5/17"              @?= (Left $ Date $ DateSpan (Just $ parsedate "2012/05/17") Nothing)+     parseQueryTerm nulldate "date:20180101-201804"             @?= (Left $ Date $ DateSpan (Just $ parsedate "2018/01/01") (Just $ parsedate "2018/04/01"))+     parseQueryTerm nulldate "inacct:a"                         @?= (Right $ QueryOptInAcct "a")+     parseQueryTerm nulldate "tag:a"                            @?= (Left $ Tag "a" Nothing)+     parseQueryTerm nulldate "tag:a=some value"                 @?= (Left $ Tag "a" (Just "some value"))+     parseQueryTerm nulldate "amt:<0"                           @?= (Left $ Amt Lt 0)+     parseQueryTerm nulldate "amt:>10000.10"                    @?= (Left $ Amt AbsGt 10000.1) -  ,tests "parseAmountQueryTerm" [-     parseAmountQueryTerm "<0"        `is` (Lt,0) -- special case for convenience, since AbsLt 0 would be always false-    ,parseAmountQueryTerm ">0"        `is` (Gt,0) -- special case for convenience and consistency with above-    ,parseAmountQueryTerm ">10000.10" `is` (AbsGt,10000.1)-    ,parseAmountQueryTerm "=0.23"     `is` (AbsEq,0.23)-    ,parseAmountQueryTerm "0.23"      `is` (AbsEq,0.23)-    ,parseAmountQueryTerm "<=+0.23"   `is` (LtEq,0.23)-    ,parseAmountQueryTerm "-0.23"     `is` (Eq,(-0.23))-    ,_test "number beginning with decimal mark" $ parseAmountQueryTerm "=.23" `is` (AbsEq,0.23)  -- XXX-    ]+  ,test "parseAmountQueryTerm" $ do+     parseAmountQueryTerm "<0"        @?= (Lt,0) -- special case for convenience, since AbsLt 0 would be always false+     parseAmountQueryTerm ">0"        @?= (Gt,0) -- special case for convenience and consistency with above+     parseAmountQueryTerm ">10000.10" @?= (AbsGt,10000.1)+     parseAmountQueryTerm "=0.23"     @?= (AbsEq,0.23)+     parseAmountQueryTerm "0.23"      @?= (AbsEq,0.23)+     parseAmountQueryTerm "<=+0.23"   @?= (LtEq,0.23)+     parseAmountQueryTerm "-0.23"     @?= (Eq,(-0.23))+    -- ,test "number beginning with decimal mark" $ parseAmountQueryTerm "=.23" @?= (AbsEq,0.23)  -- XXX -  ,tests "matchesAccount" [-     expect $ (Acct "b:c") `matchesAccount` "a:bb:c:d"-    ,expect $ not $ (Acct "^a:b") `matchesAccount` "c:a:b"-    ,expect $ Depth 2 `matchesAccount` "a"-    ,expect $ Depth 2 `matchesAccount` "a:b"-    ,expect $ not $ Depth 2 `matchesAccount` "a:b:c"-    ,expect $ Date nulldatespan `matchesAccount` "a"-    ,expect $ Date2 nulldatespan `matchesAccount` "a"-    ,expect $ not $ (Tag "a" Nothing) `matchesAccount` "a"-  ]+  ,test "matchesAccount" $ do+     assertBool "" $ (Acct "b:c") `matchesAccount` "a:bb:c:d"+     assertBool "" $ not $ (Acct "^a:b") `matchesAccount` "c:a:b"+     assertBool "" $ Depth 2 `matchesAccount` "a"+     assertBool "" $ Depth 2 `matchesAccount` "a:b"+     assertBool "" $ not $ Depth 2 `matchesAccount` "a:b:c"+     assertBool "" $ Date nulldatespan `matchesAccount` "a"+     assertBool "" $ Date2 nulldatespan `matchesAccount` "a"+     assertBool "" $ not $ (Tag "a" Nothing) `matchesAccount` "a"    ,tests "matchesPosting" [      test "positive match on cleared posting status"  $-      expect $ (StatusQ Cleared)  `matchesPosting` nullposting{pstatus=Cleared}+      assertBool "" $ (StatusQ Cleared)  `matchesPosting` nullposting{pstatus=Cleared}     ,test "negative match on cleared posting status"  $-      expect $ not $ (Not $ StatusQ Cleared)  `matchesPosting` nullposting{pstatus=Cleared}+      assertBool "" $ not $ (Not $ StatusQ Cleared)  `matchesPosting` nullposting{pstatus=Cleared}     ,test "positive match on unmarked posting status" $-      expect $ (StatusQ Unmarked) `matchesPosting` nullposting{pstatus=Unmarked}+      assertBool "" $ (StatusQ Unmarked) `matchesPosting` nullposting{pstatus=Unmarked}     ,test "negative match on unmarked posting status" $-      expect $ not $ (Not $ StatusQ Unmarked) `matchesPosting` nullposting{pstatus=Unmarked}+      assertBool "" $ not $ (Not $ StatusQ Unmarked) `matchesPosting` nullposting{pstatus=Unmarked}     ,test "positive match on true posting status acquired from transaction" $-      expect $ (StatusQ Cleared) `matchesPosting` nullposting{pstatus=Unmarked,ptransaction=Just nulltransaction{tstatus=Cleared}}-    ,test "real:1 on real posting" $ expect $ (Real True) `matchesPosting` nullposting{ptype=RegularPosting}-    ,test "real:1 on virtual posting fails" $ expect $ not $ (Real True) `matchesPosting` nullposting{ptype=VirtualPosting}-    ,test "real:1 on balanced virtual posting fails" $ expect $ not $ (Real True) `matchesPosting` nullposting{ptype=BalancedVirtualPosting}-    ,test "a" $ expect $ (Acct "'b") `matchesPosting` nullposting{paccount="'b"}-    ,test "b" $ expect $ not $ (Tag "a" (Just "r$")) `matchesPosting` nullposting-    ,test "c" $ expect $ (Tag "foo" Nothing) `matchesPosting` nullposting{ptags=[("foo","")]}-    ,test "d" $ expect $ (Tag "foo" Nothing) `matchesPosting` nullposting{ptags=[("foo","baz")]}-    ,test "e" $ expect $ (Tag "foo" (Just "a")) `matchesPosting` nullposting{ptags=[("foo","bar")]}-    ,test "f" $ expect $ not $ (Tag "foo" (Just "a$")) `matchesPosting` nullposting{ptags=[("foo","bar")]}-    ,test "g" $ expect $ not $ (Tag " foo " (Just "a")) `matchesPosting` nullposting{ptags=[("foo","bar")]}-    ,test "h" $ expect $ not $ (Tag "foo foo" (Just " ar ba ")) `matchesPosting` nullposting{ptags=[("foo foo","bar bar")]}-     -- a tag match on a posting also sees inherited tags-    ,test "i" $ expect $ (Tag "txntag" Nothing) `matchesPosting` nullposting{ptransaction=Just nulltransaction{ttags=[("txntag","")]}}-    ,test "j" $ expect $ not $ (Sym "$") `matchesPosting` nullposting{pamount=Mixed [usd 1]} -- becomes "^$$", ie testing for null symbol-    ,test "k" $ expect $ (Sym "\\$") `matchesPosting` nullposting{pamount=Mixed [usd 1]} -- have to quote $ for regexpr-    ,test "l" $ expect $ (Sym "shekels") `matchesPosting` nullposting{pamount=Mixed [nullamt{acommodity="shekels"}]}-    ,test "m" $ expect $ not $ (Sym "shek") `matchesPosting` nullposting{pamount=Mixed [nullamt{acommodity="shekels"}]}+      assertBool "" $ (StatusQ Cleared) `matchesPosting` nullposting{pstatus=Unmarked,ptransaction=Just nulltransaction{tstatus=Cleared}}+    ,test "real:1 on real posting" $ assertBool "" $ (Real True) `matchesPosting` nullposting{ptype=RegularPosting}+    ,test "real:1 on virtual posting fails" $ assertBool "" $ not $ (Real True) `matchesPosting` nullposting{ptype=VirtualPosting}+    ,test "real:1 on balanced virtual posting fails" $ assertBool "" $ not $ (Real True) `matchesPosting` nullposting{ptype=BalancedVirtualPosting}+    ,test "acct:" $ assertBool "" $ (Acct "'b") `matchesPosting` nullposting{paccount="'b"}+    ,test "tag:" $ do+      assertBool "" $ not $ (Tag "a" (Just "r$")) `matchesPosting` nullposting+      assertBool "" $ (Tag "foo" Nothing) `matchesPosting` nullposting{ptags=[("foo","")]}+      assertBool "" $ (Tag "foo" Nothing) `matchesPosting` nullposting{ptags=[("foo","baz")]}+      assertBool "" $ (Tag "foo" (Just "a")) `matchesPosting` nullposting{ptags=[("foo","bar")]}+      assertBool "" $ not $ (Tag "foo" (Just "a$")) `matchesPosting` nullposting{ptags=[("foo","bar")]}+      assertBool "" $ not $ (Tag " foo " (Just "a")) `matchesPosting` nullposting{ptags=[("foo","bar")]}+      assertBool "" $ not $ (Tag "foo foo" (Just " ar ba ")) `matchesPosting` nullposting{ptags=[("foo foo","bar bar")]}+    ,test "a tag match on a posting also sees inherited tags" $ assertBool "" $ (Tag "txntag" Nothing) `matchesPosting` nullposting{ptransaction=Just nulltransaction{ttags=[("txntag","")]}}+    ,test "cur:" $ do+      assertBool "" $ not $ (Sym "$") `matchesPosting` nullposting{pamount=Mixed [usd 1]} -- becomes "^$$", ie testing for null symbol+      assertBool "" $ (Sym "\\$") `matchesPosting` nullposting{pamount=Mixed [usd 1]} -- have to quote $ for regexpr+      assertBool "" $ (Sym "shekels") `matchesPosting` nullposting{pamount=Mixed [nullamt{acommodity="shekels"}]}+      assertBool "" $ not $ (Sym "shek") `matchesPosting` nullposting{pamount=Mixed [nullamt{acommodity="shekels"}]}   ] -  ,tests "matchesTransaction" [-     expect $ Any `matchesTransaction` nulltransaction-    ,expect $ not $ (Desc "x x") `matchesTransaction` nulltransaction{tdescription="x"}-    ,expect $ (Desc "x x") `matchesTransaction` nulltransaction{tdescription="x x"}+  ,test "matchesTransaction" $ do+     assertBool "" $ Any `matchesTransaction` nulltransaction+     assertBool "" $ not $ (Desc "x x") `matchesTransaction` nulltransaction{tdescription="x"}+     assertBool "" $ (Desc "x x") `matchesTransaction` nulltransaction{tdescription="x x"}      -- see posting for more tag tests-    ,expect $ (Tag "foo" (Just "a")) `matchesTransaction` nulltransaction{ttags=[("foo","bar")]}-    ,expect $ (Tag "payee" (Just "payee")) `matchesTransaction` nulltransaction{tdescription="payee|note"}-    ,expect $ (Tag "note" (Just "note")) `matchesTransaction` nulltransaction{tdescription="payee|note"}+     assertBool "" $ (Tag "foo" (Just "a")) `matchesTransaction` nulltransaction{ttags=[("foo","bar")]}+     assertBool "" $ (Tag "payee" (Just "payee")) `matchesTransaction` nulltransaction{tdescription="payee|note"}+     assertBool "" $ (Tag "note" (Just "note")) `matchesTransaction` nulltransaction{tdescription="payee|note"}      -- a tag match on a transaction also matches posting tags-    ,expect $ (Tag "postingtag" Nothing) `matchesTransaction` nulltransaction{tpostings=[nullposting{ptags=[("postingtag","")]}]}-  ]+     assertBool "" $ (Tag "postingtag" Nothing) `matchesTransaction` nulltransaction{tpostings=[nullposting{ptags=[("postingtag","")]}]}   ]

@@ -307,6 +307,8 @@ tryReaders :: InputOpts -> Maybe FilePath -> [Reader] -> Text -> IO (Either String Journal) tryReaders iopts mpath readers txt = firstSuccessOrFirstError [] readers   where+    -- TODO: #1087 when parsing csv with -f -, if the csv (rules) parser fails, +    -- we would rather see that error, not the one from the journal parser     firstSuccessOrFirstError :: [String] -> [Reader] -> IO (Either String Journal)     firstSuccessOrFirstError [] []        = return $ Left "no readers found"     firstSuccessOrFirstError errs (r:rs) = do

@@ -35,6 +35,7 @@   journalSourcePos,   parseAndFinaliseJournal,   parseAndFinaliseJournal',+  finaliseJournal,   setYear,   getYear,   setDefaultCommodityAndStyle,@@ -99,11 +100,9 @@ ) where --- * imports-import Prelude ()-import "base-compat-batteries" Prelude.Compat hiding (readFile)-import "base-compat-batteries" Control.Monad.Compat+import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail (fail) import Control.Monad.Except (ExceptT(..), runExceptT, throwError)-import Control.Monad.State.Strict+import Control.Monad.State.Strict hiding (fail) import Data.Bifunctor (bimap, second) import Data.Char import Data.Data@@ -178,11 +177,11 @@  rawOptsToInputOpts :: RawOpts -> InputOpts rawOptsToInputOpts rawopts = InputOpts{-   -- files_             = map (T.unpack . stripquotes . T.pack) $ listofstringopt "file" rawopts+   -- files_             = listofstringopt "file" rawopts    mformat_           = Nothing   ,mrules_file_       = maybestringopt "rules-file" rawopts   ,separator_         = fromMaybe ',' (maybecharopt "separator" rawopts)-  ,aliases_           = map (T.unpack . stripquotes . T.pack) $ listofstringopt "alias" rawopts+  ,aliases_           = listofstringopt "alias" rawopts   ,anon_              = boolopt "anon" rawopts   ,ignore_assertions_ = boolopt "ignore-assertions" rawopts   ,new_               = boolopt "new" rawopts@@ -228,101 +227,94 @@   -- | Given a megaparsec ParsedJournal parser, input options, file--- path and file content: parse and post-process a Journal, or give an error.+-- path and file content: parse and finalise a Journal, or give an error. parseAndFinaliseJournal :: ErroringJournalParser IO ParsedJournal -> InputOpts                            -> FilePath -> Text -> ExceptT String IO Journal parseAndFinaliseJournal parser iopts f txt = do-  t <- liftIO getClockTime   y <- liftIO getCurrentYear-  let initJournal = nulljournal-        { jparsedefaultyear = Just y-        , jincludefilestack = [f] }-  eep <- liftIO $ runExceptT $-    runParserT (evalStateT parser initJournal) f txt+  let initJournal = nulljournal{ jparsedefaultyear = Just y, jincludefilestack = [f] }+  eep <- liftIO $ runExceptT $ runParserT (evalStateT parser initJournal) f txt+  -- TODO: urgh.. clean this up somehow   case eep of-    Left finalParseError ->-      throwError $ finalErrorBundlePretty $ attachSource f txt finalParseError-+    Left finalParseError -> throwError $ finalErrorBundlePretty $ attachSource f txt finalParseError     Right ep -> case ep of-      Left e -> throwError $ customErrorBundlePretty e--      Right pj ->-        -- If we are using automated transactions, we finalize twice:-        -- once before and once after. However, if we are running it-        -- twice, we don't check assertions the first time (they might-        -- be false pending modifiers) and we don't reorder the second-        -- time. If we are only running once, we reorder and follow-        -- the options for checking assertions.-        let fj = if auto_ iopts && (not . null . jtxnmodifiers) pj--                 -- transaction modifiers are active-                 then-                   -- first pass, doing most of the work-                     (-                      (journalModifyTransactions <$>) $  -- add auto postings after balancing ? #893b fails-                      journalBalanceTransactions False $-                      -- journalModifyTransactions <$>   -- add auto postings before balancing ? probably #893a, #928, #938 fail-                      journalReverse $-                      journalAddFile (f, txt) $-                      journalApplyCommodityStyles pj)-                   -- second pass, checking balance assertions-                   >>= (\j ->-                      journalBalanceTransactions (not $ ignore_assertions_ iopts) $-                      journalSetLastReadTime t $-                      j)--                 -- transaction modifiers are not active-                 else journalBalanceTransactions (not $ ignore_assertions_ iopts) $-                      journalReverse $-                      journalAddFile (f, txt) $-                      journalApplyCommodityStyles $-                      journalSetLastReadTime t $-                      pj-        in-          case fj of-            Right j -> return j-            Left e  -> throwError e+                  Left e   -> throwError $ customErrorBundlePretty e+                  Right pj -> finaliseJournal iopts f txt pj --- Like parseAndFinaliseJournal but takes a (non-Erroring) JournalParser.--- Used for timeclock/timedot. XXX let them use parseAndFinaliseJournal instead+-- | Like parseAndFinaliseJournal but takes a (non-Erroring) JournalParser.+-- Used for timeclock/timedot.+-- TODO: get rid of this, use parseAndFinaliseJournal instead parseAndFinaliseJournal' :: JournalParser IO ParsedJournal -> InputOpts                            -> FilePath -> Text -> ExceptT String IO Journal parseAndFinaliseJournal' parser iopts f txt = do-  t <- liftIO getClockTime   y <- liftIO getCurrentYear   let initJournal = nulljournal         { jparsedefaultyear = Just y         , jincludefilestack = [f] }   ep <- liftIO $ runParserT (evalStateT parser initJournal) f txt+  -- see notes above   case ep of     Left e   -> throwError $ customErrorBundlePretty e+    Right pj -> finaliseJournal iopts f txt pj -    Right pj ->-      -- If we are using automated transactions, we finalize twice:-      -- once before and once after. However, if we are running it-      -- twice, we don't check assertions the first time (they might-      -- be false pending modifiers) and we don't reorder the second-      -- time. If we are only running once, we reorder and follow the-      -- options for checking assertions.-      let fj = if auto_ iopts && (not . null . jtxnmodifiers) pj-               then journalModifyTransactions <$>-                    (journalBalanceTransactions False $-                     journalReverse $-                     journalApplyCommodityStyles pj) >>=-                    (\j -> journalBalanceTransactions (not $ ignore_assertions_ iopts) $-                           journalAddFile (f, txt) $-                           journalSetLastReadTime t $-                           j)-               else journalBalanceTransactions (not $ ignore_assertions_ iopts) $-                    journalReverse $-                    journalAddFile (f, txt) $-                    journalApplyCommodityStyles $-                    journalSetLastReadTime t $-                    pj+-- | Post-process a Journal that has just been parsed or generated, in this order:+--+-- - apply canonical amount styles,+--+-- - save misc info and reverse transactions into their original parse order,+--+-- - evaluate balance assignments and balance each transaction,+--+-- - apply transaction modifiers (auto postings) if enabled,+--+-- - check balance assertions if enabled.+--+finaliseJournal :: InputOpts -> FilePath -> Text -> Journal -> ExceptT String IO Journal+finaliseJournal iopts f txt pj = do+  t <- liftIO getClockTime+  -- Infer and apply canonical styles for each commodity (or fail).+  -- TODO: since #903's refactoring for hledger 1.12,+  -- journalApplyCommodityStyles here is seeing the+  -- transactions before they get reversesd to normal order.+  case journalApplyCommodityStyles pj of+    Left e    -> throwError e+    Right pj' -> +      -- Finalise the parsed journal.+      let fj =+            if auto_ iopts && (not . null . jtxnmodifiers) pj+            then+              -- When automatic postings are active, we finalise twice:+              -- once before and once after. However, if we are running it+              -- twice, we don't check assertions the first time (they might+              -- be false pending modifiers) and we don't reorder the second+              -- time. If we are only running once, we reorder and follow+              -- the options for checking assertions.+              --+              -- first pass, doing most of the work+              (+               (journalModifyTransactions <$>) $  -- add auto postings after balancing ? #893b fails+               journalBalanceTransactions False $ -- balance transactions without checking assertions+               -- journalModifyTransactions <$>   -- add auto postings before balancing ? probably #893a, #928, #938 fail+               journalReverse $+               journalSetLastReadTime t $+               journalAddFile (f, txt) $+               pj')+              -- balance transactions a second time, now just checking balance assertions+              >>= (\j ->+                 journalBalanceTransactions (not $ ignore_assertions_ iopts) $+                 j)++            else+              -- automatic postings are not active+              journalBalanceTransactions (not $ ignore_assertions_ iopts) $+              journalReverse $+              journalSetLastReadTime t $+              journalAddFile (f, txt) $+              pj'       in         case fj of-          Right j -> return j           Left e  -> throwError e+          Right j -> return j  setYear :: Year -> JournalParser m () setYear y = modify' (\j -> j{jparsedefaultyear=Just y})@@ -781,7 +773,7 @@     dbg8 "numberp suggestedStyle" suggestedStyle `seq` return ()     case dbg8 "numberp quantity,precision,mdecimalpoint,mgrps"            $ fromRawNumber rawNum mExp of-      Left errMsg -> fail errMsg+      Left errMsg -> Fail.fail errMsg       Right (q, p, d, g) -> pure (sign q, p, d, g)  exponentp :: TextParser m Int@@ -860,14 +852,14 @@ -- | Parse and interpret the structure of a number without external hints. -- Numbers are digit strings, possibly separated into digit groups by one -- of two types of separators. (1) Numbers may optionally have a decimal--- point, which may be either a period or comma. (2) Numbers may--- optionally contain digit group separators, which must all be either a+-- mark, which may be either a period or comma. (2) Numbers may+-- optionally contain digit group marks, which must all be either a -- period, a comma, or a space. ----- It is our task to deduce the identities of the decimal point and digit--- separator characters, based on the allowed syntax. For instance, we--- make use of the fact that a decimal point can occur at most once and--- must succeed all digit group separators.+-- It is our task to deduce the characters used as decimal mark and+-- digit group mark, based on the allowed syntax. For instance, we+-- make use of the fact that a decimal mark can occur at most once and+-- must be to the right of all digit group marks. -- -- >>> parseTest rawnumberp "1,234,567.89" -- Right (WithSeparators ',' ["1","234","567"] (Just ('.',"89")))@@ -883,7 +875,7 @@   -- Guard against mistyped numbers   mExtraDecimalSep <- optional $ lookAhead $ satisfy isDecimalPointChar   when (isJust mExtraDecimalSep) $-    fail "invalid number (invalid use of separator)"+    Fail.fail "invalid number (invalid use of separator)"    mExtraFragment <- optional $ lookAhead $ try $     char ' ' *> getOffset <* digitChar@@ -946,12 +938,30 @@ isDigitSeparatorChar :: Char -> Bool isDigitSeparatorChar c = isDecimalPointChar c || c == ' ' +-- | Some kinds of number literal we might parse.+data RawNumber+  = NoSeparators   DigitGrp (Maybe (Char, DigitGrp))+    -- ^ A number with no digit group marks (eg 100),+    --   or with a leading or trailing comma or period+    --   which (apparently) we interpret as a decimal mark (like 100. or .100)+  | WithSeparators Char [DigitGrp] (Maybe (Char, DigitGrp))+    -- ^ A number with identifiable digit group marks+    --   (eg 1,000,000 or 1,000.50 or 1 000)+  deriving (Show, Eq) +-- | Another kind of number literal: this one contains either a digit+-- group separator or a decimal mark, we're not sure which (eg 1,000 or 100.50).+data AmbiguousNumber = AmbiguousNumber DigitGrp Char DigitGrp+  deriving (Show, Eq)++-- | Description of a single digit group in a number literal.+-- "Thousands" is one well known digit grouping, but there are others. data DigitGrp = DigitGrp {-  digitGroupLength :: !Int,-  digitGroupNumber :: !Integer+  digitGroupLength :: !Int,    -- ^ The number of digits in this group.+  digitGroupNumber :: !Integer -- ^ The natural number formed by this group's digits. } deriving (Eq) +-- | A custom show instance, showing digit groups as the parser saw them. instance Show DigitGrp where   show (DigitGrp len num)     | len > 0 = "\"" ++ padding ++ numStr ++ "\""@@ -973,14 +983,6 @@     makeGroup = uncurry DigitGrp . foldl' step (0, 0) . T.unpack     step (!l, !a) c = (l+1, a*10 + fromIntegral (digitToInt c)) -data RawNumber-  = NoSeparators   DigitGrp (Maybe (Char, DigitGrp))        -- 100 or 100. or .100 or 100.50-  | WithSeparators Char [DigitGrp] (Maybe (Char, DigitGrp)) -- 1,000,000 or 1,000.50-  deriving (Show, Eq)--data AmbiguousNumber = AmbiguousNumber DigitGrp Char DigitGrp  -- 1,000-  deriving (Show, Eq)- --- ** comments  multilinecommentp :: TextParser m ()@@ -1016,7 +1018,7 @@ -- Several journal items may be followed by comments, which begin with -- semicolons and extend to the end of the line. Such comments may span -- multiple lines, but comment lines below the journal item must be--- preceeded by leading whitespace.+-- preceded by leading whitespace. -- -- This parser combinator accepts a parser that consumes all input up -- until the next newline. This parser should extract the "content" from@@ -1073,7 +1075,7 @@ -- The first line of a transaction may be followed by comments, which -- begin with semicolons and extend to the end of the line. Transaction -- comments may span multiple lines, but comment lines below the--- transaction must be preceeded by leading whitespace.+-- transaction must be preceded by leading whitespace. -- -- 2000/1/1 ; a transaction comment starting on the same line ... --   ; extending to the next line@@ -1127,7 +1129,7 @@ -- -- Postings may be followed by comments, which begin with semicolons and -- extend to the end of the line. Posting comments may span multiple--- lines, but comment lines below the posting must be preceeded by+-- lines, but comment lines below the posting must be preceded by -- leading whitespace. -- -- 2000/1/1@@ -1273,7 +1275,7 @@        $ between (char '[') (char ']')        $ takeWhile1P Nothing isBracketedDateChar     unless (T.any isDigit s && T.any isDateSepChar s) $-      fail "not a bracketed date"+      Fail.fail "not a bracketed date"   -- Looks sufficiently like a bracketed date to commit to parsing a date    between (char '[') (char ']') $ do@@ -1304,9 +1306,9 @@ tests_Common = tests "Common" [     tests "amountp" [-    test "basic"                  $ expectParseEq amountp "$47.18"     (usd 47.18)-   ,test "ends with decimal mark" $ expectParseEq amountp "$1."        (usd 1  `withPrecision` 0)-   ,test "unit price"             $ expectParseEq amountp "$10 @ €0.5"+    test "basic"                  $ assertParseEq amountp "$47.18"     (usd 47.18)+   ,test "ends with decimal mark" $ assertParseEq amountp "$1."        (usd 1  `withPrecision` 0)+   ,test "unit price"             $ assertParseEq amountp "$10 @ €0.5"       -- not precise enough:       -- (usd 10 `withPrecision` 0 `at` (eur 0.5 `withPrecision` 1)) -- `withStyle` asdecimalpoint=Just '.'       amount{@@ -1320,7 +1322,7 @@             ,astyle=amountstyle{asprecision=1, asdecimalpoint=Just '.'}             }         }-   ,test "total price"            $ expectParseEq amountp "$10 @@ €5"+   ,test "total price"            $ assertParseEq amountp "$10 @@ €5"       amount{          acommodity="$"         ,aquantity=10@@ -1335,32 +1337,31 @@     ]    ,let p = lift (numberp Nothing) :: JournalParser IO (Quantity, Int, Maybe Char, Maybe DigitGroupStyle) in-   tests "numberp" [-     test "." $ expectParseEq p "0"          (0, 0, Nothing, Nothing)-    ,test "." $ expectParseEq p "1"          (1, 0, Nothing, Nothing)-    ,test "." $ expectParseEq p "1.1"        (1.1, 1, Just '.', Nothing)-    ,test "." $ expectParseEq p "1,000.1"    (1000.1, 1, Just '.', Just $ DigitGroups ',' [3])-    ,test "." $ expectParseEq p "1.00.000,1" (100000.1, 1, Just ',', Just $ DigitGroups '.' [3,2])-    ,test "." $ expectParseEq p "1,000,000"  (1000000, 0, Nothing, Just $ DigitGroups ',' [3,3])  -- could be simplified to [3]-    ,test "." $ expectParseEq p "1."         (1, 0, Just '.', Nothing)-    ,test "." $ expectParseEq p "1,"         (1, 0, Just ',', Nothing)-    ,test "." $ expectParseEq p ".1"         (0.1, 1, Just '.', Nothing)-    ,test "." $ expectParseEq p ",1"         (0.1, 1, Just ',', Nothing)-    ,test "." $ expectParseError p "" ""-    ,test "." $ expectParseError p "1,000.000,1" ""-    ,test "." $ expectParseError p "1.000,000.1" ""-    ,test "." $ expectParseError p "1,000.000.1" ""-    ,test "." $ expectParseError p "1,,1" ""-    ,test "." $ expectParseError p "1..1" ""-    ,test "." $ expectParseError p ".1," ""-    ,test "." $ expectParseError p ",1." ""-    ]+   test "numberp" $ do+     assertParseEq p "0"          (0, 0, Nothing, Nothing)+     assertParseEq p "1"          (1, 0, Nothing, Nothing)+     assertParseEq p "1.1"        (1.1, 1, Just '.', Nothing)+     assertParseEq p "1,000.1"    (1000.1, 1, Just '.', Just $ DigitGroups ',' [3])+     assertParseEq p "1.00.000,1" (100000.1, 1, Just ',', Just $ DigitGroups '.' [3,2])+     assertParseEq p "1,000,000"  (1000000, 0, Nothing, Just $ DigitGroups ',' [3,3])  -- could be simplified to [3]+     assertParseEq p "1."         (1, 0, Just '.', Nothing)+     assertParseEq p "1,"         (1, 0, Just ',', Nothing)+     assertParseEq p ".1"         (0.1, 1, Just '.', Nothing)+     assertParseEq p ",1"         (0.1, 1, Just ',', Nothing)+     assertParseError p "" ""+     assertParseError p "1,000.000,1" ""+     assertParseError p "1.000,000.1" ""+     assertParseError p "1,000.000.1" ""+     assertParseError p "1,,1" ""+     assertParseError p "1..1" ""+     assertParseError p ".1," ""+     assertParseError p ",1." ""    ,tests "spaceandamountormissingp" [-     test "space and amount" $ expectParseEq spaceandamountormissingp " $47.18" (Mixed [usd 47.18])-    ,test "empty string" $ expectParseEq spaceandamountormissingp "" missingmixedamt-    ,_test "just space" $ expectParseEq spaceandamountormissingp " " missingmixedamt  -- XXX should it ?-    -- ,test "just amount" $ expectParseError spaceandamountormissingp "$47.18" ""  -- succeeds, consuming nothing+     test "space and amount" $ assertParseEq spaceandamountormissingp " $47.18" (Mixed [usd 47.18])+    ,test "empty string" $ assertParseEq spaceandamountormissingp "" missingmixedamt+    -- ,test "just space" $ assertParseEq spaceandamountormissingp " " missingmixedamt  -- XXX should it ?+    -- ,test "just amount" $ assertParseError spaceandamountormissingp "$47.18" ""  -- succeeds, consuming nothing     ]    ]

@@ -30,13 +30,15 @@   tests_CsvReader, ) where-import Prelude ()-import "base-compat-batteries" Prelude.Compat-import Control.Exception hiding (try)-import Control.Monad-import Control.Monad.Except+import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail (fail)+import Control.Exception          (IOException, handle, throw)+import Control.Monad              (liftM, unless, when)+import Control.Monad.Except       (ExceptT, throwError)+import Control.Monad.IO.Class     (liftIO) import Control.Monad.State.Strict (StateT, get, modify', evalStateT)-import Data.Char (toLower, isDigit, isSpace, ord)+import Control.Monad.Trans.Class  (lift)+import Data.Char                  (toLower, isDigit, isSpace, ord)+import Data.Bifunctor             (first) import "base-compat-batteries" Data.List.Compat import Data.Maybe import Data.Ord@@ -67,7 +69,7 @@  import Hledger.Data import Hledger.Utils-import Hledger.Read.Common (Reader(..),InputOpts(..),amountp, statusp, genericSourcePos)+import Hledger.Read.Common (Reader(..),InputOpts(..),amountp, statusp, genericSourcePos, finaliseJournal)  type CSV = [Record] @@ -84,15 +86,21 @@   }  -- | Parse and post-process a "Journal" from CSV data, or give an error.--- XXX currently ignores the string and reads from the file path+-- Does not check balance assertions.+-- XXX currently ignores the provided data, reads it from the file path instead. parse :: InputOpts -> FilePath -> Text -> ExceptT String IO Journal parse iopts f t = do   let rulesfile = mrules_file_ iopts   let separator = separator_ iopts   r <- liftIO $ readJournalFromCsv separator rulesfile f t-  case r of Left e -> throwError e-            Right j -> return $ journalNumberAndTieTransactions j--- XXX does not use parseAndFinaliseJournal like the other readers+  case r of Left e   -> throwError e+            Right pj -> finaliseJournal iopts{ignore_assertions_=True} f t pj'+              where+                -- finaliseJournal assumes the journal's items are+                -- reversed, as produced by JournalReader's parser.+                -- But here they are already properly ordered. So we'd+                -- better preemptively reverse them once more. XXX inefficient+                pj' = journalReverse pj  -- | Read a Journal from the given CSV data (and filename, used for error -- messages), or return an error. Proceed as follows:@@ -108,32 +116,35 @@ readJournalFromCsv :: Char -> Maybe FilePath -> FilePath -> Text -> IO (Either String Journal) readJournalFromCsv _ Nothing "-" _ = return $ Left "please use --rules-file when reading CSV from stdin" readJournalFromCsv separator mrulesfile csvfile csvdata =- handle (\e -> return $ Left $ show (e :: IOException)) $ do+ handle (\(e::IOException) -> return $ Left $ show e) $ do++  -- make and throw an IO exception.. which we catch and convert to an Either above ?   let throwerr = throw . userError -  -- parse rules+  -- parse the csv rules   let rulesfile = fromMaybe (rulesFileFor csvfile) mrulesfile   rulesfileexists <- doesFileExist rulesfile   rulestext <-     if rulesfileexists     then do       dbg1IO "using conversion rules file" rulesfile-      liftIO $ (readFilePortably rulesfile >>= expandIncludes (takeDirectory rulesfile))-    else return $ defaultRulesText rulesfile-  rules <- liftIO (runExceptT $ parseAndValidateCsvRules rulesfile rulestext) >>= either throwerr return+      readFilePortably rulesfile >>= expandIncludes (takeDirectory rulesfile)+    else +      return $ defaultRulesText rulesfile+  rules <- either throwerr return $ parseAndValidateCsvRules rulesfile rulestext   dbg2IO "rules" rules -  -- apply skip directive-  let skip = maybe 0 oneorerror $ getDirective "skip" rules-        where-          oneorerror "" = 1-          oneorerror s  = readDef (throwerr $ "could not parse skip value: " ++ show s) s+  -- parse the skip directive's value, if any+  let skiplines = case getDirective "skip" rules of+                    Nothing -> 0+                    Just "" -> 1+                    Just s  -> readDef (throwerr $ "could not parse skip value: " ++ show s) s    -- parse csv-  -- parsec seems to fail if you pass it "-" here XXX try again with megaparsec+  -- parsec seems to fail if you pass it "-" here TODO: try again with megaparsec   let parsecfilename = if csvfile == "-" then "(stdin)" else csvfile   records <- (either throwerr id .-              dbg2 "validateCsv" . validateCsv skip .+              dbg2 "validateCsv" . validateCsv rules skiplines .               dbg2 "parseCsv")              `fmap` parseCsv separator parsecfilename csvdata   dbg1IO "first 3 csv records" $ take 3 records@@ -156,10 +167,10 @@                    (initialPos parsecfilename) records      -- Ensure transactions are ordered chronologically.-    -- First, reverse them to get same-date transactions ordered chronologically,-    -- if the CSV records seem to be most-recent-first, ie if there's an explicit-    -- "newest-first" directive, or if there's more than one date and the first date-    -- is more recent than the last.+    -- First, if the CSV records seem to be most-recent-first (because+    -- there's an explicit "newest-first" directive, or there's more+    -- than one date and the first date is more recent than the last):+    -- reverse them to get same-date transactions ordered chronologically.     txns' =       (if newestfirst || mseemsnewestfirst == Just True then reverse else id) txns       where@@ -209,20 +220,29 @@           unlined = concat . intersperse "\n"  -- | Return the cleaned up and validated CSV data (can be empty), or an error.-validateCsv :: Int -> Either String CSV -> Either String [CsvRecord]-validateCsv _           (Left err) = Left err-validateCsv numhdrlines (Right rs) = validate $ drop numhdrlines $ filternulls rs+validateCsv :: CsvRules -> Int -> Either String CSV -> Either String [CsvRecord]+validateCsv _ _           (Left err) = Left err+validateCsv rules numhdrlines (Right rs) = validate $ applyConditionalSkips $ drop numhdrlines $ filternulls rs   where     filternulls = filter (/=[""])+    skipCount r =+      case (getEffectiveAssignment rules r "end", getEffectiveAssignment rules r "skip") of+        (Nothing, Nothing) -> Nothing+        (Just _, _) -> Just maxBound+        (Nothing, Just "") -> Just 1+        (Nothing, Just x) -> Just (read x)+    applyConditionalSkips [] = []+    applyConditionalSkips (r:rest) =+      case skipCount r of+        Nothing -> r:(applyConditionalSkips rest)+        Just cnt -> applyConditionalSkips (drop (cnt-1) rest)     validate [] = Right []-    validate rs@(first:_)-      | isJust lessthan2 = let r = fromJust lessthan2 in Left $ printf "CSV record %s has less than two fields" (show r)-      | isJust different = let r = fromJust different in Left $ printf "the first CSV record %s has %d fields but %s has %d" (show first) length1 (show r) (length r)+    validate rs@(_first:_)+      | isJust lessthan2 = let r = fromJust lessthan2 in+          Left $ printf "CSV record %s has less than two fields" (show r)       | otherwise        = Right rs       where-        length1   = length first         lessthan2 = headMay $ filter ((<2).length) rs-        different = headMay $ filter ((/=length1).length) rs  -- -- | The highest (0-based) field index referenced in the field -- -- definitions, or -1 if no fields are defined.@@ -256,7 +276,7 @@   ,""   ,"account1 assets:bank:checking"   ,""-  ,"fields date, description, amount"+  ,"fields date, description, amount1"   ,""   ,"#skip 1"   ,"#newest-first"@@ -364,7 +384,7 @@ type DateFormat       = String type RegexpPattern           = String -rules = CsvRules {+defrules = CsvRules {   rdirectives=[],   rcsvfieldindexes=[],   rassignments=[],@@ -399,16 +419,19 @@ instance ShowErrorComponent String where   showErrorComponent = id --- | An error-throwing action that parses this file's content+-- Not used by hledger; just for lib users, +-- | An pure-exception-throwing IO action that parses this file's content -- as CSV conversion rules, interpolating any included files first, -- and runs some extra validation checks. parseRulesFile :: FilePath -> ExceptT String IO CsvRules parseRulesFile f =-  liftIO (readFilePortably f >>= expandIncludes (takeDirectory f)) >>= parseAndValidateCsvRules f+  liftIO (readFilePortably f >>= expandIncludes (takeDirectory f))+    >>= either throwError return . parseAndValidateCsvRules f  -- | Inline all files referenced by include directives in this hledger CSV rules text, recursively. -- Included file paths may be relative to the directory of the provided file path.--- This is a cheap hack to avoid rewriting the CSV rules parser.+-- This is done as a pre-parse step to simplify the CSV rules parser.+expandIncludes :: FilePath -> Text -> IO Text expandIncludes dir content = mapM (expandLine dir) (T.lines content) >>= return . T.unlines   where     expandLine dir line =@@ -419,47 +442,30 @@             dir' = takeDirectory f'         _ -> return line --- | An error-throwing action that parses this text as CSV conversion rules--- and runs some extra validation checks. The file path is for error messages.-parseAndValidateCsvRules :: FilePath -> T.Text -> ExceptT String IO CsvRules-parseAndValidateCsvRules rulesfile s = do-  let rules = parseCsvRules rulesfile s-  case rules of-    Left e -> ExceptT $ return $ Left $ customErrorBundlePretty e-    Right r -> do-               r_ <- liftIO $ runExceptT $ validateRules r-               ExceptT $ case r_ of-                 Left  s -> return $ Left $ parseErrorPretty $ makeParseError s-                 Right r -> return $ Right r-+-- | An error-throwing IO action that parses this text as CSV conversion rules+-- and runs some extra validation checks. The file path is used in error messages.+parseAndValidateCsvRules :: FilePath -> T.Text -> Either String CsvRules+parseAndValidateCsvRules rulesfile s =+  case parseCsvRules rulesfile s of+    Left err    -> Left $ customErrorBundlePretty err+    Right rules -> first makeFancyParseError $ validateRules rules   where-    makeParseError :: String -> ParseError T.Text String-    makeParseError s = FancyError 0 (S.singleton $ ErrorFail s)+    makeFancyParseError :: String -> String+    makeFancyParseError s = +      parseErrorPretty (FancyError 0 (S.singleton $ ErrorFail s) :: ParseError Text String)  -- | Parse this text as CSV conversion rules. The file path is for error messages. parseCsvRules :: FilePath -> T.Text -> Either (ParseErrorBundle T.Text CustomErr) CsvRules -- parseCsvRules rulesfile s = runParser csvrulesfile nullrules{baseAccount=takeBaseName rulesfile} rulesfile s parseCsvRules rulesfile s =-  runParser (evalStateT rulesp rules) rulesfile s+  runParser (evalStateT rulesp defrules) rulesfile s  -- | Return the validated rules, or an error.-validateRules :: CsvRules -> ExceptT String IO CsvRules+validateRules :: CsvRules -> Either String CsvRules validateRules rules = do-  unless (isAssigned "date")   $ ExceptT $ return $ Left "Please specify (at top level) the date field. Eg: date %1\n"-  unless ((amount && not (amountin || amountout)) ||-          (not amount && (amountin && amountout)) ||-          balance)-    $ ExceptT $ return $ Left $ unlines [-       "Please specify (as a top level CSV rule) either the amount field,"-      ,"both the amount-in and amount-out fields, or the balance field. Eg:"-      ,"amount %2\n"-      ]-  ExceptT $ return $ Right rules+  unless (isAssigned "date")   $ Left "Please specify (at top level) the date field. Eg: date %1\n"+  Right rules   where-    amount    = isAssigned "amount"-    amountin  = isAssigned "amount-in"-    amountout = isAssigned "amount-out"-    balance   = isAssigned "balance" || isAssigned "balance1" || isAssigned "balance2"     isAssigned f = isJust $ getEffectiveAssignment rules [] f  -- parsers@@ -510,6 +516,7 @@   ,"newest-first"    -- ,"base-account"    -- ,"base-currency"+  , "balance-type"   ]  directivevalp :: CsvRulesParser String@@ -545,8 +552,9 @@ fieldassignmentp = do   lift $ dbgparse 3 "trying fieldassignmentp"   f <- journalfieldnamep-  assignmentseparatorp-  v <- fieldvalp+  v <- choiceInState [ assignmentseparatorp >> fieldvalp+                     , lift eolof >> return ""+                     ]   return (f,v)   <?> "field assignment" @@ -558,14 +566,19 @@ -- Transaction fields and pseudo fields for CSV conversion. -- Names must precede any other name they contain, for the parser -- (amount-in before amount; date2 before date). TODO: fix-journalfieldnames = [-   "account1"-  ,"account2"-  ,"amount-in"+journalfieldnames =+  concat [[ "account" ++ i+          ,"amount" ++ i ++ "-in"+          ,"amount" ++ i ++ "-out"+          ,"amount" ++ i+          ,"balance" ++ i+          ,"comment" ++ i+          ,"currency" ++ i+          ] | x <- [1..9], let i = show x]+  +++  ["amount-in"   ,"amount-out"   ,"amount"-  ,"balance1"-  ,"balance2"   ,"balance"   ,"code"   ,"comment"@@ -574,17 +587,16 @@   ,"date"   ,"description"   ,"status"+  ,"skip" -- skip and end are not really fields, but we list it here to allow conditional rules that skip records+  ,"end"   ]  assignmentseparatorp :: CsvRulesParser () assignmentseparatorp = do   lift $ dbgparse 3 "trying assignmentseparatorp"-  choice [-    -- try (lift (skipMany spacenonewline) >> oneOf ":="),-    try (lift (skipMany spacenonewline) >> char ':'),-    spaceChar-    ]-  _ <- lift (skipMany spacenonewline)+  _ <- choiceInState [ lift (skipMany spacenonewline) >> char ':' >> lift (skipMany spacenonewline)+                     , lift (skipSome spacenonewline)+                     ]   return ()  fieldvalp :: CsvRulesParser String@@ -597,9 +609,9 @@   lift $ dbgparse 3 "trying conditionalblockp"   string "if" >> lift (skipMany spacenonewline) >> optional newline   ms <- some recordmatcherp-  as <- many (lift (skipSome spacenonewline) >> fieldassignmentp)+  as <- many (try $ lift (skipSome spacenonewline) >> fieldassignmentp)   when (null as) $-    fail "start of conditional block found, but no assignment rules afterward\n(assignment rules in a conditional block should be indented)\n"+    Fail.fail "start of conditional block found, but no assignment rules afterward\n(assignment rules in a conditional block should be indented)\n"   return (ms, as)   <?> "conditional block" @@ -610,7 +622,7 @@   _  <- optional (matchoperatorp >> lift (skipMany spacenonewline) >> optional newline)   ps <- patternsp   when (null ps) $-    fail "start of record matcher found, but no patterns afterward\n(patterns should not be indented)\n"+    Fail.fail "start of record matcher found, but no patterns afterward\n(patterns should not be indented)\n"   return ps   <?> "record matcher" @@ -654,8 +666,9 @@  type CsvRecord = [String] --- Convert a CSV record to a transaction using the rules, or raise an--- error if the data can not be parsed.+showRules rules record =+  unlines $ catMaybes [ (("the "++fld++" rule is: ")++) <$> getEffectiveAssignment rules record fld | fld <- journalfieldnames]+ transactionFromCsvRecord :: SourcePos -> CsvRules -> CsvRecord -> Transaction transactionFromCsvRecord sourcepos rules record = t   where@@ -671,11 +684,11 @@     mdateformat = mdirective "date-format"     date        = render $ fromMaybe "" $ mfieldtemplate "date"     date'       = fromMaybe (error' $ dateerror "date" date mdateformat) $ mparsedate date-    mdate2      = maybe Nothing (Just . render) $ mfieldtemplate "date2"+    mdate2      = render <$> mfieldtemplate "date2"     mdate2'     = maybe Nothing (maybe (error' $ dateerror "date2" (fromMaybe "" mdate2) mdateformat) Just . mparsedate) mdate2     dateerror datefield value mdateformat = unlines       ["error: could not parse \""++value++"\" as a date using date format "++maybe "\"YYYY/M/D\", \"YYYY-M-D\" or \"YYYY.M.D\"" show mdateformat-      ,"the CSV record is:  "++showRecord record+      , showRecord record       ,"the "++datefield++" rule is:   "++(fromMaybe "required, but missing" $ mfieldtemplate datefield)       ,"the date-format is: "++fromMaybe "unspecified" mdateformat       ,"you may need to "@@ -695,58 +708,108 @@               ["error: could not parse \""++str++"\" as a cleared status (should be *, ! or empty)"               ,"the parse error is:      "++customErrorBundlePretty err               ]-    code        = maybe "" render $ mfieldtemplate "code"-    description = maybe "" render $ mfieldtemplate "description"-    comment     = maybe "" render $ mfieldtemplate "comment"-    precomment  = maybe "" render $ mfieldtemplate "precomment"-    currency    = maybe (fromMaybe "" mdefaultcurrency) render $ mfieldtemplate "currency"-    amountstr   = (currency++) <$> simplifySign <$> getAmountStr rules record-    maybeamount      = either amounterror (Mixed . (:[])) <$> runParser (evalStateT (amountp <* eof) mempty) "" <$> T.pack <$> amountstr-    amounterror err = error' $ unlines-      ["error: could not parse \""++fromJust amountstr++"\" as an amount"-      ,showRecord record-      ,"the amount rule is:      "++(fromMaybe "" $ mfieldtemplate "amount")-      ,"the currency rule is:    "++(fromMaybe "unspecified" $ mfieldtemplate "currency")-      ,"the default-currency is: "++fromMaybe "unspecified" mdefaultcurrency-      ,"the parse error is:      "++customErrorBundlePretty err-      ,"you may need to "-       ++"change your amount or currency rules, "-       ++"or "++maybe "add a" (const "change your") mskip++" skip rule"-      ]-    amount1 = case maybeamount of-                Just a -> a-                Nothing | balance1 /= Nothing || balance2 /= Nothing -> nullmixedamt-                Nothing -> error' $ "amount and balance have no value\n"++showRecord record-    -- convert balancing amount to cost like hledger print, so eg if-    -- amount1 is "10 GBP @@ 15 USD", amount2 will be "-15 USD".-    amount2        = costOfMixedAmount (-amount1)+    code        = singleline $ maybe "" render $ mfieldtemplate "code"+    description = singleline $ maybe "" render $ mfieldtemplate "description"+    comment     = singleline $ maybe "" render $ mfieldtemplate "comment"+    precomment  = singleline $ maybe "" render $ mfieldtemplate "precomment"+     s `or` def  = if null s then def else s-    defaccount1 = fromMaybe "unknown" $ mdirective "default-account1"-    defaccount2 = case isNegativeMixedAmount amount2 of-                   Just True -> "income:unknown"-                   _         -> "expenses:unknown"-    account1    = T.pack $ maybe "" render (mfieldtemplate "account1") `or` defaccount1-    account2    = T.pack $ maybe "" render (mfieldtemplate "account2") `or` defaccount2-    balance1template =-      case (mfieldtemplate "balance", mfieldtemplate "balance1") of-        (Nothing, Nothing)  -> Nothing-        (balance, Nothing)  -> balance-        (Nothing, balance1) -> balance1-        (Just _, Just _)    -> error' "Please use either balance or balance1, but not both"-    balance1     = maybe Nothing (parsebalance "1".render) $ balance1template-    balance2     = maybe Nothing (parsebalance "2".render) $ mfieldtemplate "balance2"-    parsebalance n str+    parsebalance currency n str       | all isSpace str  = Nothing       | otherwise = Just $ (either (balanceerror n str) id $ runParser (evalStateT (amountp <* eof) mempty) "" $ T.pack $ (currency++) $ simplifySign str, nullsourcepos)     balanceerror n str err = error' $ unlines       ["error: could not parse \""++str++"\" as balance"++n++" amount"       ,showRecord record-      ,"the balance"++n++" rule is:      "++(fromMaybe "" $ mfieldtemplate ("balance"++n))-      ,"the currency rule is:    "++(fromMaybe "unspecified" $ mfieldtemplate "currency")+      ,showRules rules record       ,"the default-currency is: "++fromMaybe "unspecified" mdefaultcurrency       ,"the parse error is:      "++customErrorBundlePretty err       ] +    parsePosting' number accountFld amountFld amountInFld amountOutFld balanceFld commentFld =+      let currency = maybe (fromMaybe "" mdefaultcurrency) render $+                      (mfieldtemplate ("currency"++number) `or `mfieldtemplate "currency")+          amount = chooseAmount rules record currency amountFld amountInFld amountOutFld                      +          account' = ((T.pack . render) <$> (mfieldtemplate accountFld+                                           `or` mdirective ("default-account" ++ number)))+          balance = (parsebalance currency number.render) =<< mfieldtemplate balanceFld+          comment = T.pack $ maybe "" render $ mfieldtemplate commentFld+          account =+            case account' of+              -- If account is explicitly "unassigned", suppress posting+              -- Otherwise, generate posting with "expenses:unknown" account if we have amount/balance information+              Just "" -> Nothing+              Just account -> Just account+              Nothing ->+                -- If we have amount or balance assertion (which implies potential amount change),+                -- but no account name, lets generate "expenses:unknown" account name.+                case (amount, balance) of+                  (Just _, _ ) -> Just "expenses:unknown"+                  (_, Just _)  -> Just "expenses:unknown"+                  (Nothing, Nothing) -> Nothing+          in+        case account of+          Nothing -> Nothing+          Just account -> +            Just $ (number, posting {paccount=accountNameWithoutPostingType account+                                    , pamount=fromMaybe missingmixedamt amount+                                    , ptransaction=Just t+                                    , pbalanceassertion=toAssertion <$> balance+                                    , pcomment = comment+                                    , ptype = accountNamePostingType account})++    parsePosting number =              +      parsePosting' number+      ("account"++number)+      ("amount"++number)+      ("amount"++number++"-in")+      ("amount"++number++"-out")+      ("balance"++number)+      ("comment" ++ number)+      +    withAlias fld alias =+      case (mfieldtemplate fld, mfieldtemplate alias) of+        (Just fld, Just alias) -> error' $ unlines+          [ "error: both \"" ++ fld ++ "\" and \"" ++ alias ++ "\" have values."+          , showRecord record+          , showRules rules record+          ]+        (Nothing, Just _) -> alias+        (_, Nothing)      -> fld++    posting1 = parsePosting' "1"+               ("account1" `withAlias` "account")+               ("amount1" `withAlias` "amount")+               ("amount1-in" `withAlias` "amount-in")+               ("amount1-out" `withAlias` "amount-out")+               ("balance1" `withAlias` "balance")+               "comment1" -- comment1 does not have legacy alias++    postings' = catMaybes $ posting1:[ parsePosting i | x<-[2..9], let i = show x]++    improveUnknownAccountName p =+      if paccount p /="expenses:unknown"+      then p+      else case isNegativeMixedAmount (pamount p) of+        Just True -> p{paccount = "income:unknown"}+        Just False -> p{paccount = "expenses:unknown"}+        _ -> p+        +    postings =+      case postings' of+        -- To be compatible with the behavior of the old code which allowed two postings only, we enforce+        -- second posting when rules generated just first of them, and posting is of type that should be balanced.+        -- When we have srictly first and second posting, but second posting does not have amount, we fill it in.+        [("1",posting1)] ->+          case ptype posting1 of+            VirtualPosting -> [posting1]+            _ ->+              [posting1,improveUnknownAccountName (posting{paccount="expenses:unknown", pamount=costOfMixedAmount(-(pamount posting1)), ptransaction=Just t})]+        [("1",posting1),("2",posting2)] ->+          case (pamount posting1 == missingmixedamt , pamount posting2 == missingmixedamt) of+            (False, True) -> [posting1, improveUnknownAccountName (posting2{pamount=costOfMixedAmount(-(pamount posting1))})]+            _  -> [posting1, posting2]+        _ -> map snd postings'+             -- build the transaction     t = nulltransaction{       tsourcepos               = genericSourcePos sourcepos,@@ -756,40 +819,71 @@       tcode                    = T.pack code,       tdescription             = T.pack description,       tcomment                 = T.pack comment,-      tprecedingcomment = T.pack precomment,-      tpostings                =-        [posting {paccount=account1, pamount=amount1, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance1}-        ,posting {paccount=account2, pamount=amount2, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance2}-        ]+      tprecedingcomment        = T.pack precomment,+      tpostings                = postings       }-    toAssertion (a, b) = assertion{++    defaultAssertion =+      case mdirective "balance-type" of+        Nothing -> assertion+        Just "=" -> assertion+        Just "==" -> assertion {batotal=True}+        Just "=*" -> assertion {bainclusive=True}+        Just "==*" -> assertion{batotal=True, bainclusive=True}+        Just x -> error' $ unlines+          [ "balance-type \"" ++ x ++"\" is invalid. Use =, ==, =* or ==*." +          , showRecord record+          , showRules rules record+          ]+        +    toAssertion (a, b) = defaultAssertion{       baamount   = a,       baposition = b       } -getAmountStr :: CsvRules -> CsvRecord -> Maybe String-getAmountStr rules record =+chooseAmount :: CsvRules -> CsvRecord -> String -> String -> String -> String -> Maybe MixedAmount+chooseAmount rules record currency amountFld amountInFld amountOutFld =  let-   mamount    = getEffectiveAssignment rules record "amount"-   mamountin  = getEffectiveAssignment rules record "amount-in"-   mamountout = getEffectiveAssignment rules record "amount-out"-   render     = fmap (strip . renderTemplate rules record)+   mamount    = getEffectiveAssignment rules record amountFld+   mamountin  = getEffectiveAssignment rules record amountInFld+   mamountout = getEffectiveAssignment rules record amountOutFld+   parse  amt = notZero =<< (parseAmount currency <$> notEmpty =<< (strip . renderTemplate rules record) <$> amt)  in-  case (render mamount, render mamountin, render mamountout) of-    (Just "", Nothing, Nothing) -> Nothing+  case (parse mamount, parse mamountin, parse mamountout) of+    (Nothing, Nothing, Nothing) -> Nothing     (Just a,  Nothing, Nothing) -> Just a-    (Nothing, Just "", Just "") -> error' $    "neither amount-in or amount-out has a value\n"-                                            ++ "    record: " ++ showRecord record-    (Nothing, Just i,  Just "") -> Just i-    (Nothing, Just "", Just o)  -> Just $ negateStr o-    (Nothing, Just i,  Just o)  -> error' $    "both amount-in and amount-out have a value\n"-                                            ++ "    amount-in: "  ++ i ++ "\n"-                                            ++ "    amount-out: " ++ o ++ "\n"+    (Nothing, Just i,  Nothing) -> Just i+    (Nothing, Nothing, Just o)  -> Just $ negate o+    (Nothing, Just i,  Just o)  -> error' $    "both "++amountInFld++" and "++amountOutFld++" have a value\n"+                                            ++ "    "++amountInFld++": "  ++ show i ++ "\n"+                                            ++ "    "++amountOutFld++": " ++ show o ++ "\n"                                             ++ "    record: "     ++ showRecord record-    _                           -> error' $    "found values for amount and for amount-in/amount-out\n"-                                            ++ "please use either amount or amount-in/amount-out\n"+    _                           -> error' $    "found values for "++amountFld++" and for "++amountInFld++"/"++amountOutFld++"\n"+                                            ++ "please use either "++amountFld++" or "++amountInFld++"/"++amountOutFld++"\n"                                             ++ "    record: " ++ showRecord record+ where+   notZero amt = if isZeroMixedAmount amt then Nothing else Just amt+   notEmpty str = if str=="" then Nothing else Just str +   parseAmount currency amountstr =+     either (amounterror amountstr) (Mixed . (:[]))+     <$> runParser (evalStateT (amountp <* eof) mempty) ""+     <$> T.pack+     <$> (currency++)+     <$> simplifySign+     <$> amountstr++   amounterror amountstr err = error' $ unlines+     ["error: could not parse \""++fromJust amountstr++"\" as an amount"+     ,showRecord record+     ,showRules rules record+     ,"the default-currency is: "++fromMaybe "unspecified" (getDirective "default-currency" rules)+     ,"the parse error is:      "++customErrorBundlePretty err+     ,"you may need to "+      ++"change your amount or currency rules, "+      ++"or add or change your skip rule"+     ]+ type CsvAmountString = String  -- | Canonicalise the sign in a CSV amount string.@@ -853,7 +947,7 @@ -- | Render a field assigment's template, possibly interpolating referenced -- CSV field values. Outer whitespace is removed from interpolated values. renderTemplate ::  CsvRules -> CsvRecord -> FieldTemplate -> String-renderTemplate rules record t = regexReplaceBy "%[A-z0-9]+" replace t+renderTemplate rules record t = regexReplaceBy "%[A-z0-9_-]+" replace t   where     replace ('%':pat) = maybe pat (\i -> strip $ atDef "" record (i-1)) mindex       where@@ -891,22 +985,26 @@  tests_CsvReader = tests "CsvReader" [    tests "parseCsvRules" [-     test "empty file" $-      parseCsvRules "unknown" "" `is` Right rules+     test"empty file" $+      parseCsvRules "unknown" "" @?= Right defrules     ]   ,tests "rulesp" [-     test "trailing comments" $-      parseWithState' rules rulesp "skip\n# \n#\n" `is` Right rules{rdirectives = [("skip","")]}--    ,test "trailing blank lines" $-      parseWithState' rules rulesp "skip\n\n  \n" `is` (Right rules{rdirectives = [("skip","")]})+     test"trailing comments" $+      parseWithState' defrules rulesp "skip\n# \n#\n" @?= Right defrules{rdirectives = [("skip","")]} -    ,test "no final newline" $-      parseWithState' rules rulesp "skip" `is` (Right rules{rdirectives=[("skip","")]})+    ,test"trailing blank lines" $+      parseWithState' defrules rulesp "skip\n\n  \n" @?= (Right defrules{rdirectives = [("skip","")]}) -    ,test "assignment with empty value" $-      parseWithState' rules rulesp "account1 \nif foo\n  account2 foo\n" `is`-        (Right rules{rassignments = [("account1","")], rconditionalblocks = [([["foo"]],[("account2","foo")])]})+    ,test"no final newline" $+      parseWithState' defrules rulesp "skip" @?= (Right defrules{rdirectives=[("skip","")]}) -    ]+    ,test"assignment with empty value" $+      parseWithState' defrules rulesp "account1 \nif foo\n  account2 foo\n" @?=+        (Right defrules{rassignments = [("account1","")], rconditionalblocks = [([["foo"]],[("account2","foo")])]})+  ]+  ,tests "conditionalblockp" [+    test"space after conditional" $ -- #1120+      parseWithState' defrules conditionalblockp "if a\n account2 b\n \n" @?=+        (Right ([["a"]],[("account2","b")]))+  ]   ]

@@ -63,13 +63,17 @@ ) where --- * imports-import Prelude ()-import "base-compat-batteries" Prelude.Compat hiding (readFile)+import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail (fail) import qualified Control.Exception as C-import Control.Monad+import Control.Monad (forM_, when, void)+import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Except (ExceptT(..), runExceptT)-import Control.Monad.State.Strict+import Control.Monad.State.Strict (get,modify',put)+import Control.Monad.Trans.Class (lift) import qualified Data.Map.Strict as M+#if !(MIN_VERSION_base(4,11,0))+import Data.Monoid ((<>))+#endif import Data.Text (Text) import Data.String import Data.List@@ -215,7 +219,7 @@        let parentfilestack = jincludefilestack parentj       when (filepath `elem` parentfilestack) $-        fail ("Cyclic include: " ++ filepath)+        Fail.fail ("Cyclic include: " ++ filepath)        childInput <- lift $ readFilePortably filepath                             `orRethrowIOError` (show parentpos ++ " reading " ++ filepath)@@ -251,7 +255,7 @@   eResult <- liftIO $ (Right <$> io) `C.catch` \(e::C.IOException) -> pure $ Left $ printf "%s:\n%s" msg (show e)   case eResult of     Right res -> pure res-    Left errMsg -> fail errMsg+    Left errMsg -> Fail.fail errMsg  -- Parse an account directive, adding its info to the journal's -- list of account declarations.@@ -664,46 +668,46 @@     let p = lift accountnamep :: JournalParser IO AccountName in    tests "accountnamep" [-     test "basic" $ expectParse p "a:b:c"-    ,_test "empty inner component" $ expectParseError p "a::c" ""  -- TODO-    ,_test "empty leading component" $ expectParseError p ":b:c" "x"-    ,_test "empty trailing component" $ expectParseError p "a:b:" "x"+     test "basic" $ assertParse p "a:b:c"+    -- ,test "empty inner component" $ assertParseError p "a::c" ""  -- TODO+    -- ,test "empty leading component" $ assertParseError p ":b:c" "x"+    -- ,test "empty trailing component" $ assertParseError p "a:b:" "x"     ]    -- "Parse a date in YYYY/MM/DD format.   -- Hyphen (-) and period (.) are also allowed as separators.   -- The year may be omitted if a default year has been set.   -- Leading zeroes may be omitted."-  ,test "datep" $ do-    test "YYYY/MM/DD" $ expectParseEq datep "2018/01/01" (fromGregorian 2018 1 1)-    test "YYYY-MM-DD" $ expectParse datep "2018-01-01"-    test "YYYY.MM.DD" $ expectParse datep "2018.01.01"-    test "yearless date with no default year" $ expectParseError datep "1/1" "current year is unknown"-    test "yearless date with default year" $ do+  ,tests "datep" [+     test "YYYY/MM/DD" $ assertParseEq datep "2018/01/01" (fromGregorian 2018 1 1)+    ,test "YYYY-MM-DD" $ assertParse datep "2018-01-01"+    ,test "YYYY.MM.DD" $ assertParse datep "2018.01.01"+    ,test "yearless date with no default year" $ assertParseError datep "1/1" "current year is unknown"+    ,test "yearless date with default year" $ do       let s = "1/1"       ep <- parseWithState mempty{jparsedefaultyear=Just 2018} datep s-      either (fail.("parse error at "++).customErrorBundlePretty) (const ok) ep-    test "no leading zero" $ expectParse datep "2018/1/1"-+      either (assertFailure . ("parse error at "++) . customErrorBundlePretty) (const $ return ()) ep+    ,test "no leading zero" $ assertParse datep "2018/1/1"+    ]   ,test "datetimep" $ do-      let-        good = expectParse datetimep-        bad = (\t -> expectParseError datetimep t "")-      good "2011/1/1 00:00"-      good "2011/1/1 23:59:59"-      bad "2011/1/1"-      bad "2011/1/1 24:00:00"-      bad "2011/1/1 00:60:00"-      bad "2011/1/1 00:00:60"-      bad "2011/1/1 3:5:7"-      test "timezone is parsed but ignored" $ do-        let t = LocalTime (fromGregorian 2018 1 1) (TimeOfDay 0 0 (fromIntegral 0))-        expectParseEq datetimep "2018/1/1 00:00-0800" t-        expectParseEq datetimep "2018/1/1 00:00+1234" t+     let+       good = assertParse datetimep+       bad  = (\t -> assertParseError datetimep t "")+     good "2011/1/1 00:00"+     good "2011/1/1 23:59:59"+     bad "2011/1/1"+     bad "2011/1/1 24:00:00"+     bad "2011/1/1 00:60:00"+     bad "2011/1/1 00:00:60"+     bad "2011/1/1 3:5:7"+     -- timezone is parsed but ignored+     let t = LocalTime (fromGregorian 2018 1 1) (TimeOfDay 0 0 (fromIntegral 0))+     assertParseEq datetimep "2018/1/1 00:00-0800" t+     assertParseEq datetimep "2018/1/1 00:00+1234" t    ,tests "periodictransactionp" [ -    test "more period text in comment after one space" $ expectParseEq periodictransactionp+    test "more period text in comment after one space" $ assertParseEq periodictransactionp       "~ monthly from 2018/6 ;In 2019 we will change this\n"       nullperiodictransaction {          ptperiodexpr  = "monthly from 2018/6"@@ -713,7 +717,7 @@         ,ptcomment     = "In 2019 we will change this\n"         } -    ,test "more period text in description after two spaces" $ expectParseEq periodictransactionp+    ,test "more period text in description after two spaces" $ assertParseEq periodictransactionp       "~ monthly from 2018/6   In 2019 we will change this\n"       nullperiodictransaction {          ptperiodexpr  = "monthly from 2018/6"@@ -723,7 +727,7 @@         ,ptcomment     = ""         } -    ,test "Next year in description" $ expectParseEq periodictransactionp+    ,test "Next year in description" $ assertParseEq periodictransactionp       "~ monthly  Next year blah blah\n"       nullperiodictransaction {          ptperiodexpr  = "monthly"@@ -733,7 +737,7 @@         ,ptcomment     = ""         } -    ,test "Just date, no description" $ expectParseEq periodictransactionp+    ,test "Just date, no description" $ assertParseEq periodictransactionp       "~ 2019-01-04\n"       nullperiodictransaction {          ptperiodexpr  = "2019-01-04"@@ -743,13 +747,13 @@         ,ptcomment     = ""         } -    ,test "Just date, no description + empty transaction comment" $ expectParse periodictransactionp+    ,test "Just date, no description + empty transaction comment" $ assertParse periodictransactionp       "~ 2019-01-04\n  ;\n  a  1\n  b\n"      ]    ,tests "postingp" [-     test "basic" $ expectParseEq (postingp Nothing)+     test "basic" $ assertParseEq (postingp Nothing)       "  expenses:food:dining  $10.00   ; a: a a \n   ; b: b b \n"       posting{         paccount="expenses:food:dining",@@ -758,7 +762,7 @@         ptags=[("a","a a"), ("b","b b")]         } -    ,test "posting dates" $ expectParseEq (postingp Nothing)+    ,test "posting dates" $ assertParseEq (postingp Nothing)       " a  1. ; date:2012/11/28, date2=2012/11/29,b:b\n"       nullposting{          paccount="a"@@ -769,7 +773,7 @@         ,pdate2=Nothing  -- Just $ fromGregorian 2012 11 29         } -    ,test "posting dates bracket syntax" $ expectParseEq (postingp Nothing)+    ,test "posting dates bracket syntax" $ assertParseEq (postingp Nothing)       " a  1. ; [2012/11/28=2012/11/29]\n"       nullposting{          paccount="a"@@ -780,16 +784,16 @@         ,pdate2=Just $ fromGregorian 2012 11 29         } -    ,test "quoted commodity symbol with digits" $ expectParse (postingp Nothing) "  a  1 \"DE123\"\n"+    ,test "quoted commodity symbol with digits" $ assertParse (postingp Nothing) "  a  1 \"DE123\"\n" -    ,test "balance assertion and fixed lot price" $ expectParse (postingp Nothing) "  a  1 \"DE123\" =$1 { =2.2 EUR} \n"+    ,test "balance assertion and fixed lot price" $ assertParse (postingp Nothing) "  a  1 \"DE123\" =$1 { =2.2 EUR} \n" -    ,test "balance assertion over entire contents of account" $ expectParse (postingp Nothing) "  a  $1 == $1\n"+    ,test "balance assertion over entire contents of account" $ assertParse (postingp Nothing) "  a  $1 == $1\n"     ]    ,tests "transactionmodifierp" [ -    test "basic" $ expectParseEq transactionmodifierp+    test "basic" $ assertParseEq transactionmodifierp       "= (some value expr)\n some:postings  1.\n"       nulltransactionmodifier {         tmquerytxt = "(some value expr)"@@ -799,9 +803,9 @@    ,tests "transactionp" [ -     test "just a date" $ expectParseEq transactionp "2015/1/1\n" nulltransaction{tdate=fromGregorian 2015 1 1}+     test "just a date" $ assertParseEq transactionp "2015/1/1\n" nulltransaction{tdate=fromGregorian 2015 1 1} -    ,test "more complex" $ expectParseEq transactionp+    ,test "more complex" $ assertParseEq transactionp       (T.unlines [         "2012/05/14=2012/05/15 (code) desc  ; tcomment1",         "    ; tcomment2",@@ -836,7 +840,7 @@       }      ,test "parses a well-formed transaction" $-      expect $ isRight $ rjp transactionp $ T.unlines+      assertBool "" $ isRight $ rjp transactionp $ T.unlines         ["2007/01/28 coopportunity"         ,"    expenses:food:groceries                   $47.18"         ,"    assets:checking                          $-47.18"@@ -844,18 +848,18 @@         ]      ,test "does not parse a following comment as part of the description" $-      expectParseEqOn transactionp "2009/1/1 a ;comment\n b 1\n" tdescription "a"+      assertParseEqOn transactionp "2009/1/1 a ;comment\n b 1\n" tdescription "a" -    ,test "transactionp parses a following whitespace line" $-      expect $ isRight $ rjp transactionp $ T.unlines+    ,test "parses a following whitespace line" $+      assertBool "" $ isRight $ rjp transactionp $ T.unlines         ["2012/1/1"         ,"  a  1"         ,"  b"         ," "         ] -    ,test "transactionp parses an empty transaction comment following whitespace line" $-      expect $ isRight $ rjp transactionp $ T.unlines+    ,test "parses an empty transaction comment following whitespace line" $+      assertBool "" $ isRight $ rjp transactionp $ T.unlines         ["2012/1/1"         ,"  ;"         ,"  a  1"@@ -864,7 +868,7 @@         ]      ,test "comments everywhere, two postings parsed" $-      expectParseEqOn transactionp+      assertParseEqOn transactionp         (T.unlines           ["2009/1/1 x  ; transaction comment"           ," a  1  ; posting 1 comment"@@ -881,42 +885,45 @@    ,tests "directivep" [     test "supports !" $ do-      expectParseE directivep "!account a\n"-      expectParseE directivep "!D 1.0\n"-    ]+        assertParseE directivep "!account a\n"+        assertParseE directivep "!D 1.0\n"+     ] -  ,test "accountdirectivep" $ do-    test "with-comment"       $ expectParse accountdirectivep "account a:b  ; a comment\n"-    test "does-not-support-!" $ expectParseError accountdirectivep "!account a:b\n" ""-    test "account-type-code"  $ expectParse accountdirectivep "account a:b  A\n"-    test "account-type-tag"   $ expectParseStateOn accountdirectivep "account a:b  ; type:asset\n"-      jdeclaredaccounts-      [("a:b", AccountDeclarationInfo{adicomment          = "type:asset\n"-                                     ,aditags             = [("type","asset")]-                                     ,adideclarationorder = 1-                                     })+  ,tests "accountdirectivep" [+       test "with-comment"       $ assertParse accountdirectivep "account a:b  ; a comment\n"+      ,test "does-not-support-!" $ assertParseError accountdirectivep "!account a:b\n" ""+      ,test "account-type-code"  $ assertParse accountdirectivep "account a:b  A\n"+      ,test "account-type-tag"   $ assertParseStateOn accountdirectivep "account a:b  ; type:asset\n"+        jdeclaredaccounts+        [("a:b", AccountDeclarationInfo{adicomment          = "type:asset\n"+                                       ,aditags             = [("type","asset")]+                                       ,adideclarationorder = 1+                                       })+        ]       ]    ,test "commodityconversiondirectivep" $ do-     expectParse commodityconversiondirectivep "C 1h = $50.00\n"+     assertParse commodityconversiondirectivep "C 1h = $50.00\n"    ,test "defaultcommoditydirectivep" $ do-     expectParse defaultcommoditydirectivep "D $1,000.0\n"-     expectParseError defaultcommoditydirectivep "D $1000\n" "please include a decimal separator"+      assertParse defaultcommoditydirectivep "D $1,000.0\n"+      assertParseError defaultcommoditydirectivep "D $1000\n" "please include a decimal separator" -  ,test "defaultyeardirectivep" $ do-    test "1000" $ expectParse defaultyeardirectivep "Y 1000" -- XXX no \n like the others-    test "999" $ expectParseError defaultyeardirectivep "Y 999" "bad year number"-    test "12345" $ expectParse defaultyeardirectivep "Y 12345"+  ,tests "defaultyeardirectivep" [+      test "1000" $ assertParse defaultyeardirectivep "Y 1000" -- XXX no \n like the others+     ,test "999" $ assertParseError defaultyeardirectivep "Y 999" "bad year number"+     ,test "12345" $ assertParse defaultyeardirectivep "Y 12345"+     ]    ,test "ignoredpricecommoditydirectivep" $ do-     expectParse ignoredpricecommoditydirectivep "N $\n"+     assertParse ignoredpricecommoditydirectivep "N $\n" -  ,test "includedirectivep" $ do-    test "include" $ expectParseErrorE includedirectivep "include nosuchfile\n" "No existing files match pattern: nosuchfile"-    test "glob" $ expectParseErrorE includedirectivep "include nosuchfile*\n" "No existing files match pattern: nosuchfile*"+  ,tests "includedirectivep" [+      test "include" $ assertParseErrorE includedirectivep "include nosuchfile\n" "No existing files match pattern: nosuchfile"+     ,test "glob" $ assertParseErrorE includedirectivep "include nosuchfile*\n" "No existing files match pattern: nosuchfile*"+     ] -  ,test "marketpricedirectivep" $ expectParseEq marketpricedirectivep+  ,test "marketpricedirectivep" $ assertParseEq marketpricedirectivep     "P 2017/01/30 BTC $922.83\n"     PriceDirective{       pddate      = fromGregorian 2017 1 30,@@ -925,23 +932,20 @@       }    ,test "tagdirectivep" $ do-     expectParse tagdirectivep "tag foo \n"+     assertParse tagdirectivep "tag foo \n"    ,test "endtagdirectivep" $ do-     expectParse endtagdirectivep "end tag \n"-     expectParse endtagdirectivep "pop \n"-+      assertParse endtagdirectivep "end tag \n"+      assertParse endtagdirectivep "pop \n"    ,tests "journalp" [-    test "empty file" $ expectParseEqE journalp "" nulljournal+    test "empty file" $ assertParseEqE journalp "" nulljournal     ]     -- these are defined here rather than in Common so they can use journalp-  ,tests "parseAndFinaliseJournal" [-    test "basic" $ do-        ej <- io $ runExceptT $ parseAndFinaliseJournal journalp definputopts "" "2019-1-1\n"-        let Right j = ej-        expectEqPP [""] $ journalFilePaths j-   ]+  ,test "parseAndFinaliseJournal" $ do+      ej <- runExceptT $ parseAndFinaliseJournal journalp definputopts "" "2019-1-1\n"+      let Right j = ej+      assertEqual "" [""] $ journalFilePaths j    ]

@@ -82,26 +82,44 @@ balancelabel = "Historical Total"  accountTransactionsReport :: ReportOpts -> Journal -> Query -> Query -> AccountTransactionsReport-accountTransactionsReport opts j reportq thisacctq = (label, items)+accountTransactionsReport ropts j reportq thisacctq = (label, items)   where     -- a depth limit does not affect the account transactions report     -- seems unnecessary for some reason XXX     reportq' = -- filterQuery (not . queryIsDepth)                reportq-    -- get all transactions, with amounts converted to cost basis if -B-    ts1 = jtxns $ journalSelectingAmountFromOpts opts j++    -- get all transactions+    ts1 = jtxns j+     -- apply any cur:SYM filters in reportq'     symq  = filterQuery queryIsSym reportq'     ts2 = (if queryIsNull symq then id else map (filterTransactionAmounts symq)) ts1+     -- keep just the transactions affecting this account (via possibly realness or status-filtered postings)     realq = filterQuery queryIsReal reportq'     statusq = filterQuery queryIsStatus reportq'     ts3 = filter (matchesTransaction thisacctq . filterTransactionPostings (And [realq, statusq])) ts2++    -- maybe convert these transactions to cost or value+    prices = journalPriceOracle j+    styles = journalCommodityStyles j+    periodlast =+      fromMaybe (error' "journalApplyValuation: expected a non-empty journal") $ -- XXX shouldn't happen+      reportPeriodOrJournalLastDay ropts j+    mreportlast = reportPeriodLastDay ropts+    today = fromMaybe (error' "journalApplyValuation: could not pick a valuation date, ReportOpts today_ is unset") $ today_ ropts+    multiperiod = interval_ ropts /= NoInterval+    tval = case value_ ropts of+             Just v  -> \t -> transactionApplyValuation prices styles periodlast mreportlast today multiperiod t v+             Nothing -> id+    ts4 = map tval ts3 +     -- sort by the transaction's register date, for accurate starting balance-    ts = sortBy (comparing (transactionRegisterDate reportq' thisacctq)) ts3+    ts = sortBy (comparing (transactionRegisterDate reportq' thisacctq)) ts4      (startbal,label)-      | balancetype_ opts == HistoricalBalance = (sumPostings priorps, balancelabel)+      | balancetype_ ropts == HistoricalBalance = (sumPostings priorps, balancelabel)       | otherwise                              = (nullmixedamt,        totallabel)       where         priorps = dbg1 "priorps" $@@ -113,7 +131,7 @@           case mstartdate of             Just _  -> Date (DateSpan Nothing mstartdate)             Nothing -> None  -- no start date specified, there are no prior postings-        mstartdate = queryStartDate (date2_ opts) reportq'+        mstartdate = queryStartDate (date2_ ropts) reportq'         datelessreportq = filterQuery (not . queryIsDateOrDate2) reportq'      items = reverse $

@@ -12,6 +12,8 @@   balanceReport,   flatShowsExclusiveBalance,   sortAccountItemsLike,+  unifyMixedAmount,+  perdivide,    -- * Tests   tests_BalanceReport@@ -66,7 +68,7 @@ balanceReport :: ReportOpts -> Query -> Journal -> BalanceReport balanceReport ropts@ReportOpts{..} q j@Journal{..} =   (if invert_ then brNegate  else id) $-  (sorteditems, total)+  (mappedsorteditems, mappedtotal)     where       -- dbg1 = const id -- exclude from debug output       dbg1 s = let p = "balanceReport" in Hledger.Utils.dbg1 (p++" "++s)  -- add prefix in debug output@@ -142,6 +144,14 @@                                   if flatShowsExclusiveBalance                                   then sum $ map fourth4 items                                   else sum $ map aebalance $ clipAccountsAndAggregate 1 displayaccts+      +      -- Calculate percentages if needed.+      mappedtotal | percent_  = dbg1 "mappedtotal" $ total `perdivide` total+                  | otherwise = total+      mappedsorteditems | percent_ =+                            dbg1 "mappedsorteditems" $+                            map (\(fname, sname, indent, amount) -> (fname, sname, indent, amount `perdivide` total)) sorteditems+                        | otherwise = sorteditems  -- | A sorting helper: sort a list of things (eg report rows) keyed by account name -- to match the provided ordering of those same account names.@@ -185,7 +195,33 @@   where     brItemNegate (a, a', d, amt) = (a, a', d, -amt) +-- | Helper to unify a MixedAmount to a single commodity value.+-- Like normaliseMixedAmount, this consolidates amounts of the same commodity+-- and discards zero amounts; but this one insists on simplifying to+-- a single commodity, and will throw a program-terminating error if+-- this is not possible.+unifyMixedAmount :: MixedAmount -> Amount+unifyMixedAmount mixedAmount = foldl combine (num 0) (amounts mixedAmount)+  where+    combine amount result =+      if isReallyZeroAmount amount+      then result+      else if isReallyZeroAmount result+        then amount+        else if acommodity amount == acommodity result+          then amount + result+          else error' "Cannot calculate percentages for accounts with multiple commodities. (Hint: Try --cost, -V or similar flags.)" +-- | Helper to calculate the percentage from two mixed. Keeps the sign of the first argument.+-- Uses unifyMixedAmount to unify each argument and then divides them.+perdivide :: MixedAmount -> MixedAmount -> MixedAmount+perdivide a b =+  let a' = unifyMixedAmount a+      b' = unifyMixedAmount b+  in if isReallyZeroAmount a' || isReallyZeroAmount b' || acommodity a' == acommodity b'+    then mixed [per $ if aquantity b' == 0 then 0 else (aquantity a' / abs (aquantity b') * 100)]+    else error' "Cannot calculate percentages if accounts have different commodities. (Hint: Try --cost, -V or similar flags.)"+ -- tests  Right samplejournal2 =@@ -212,21 +248,21 @@     }  tests_BalanceReport = tests "BalanceReport" [-  tests "balanceReport" $-    let-      (opts,journal) `gives` r = do-        let (eitems, etotal) = r-            (aitems, atotal) = balanceReport opts (queryFromOpts nulldate opts) journal-            showw (acct,acct',indent,amt) = (acct, acct', indent, showMixedAmountDebug amt)-        (map showw eitems) `is` (map showw aitems)-        (showMixedAmountDebug etotal) `is` (showMixedAmountDebug atotal)-      usd0 = usd 0-    in [ -     test "balanceReport with no args on null journal" $+  let+    (opts,journal) `gives` r = do+      let (eitems, etotal) = r+          (aitems, atotal) = balanceReport opts (queryFromOpts nulldate opts) journal+          showw (acct,acct',indent,amt) = (acct, acct', indent, showMixedAmountDebug amt)+      (map showw eitems) @?= (map showw aitems)+      (showMixedAmountDebug etotal) @?= (showMixedAmountDebug atotal)+  in+    tests "balanceReport" [++     test "no args, null journal" $      (defreportopts, nulljournal) `gives` ([], Mixed [nullamt]) -    ,test "balanceReport with no args on sample journal" $+    ,test "no args, sample journal" $      (defreportopts, samplejournal) `gives`       ([         ("assets","assets",0, mamountp' "$0.00")@@ -241,45 +277,46 @@        ,("income:gifts","gifts",1, mamountp' "$-1.00")        ,("income:salary","salary",1, mamountp' "$-1.00")        ],-       Mixed [usd0])+       Mixed [usd 0]) -    ,test "balanceReport with --depth=N" $+    ,test "with --depth=N" $      (defreportopts{depth_=Just 1}, samplejournal) `gives`       ([        ("expenses",    "expenses",    0, mamountp'  "$2.00")        ,("income",      "income",      0, mamountp' "$-2.00")        ],-       Mixed [usd0])+       Mixed [usd 0]) -    ,test "balanceReport with depth:N" $+    ,test "with depth:N" $      (defreportopts{query_="depth:1"}, samplejournal) `gives`       ([        ("expenses",    "expenses",    0, mamountp'  "$2.00")        ,("income",      "income",      0, mamountp' "$-2.00")        ],-       Mixed [usd0])+       Mixed [usd 0]) -    ,tests "balanceReport with a date or secondary date span" [+    ,test "with date:" $      (defreportopts{query_="date:'in 2009'"}, samplejournal2) `gives`       ([],        Mixed [nullamt])-     ,(defreportopts{query_="date2:'in 2009'"}, samplejournal2) `gives`++    ,test "with date2:" $+     (defreportopts{query_="date2:'in 2009'"}, samplejournal2) `gives`       ([         ("assets:bank:checking","assets:bank:checking",0,mamountp' "$1.00")        ,("income:salary","income:salary",0,mamountp' "$-1.00")        ],-       Mixed [usd0])-     ]+       Mixed [usd 0]) -    ,test "balanceReport with desc:" $+    ,test "with desc:" $      (defreportopts{query_="desc:income"}, samplejournal) `gives`       ([         ("assets:bank:checking","assets:bank:checking",0,mamountp' "$1.00")        ,("income:salary","income:salary",0, mamountp' "$-1.00")        ],-       Mixed [usd0])+       Mixed [usd 0]) -    ,test "balanceReport with not:desc:" $+    ,test "with not:desc:" $      (defreportopts{query_="not:desc:income"}, samplejournal) `gives`       ([         ("assets","assets",0, mamountp' "$-1.00")@@ -290,18 +327,18 @@        ,("expenses:supplies","supplies",1, mamountp' "$1.00")        ,("income:gifts","income:gifts",0, mamountp' "$-1.00")        ],-       Mixed [usd0])+       Mixed [usd 0]) -    ,test "balanceReport with period on a populated period" $+    ,test "with period on a populated period" $       (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 1) (fromGregorian 2008 1 2)}, samplejournal) `gives`        (         [          ("assets:bank:checking","assets:bank:checking",0, mamountp' "$1.00")         ,("income:salary","income:salary",0, mamountp' "$-1.00")         ],-        Mixed [usd0])+        Mixed [usd 0]) -     ,test "balanceReport with period on an unpopulated period" $+     ,test "with period on an unpopulated period" $       (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 2) (fromGregorian 2008 1 3)}, samplejournal) `gives`        ([],Mixed [nullamt]) @@ -413,7 +450,7 @@                 ,"  a:b          10h @ $50"                 ,"  c:d                   "                 ]) >>= either error' return-         let j' = journalCanonicaliseAmounts $ journalConvertAmountsToCost j -- enable cost basis adjustment+         let j' = journalCanonicaliseAmounts $ journalToCost j -- enable cost basis adjustment          balanceReportAsText defreportopts (balanceReport defreportopts Any j') `is`            ["                $500  a:b"            ,"               $-500  c:d"@@ -421,7 +458,7 @@            ,"                   0"            ]   -}-   ]+     ]   ] 

@@ -81,8 +81,8 @@       concatMap tpostings $       concatMap (flip runPeriodicTransaction reportspan) $       jperiodictxns j-    actualj = dbg1 "actualj" $ budgetRollUp budgetedaccts showunbudgeted j-    budgetj = dbg1 "budgetj" $ budgetJournal assrt ropts reportspan j+    actualj = dbg1With (("actualj"++).show.jtxns)  $ budgetRollUp budgetedaccts showunbudgeted j+    budgetj = dbg1With (("budgetj"++).show.jtxns)  $ budgetJournal assrt ropts reportspan j     actualreport@(MultiBalanceReport (actualspans, _, _)) = dbg1 "actualreport" $ multiBalanceReport ropts  q actualj     budgetgoalreport@(MultiBalanceReport (_, budgetgoalitems, budgetgoaltotals)) = dbg1 "budgetgoalreport" $ multiBalanceReport (ropts{empty_=True}) q budgetj     budgetgoalreport'@@ -286,11 +286,11 @@         Just (AtDate d _mc) -> ", valued at "++showDate d         Nothing             -> "")     actualwidth =-      maximum [ maybe 0 (length . showMixedAmountOneLineWithoutPrice) amt+      maximum' [ maybe 0 (length . showMixedAmountOneLineWithoutPrice) amt       | (_, _, _, amtandgoals, _, _) <- rows       , (amt, _) <- amtandgoals ]     budgetwidth =-      maximum [ maybe 0 (length . showMixedAmountOneLineWithoutPrice) goal+      maximum' [ maybe 0 (length . showMixedAmountOneLineWithoutPrice) goal       | (_, _, _, amtandgoals, _, _) <- rows       , (_, goal) <- amtandgoals ]     -- XXX lay out actual, percentage and/or goal in the single table cell for now, should probably use separate cells

@@ -49,8 +49,8 @@  tests_EntriesReport = tests "EntriesReport" [   tests "entriesReport" [-     test "not acct" $ (length $ entriesReport defreportopts (Not $ Acct "bank") samplejournal) `is` 1-    ,test "date" $ (length $ entriesReport defreportopts (Date $ mkdatespan "2008/06/01" "2008/07/01") samplejournal) `is` 3+     test "not acct" $ (length $ entriesReport defreportopts (Not $ Acct "bank") samplejournal) @?= 1+    ,test "date" $ (length $ entriesReport defreportopts (Date $ mkdatespan "2008/06/01" "2008/07/01") samplejournal) @?= 3   ]  ] 

@@ -106,7 +106,7 @@ multiBalanceReportWith :: ReportOpts -> Query -> Journal -> PriceOracle -> MultiBalanceReport multiBalanceReportWith ropts@ReportOpts{..} q j@Journal{..} priceoracle =   (if invert_ then mbrNegate else id) $-  MultiBalanceReport (colspans, sortedrows, totalsrow)+  MultiBalanceReport (colspans, mappedsortedrows, mappedtotalsrow)     where       dbg1 s = let p = "multiBalanceReport" in Hledger.Utils.dbg1 (p++" "++s)  -- add prefix in this function's debug output       -- dbg1 = const id  -- exclude this function from debug output@@ -162,7 +162,7 @@       -- These balances are unvalued except maybe converted to cost.       startbals :: [(AccountName, MixedAmount)] = dbg1 "startbals" $ map (\(a,_,_,b) -> (a,b)) startbalanceitems         where-          (startbalanceitems,_) = dbg1 "starting balance report" $ balanceReport ropts''{value_=Nothing} startbalq j'+          (startbalanceitems,_) = dbg1 "starting balance report" $ balanceReport ropts''{value_=Nothing, percent_=False} startbalq j'             where               ropts' | tree_ ropts = ropts{no_elide_=True}                      | otherwise   = ropts{accountlistmode_=ALFlat}@@ -344,6 +344,25 @@       totalsrow :: MultiBalanceReportTotals =         dbg1 "totalsrow" (coltotals, grandtotal, grandaverage) +      ----------------------------------------------------------------------+      -- 9. Map the report rows to percentages if needed+      -- It is not correct to do this before step 6 due to the total and average columns.+      -- This is not done in step 6, since the report totals are calculated in 8.+      +      -- Perform the divisions to obtain percentages+      mappedsortedrows :: [MultiBalanceReportRow] =+        if not percent_ then sortedrows+        else dbg1 "mappedsortedrows"+          [(aname, alname, alevel, zipWith perdivide rowvals coltotals, rowtotal `perdivide` grandtotal, rowavg `perdivide` grandaverage)+           | (aname, alname, alevel, rowvals, rowtotal, rowavg) <- sortedrows+          ]+      mappedtotalsrow :: MultiBalanceReportTotals =+        if not percent_ then totalsrow+        else dbg1 "mappedtotalsrow" (+          map (\t -> perdivide t t) coltotals,+          perdivide grandtotal grandtotal,+          perdivide grandaverage grandaverage)+ -- | Given a MultiBalanceReport and its normal balance sign, -- if it is known to be normally negative, convert it to normally positive. mbrNormaliseSign :: NormalSign -> MultiBalanceReport -> MultiBalanceReport@@ -397,15 +416,15 @@ -- tests  tests_MultiBalanceReport = tests "MultiBalanceReport" [+   let+    amt0 = Amount {acommodity="$", aquantity=0, aprice=Nothing, astyle=AmountStyle {ascommodityside = L, ascommodityspaced = False, asprecision = 2, asdecimalpoint = Just '.', asdigitgroups = Nothing}, aismultiplier=False}     (opts,journal) `gives` r = do       let (eitems, etotal) = r           (MultiBalanceReport (_, aitems, atotal)) = multiBalanceReport opts (queryFromOpts nulldate opts) journal           showw (acct,acct',indent,lAmt,amt,amt') = (acct, acct', indent, map showMixedAmountDebug lAmt, showMixedAmountDebug amt, showMixedAmountDebug amt')-      (map showw aitems) `is` (map showw eitems)-      ((\(_, b, _) -> showMixedAmountDebug b) atotal) `is` (showMixedAmountDebug etotal) -- we only check the sum of the totals-    usd0 = usd 0-    amount0 = Amount {acommodity="$", aquantity=0, aprice=Nothing, astyle=AmountStyle {ascommodityside = L, ascommodityspaced = False, asprecision = 2, asdecimalpoint = Just '.', asdigitgroups = Nothing}, aismultiplier=False}+      (map showw aitems) @?= (map showw eitems)+      ((\(_, b, _) -> showMixedAmountDebug b) atotal) @?= (showMixedAmountDebug etotal) -- we only check the sum of the totals   in    tests "multiBalanceReport" [       test "null journal"  $@@ -415,32 +434,32 @@       (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 1) (fromGregorian 2008 1 2), balancetype_=HistoricalBalance}, samplejournal) `gives`        (         [-         ("assets:bank:checking", "checking", 3, [mamountp' "$1.00"] , Mixed [nullamt], Mixed [amount0 {aquantity=1}])-        ,("income:salary"       ,"salary"   , 2, [mamountp' "$-1.00"], Mixed [nullamt], Mixed [amount0 {aquantity=(-1)}])+         ("assets:bank:checking", "checking", 3, [mamountp' "$1.00"] , Mixed [nullamt], Mixed [amt0 {aquantity=1}])+        ,("income:salary"       ,"salary"   , 2, [mamountp' "$-1.00"], Mixed [nullamt], Mixed [amt0 {aquantity=(-1)}])         ],         Mixed [nullamt]) -     ,_test "a valid history on an empty period"  $-      (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 2) (fromGregorian 2008 1 3), balancetype_=HistoricalBalance}, samplejournal) `gives`-       (-        [-         ("assets:bank:checking","checking",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=1}])-        ,("income:salary","salary",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amount0 {aquantity=(-1)}])-        ],-        Mixed [usd0])+     -- ,test "a valid history on an empty period"  $+     --  (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 2) (fromGregorian 2008 1 3), balancetype_=HistoricalBalance}, samplejournal) `gives`+     --   (+     --    [+     --     ("assets:bank:checking","checking",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=1}])+     --    ,("income:salary","salary",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amt0 {aquantity=(-1)}])+     --    ],+     --    Mixed [usd0]) -     ,_test "a valid history on an empty period (more complex)"  $-      (defreportopts{period_= PeriodBetween (fromGregorian 2009 1 1) (fromGregorian 2009 1 2), balancetype_=HistoricalBalance}, samplejournal) `gives`-       (-        [-        ("assets:bank:checking","checking",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=1}])-        ,("assets:bank:saving","saving",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=1}])-        ,("assets:cash","cash",2, [mamountp' "$-2.00"], mamountp' "$-2.00",Mixed [amount0 {aquantity=(-2)}])-        ,("expenses:food","food",2, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=(1)}])-        ,("expenses:supplies","supplies",2, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=(1)}])-        ,("income:gifts","gifts",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amount0 {aquantity=(-1)}])-        ,("income:salary","salary",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amount0 {aquantity=(-1)}])-        ],-        Mixed [usd0])+     -- ,test "a valid history on an empty period (more complex)"  $+     --  (defreportopts{period_= PeriodBetween (fromGregorian 2009 1 1) (fromGregorian 2009 1 2), balancetype_=HistoricalBalance}, samplejournal) `gives`+     --   (+     --    [+     --    ("assets:bank:checking","checking",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=1}])+     --    ,("assets:bank:saving","saving",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=1}])+     --    ,("assets:cash","cash",2, [mamountp' "$-2.00"], mamountp' "$-2.00",Mixed [amt0 {aquantity=(-2)}])+     --    ,("expenses:food","food",2, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=(1)}])+     --    ,("expenses:supplies","supplies",2, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=(1)}])+     --    ,("income:gifts","gifts",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amt0 {aquantity=(-1)}])+     --    ,("income:salary","salary",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amt0 {aquantity=(-1)}])+     --    ],+     --    Mixed [usd0])     ]  ]

@@ -270,22 +270,20 @@  tests_PostingsReport = tests "PostingsReport" [ -   tests "postingsReport" $-    let (query, journal) `gives` n = (length $ snd $ postingsReport defreportopts query journal) `is` n-    in [-     -- with the query specified explicitly-      (Any, nulljournal) `gives` 0-     ,(Any, samplejournal) `gives` 13-     -- register --depth just clips account names-     ,(Depth 2, samplejournal) `gives` 13-     ,(And [Depth 1, StatusQ Cleared, Acct "expenses"], samplejournal) `gives` 2-     ,(And [And [Depth 1, StatusQ Cleared], Acct "expenses"], samplejournal) `gives` 2--     -- with query and/or command-line options-     ,(length $ snd $ postingsReport defreportopts Any samplejournal) `is` 13-     ,(length $ snd $ postingsReport defreportopts{interval_=Months 1} Any samplejournal) `is` 11-     ,(length $ snd $ postingsReport defreportopts{interval_=Months 1, empty_=True} Any samplejournal) `is` 20-     ,(length $ snd $ postingsReport defreportopts (Acct "assets:bank:checking") samplejournal) `is` 5+   test "postingsReport" $ do+    let (query, journal) `gives` n = (length $ snd $ postingsReport defreportopts query journal) @?= n+    -- with the query specified explicitly+    (Any, nulljournal) `gives` 0+    (Any, samplejournal) `gives` 13+    -- register --depth just clips account names+    (Depth 2, samplejournal) `gives` 13+    (And [Depth 1, StatusQ Cleared, Acct "expenses"], samplejournal) `gives` 2+    (And [And [Depth 1, StatusQ Cleared], Acct "expenses"], samplejournal) `gives` 2+    -- with query and/or command-line options+    (length $ snd $ postingsReport defreportopts Any samplejournal) @?= 13+    (length $ snd $ postingsReport defreportopts{interval_=Months 1} Any samplejournal) @?= 11+    (length $ snd $ postingsReport defreportopts{interval_=Months 1, empty_=True} Any samplejournal) @?= 20+    (length $ snd $ postingsReport defreportopts (Acct "assets:bank:checking") samplejournal) @?= 5       -- (defreportopts, And [Acct "a a", Acct "'b"], samplejournal2) `gives` 0      -- [(Just (parsedate "2008-01-01","income"),assets:bank:checking             $1,$1)@@ -432,13 +430,9 @@          ]      -}-    ] -  ,tests "summarisePostingsByInterval" [-    tests "summarisePostingsByInterval" [-      summarisePostingsByInterval (Quarters 1) PrimaryDate 99999 False (DateSpan Nothing Nothing) [] `is` []-      ]-   ]+  ,test "summarisePostingsByInterval" $+    summarisePostingsByInterval (Quarters 1) PrimaryDate 99999 False (DateSpan Nothing Nothing) [] @?= []    -- ,tests_summarisePostingsInDateSpan = [     --  "summarisePostingsInDateSpan" ~: do

@@ -4,7 +4,7 @@  -} -{-# LANGUAGE OverloadedStrings, RecordWildCards, DeriveDataTypeable #-}+{-# LANGUAGE OverloadedStrings, RecordWildCards, LambdaCase, DeriveDataTypeable #-}  module Hledger.Reports.ReportOptions (   ReportOpts(..),@@ -38,6 +38,7 @@   reportPeriodLastDay,   reportPeriodOrJournalLastDay,   valuationTypeIsCost,+  valuationTypeIsDefaultValue,    tests_ReportOptions )@@ -98,7 +99,8 @@     ,no_elide_       :: Bool     ,real_           :: Bool     ,format_         :: Maybe FormatStr-    ,query_          :: String -- all arguments, as a string+    ,query_          :: String -- ^ All query arguments space sepeareted+                               --   and quoted if needed (see 'quoteIfNeeded')     --     ,average_        :: Bool     -- register command only@@ -111,6 +113,7 @@     ,no_total_       :: Bool     ,pretty_tables_  :: Bool     ,sort_amount_    :: Bool+    ,percent_        :: Bool     ,invert_         :: Bool  -- ^ if true, flip all amount signs in reports     ,normalbalance_  :: Maybe NormalSign       -- ^ This can be set when running balance reports on a set of accounts@@ -156,6 +159,7 @@     def     def     def+    def  rawOptsToReportOpts :: RawOpts -> IO ReportOpts rawOptsToReportOpts rawopts = checkReportOpts <$> do@@ -175,7 +179,7 @@     ,no_elide_    = boolopt "no-elide" rawopts'     ,real_        = boolopt "real" rawopts'     ,format_      = maybestringopt "format" rawopts' -- XXX move to CliOpts or move validation from Cli.CliOptions to here-    ,query_       = unwords $ listofstringopt "args" rawopts' -- doesn't handle an arg like "" right+    ,query_       = unwords . map quoteIfNeeded $ listofstringopt "args" rawopts' -- doesn't handle an arg like "" right     ,average_     = boolopt "average" rawopts'     ,related_     = boolopt "related" rawopts'     ,balancetype_ = balancetypeopt rawopts'@@ -184,6 +188,7 @@     ,row_total_   = boolopt "row-total" rawopts'     ,no_total_    = boolopt "no-total" rawopts'     ,sort_amount_ = boolopt "sort-amount" rawopts'+    ,percent_     = boolopt "percent" rawopts'     ,invert_      = boolopt "invert" rawopts'     ,pretty_tables_ = boolopt "pretty-tables" rawopts'     ,color_       = color@@ -215,18 +220,20 @@       _              -> Right ()  accountlistmodeopt :: RawOpts -> AccountListMode-accountlistmodeopt rawopts =-  case reverse $ filter (`elem` ["tree","flat"]) $ map fst rawopts of-    ("tree":_) -> ALTree-    ("flat":_) -> ALFlat-    _          -> ALDefault+accountlistmodeopt =+  fromMaybe ALDefault . choiceopt parse where+    parse = \case+      "tree" -> Just ALTree+      "flat" -> Just ALFlat+      _      -> Nothing  balancetypeopt :: RawOpts -> BalanceType-balancetypeopt rawopts =-  case reverse $ filter (`elem` ["change","cumulative","historical"]) $ map fst rawopts of-    ("historical":_) -> HistoricalBalance-    ("cumulative":_) -> CumulativeChange-    _                -> PeriodChange+balancetypeopt =+  fromMaybe PeriodChange . choiceopt parse where+    parse = \case+      "historical" -> Just HistoricalBalance+      "cumulative" -> Just CumulativeChange+      _            -> Nothing  -- Get the period specified by any -b/--begin, -e/--end and/or -p/--period -- options appearing in the command line.@@ -252,7 +259,7 @@ -- Get all begin dates specified by -b/--begin or -p/--period options, in order, -- using the given date to interpret relative date expressions. beginDatesFromRawOpts :: Day -> RawOpts -> [Day]-beginDatesFromRawOpts d = catMaybes . map (begindatefromrawopt d)+beginDatesFromRawOpts d = collectopts (begindatefromrawopt d)   where     begindatefromrawopt d (n,v)       | n == "begin" =@@ -270,7 +277,7 @@ -- Get all end dates specified by -e/--end or -p/--period options, in order, -- using the given date to interpret relative date expressions. endDatesFromRawOpts :: Day -> RawOpts -> [Day]-endDatesFromRawOpts d = catMaybes . map (enddatefromrawopt d)+endDatesFromRawOpts d = collectopts (enddatefromrawopt d)   where     enddatefromrawopt d (n,v)       | n == "end" =@@ -289,7 +296,7 @@ -- -D/--daily, -W/--weekly, -M/--monthly etc. options. -- An interval from --period counts only if it is explicitly defined. intervalFromRawOpts :: RawOpts -> Interval-intervalFromRawOpts = lastDef NoInterval . catMaybes . map intervalfromrawopt+intervalFromRawOpts = lastDef NoInterval . collectopts intervalfromrawopt   where     intervalfromrawopt (n,v)       | n == "period" =@@ -316,7 +323,7 @@ -- -P/--pending, -C/--cleared flags. -UPC is equivalent to no flags, -- so this returns a list of 0-2 unique statuses. statusesFromRawOpts :: RawOpts -> [Status]-statusesFromRawOpts = simplifyStatuses . catMaybes . map statusfromrawopt+statusesFromRawOpts = simplifyStatuses . collectopts statusfromrawopt   where     statusfromrawopt (n,_)       | n == "unmarked"  = Just Unmarked@@ -342,7 +349,7 @@ -- -B/--cost, -V, -X/--exchange, or --value flags. If there's more -- than one of these, the rightmost flag wins. valuationTypeFromRawOpts :: RawOpts -> Maybe ValuationType-valuationTypeFromRawOpts = lastDef Nothing . filter isJust . map valuationfromrawopt+valuationTypeFromRawOpts = lastMay . collectopts valuationfromrawopt   where     valuationfromrawopt (n,v)  -- option name, value       | n == "B"     = Just $ AtCost Nothing@@ -371,6 +378,12 @@     Just (AtCost _) -> True     _               -> False +valuationTypeIsDefaultValue :: ReportOpts -> Bool+valuationTypeIsDefaultValue ropts =+  case value_ ropts of+    Just (AtDefault _) -> True+    _                  -> False+ type DisplayExp = String  maybedisplayopt :: Day -> RawOpts -> Maybe DisplayExp@@ -408,7 +421,7 @@ journalSelectingAmountFromOpts :: ReportOpts -> Journal -> Journal journalSelectingAmountFromOpts opts =   case value_ opts of-    Just (AtCost _) -> journalConvertAmountsToCost+    Just (AtCost _) -> journalToCost     _               -> id  -- | Convert report options and arguments to a query.@@ -526,23 +539,19 @@ -- tests  tests_ReportOptions = tests "ReportOptions" [-   tests "queryFromOpts" [-      (queryFromOpts nulldate defreportopts) `is` Any-     ,(queryFromOpts nulldate defreportopts{query_="a"}) `is` (Acct "a")-     ,(queryFromOpts nulldate defreportopts{query_="desc:'a a'"}) `is` (Desc "a a")-     ,(queryFromOpts nulldate defreportopts{period_=PeriodFrom (parsedate "2012/01/01"),query_="date:'to 2013'" })-      `is` (Date $ mkdatespan "2012/01/01" "2013/01/01")-     ,(queryFromOpts nulldate defreportopts{query_="date2:'in 2012'"}) `is` (Date2 $ mkdatespan "2012/01/01" "2013/01/01")-     ,(queryFromOpts nulldate defreportopts{query_="'a a' 'b"}) `is` (Or [Acct "a a", Acct "'b"])-     ]+   test "queryFromOpts" $ do+       queryFromOpts nulldate defreportopts @?= Any+       queryFromOpts nulldate defreportopts{query_="a"} @?= Acct "a"+       queryFromOpts nulldate defreportopts{query_="desc:'a a'"} @?= Desc "a a"+       queryFromOpts nulldate defreportopts{period_=PeriodFrom (parsedate "2012/01/01"),query_="date:'to 2013'" }+         @?= (Date $ mkdatespan "2012/01/01" "2013/01/01")+       queryFromOpts nulldate defreportopts{query_="date2:'in 2012'"} @?= (Date2 $ mkdatespan "2012/01/01" "2013/01/01")+       queryFromOpts nulldate defreportopts{query_="'a a' 'b"} @?= Or [Acct "a a", Acct "'b"] -  ,tests "queryOptsFromOpts" [-      (queryOptsFromOpts nulldate defreportopts) `is` []-     ,(queryOptsFromOpts nulldate defreportopts{query_="a"}) `is` []-     ,(queryOptsFromOpts nulldate defreportopts{period_=PeriodFrom (parsedate "2012/01/01")-                                                                   ,query_="date:'to 2013'"-                                                                   })-      `is` []-    ]+  ,test "queryOptsFromOpts" $ do+      queryOptsFromOpts nulldate defreportopts @?= []+      queryOptsFromOpts nulldate defreportopts{query_="a"} @?= []+      queryOptsFromOpts nulldate defreportopts{period_=PeriodFrom (parsedate "2012/01/01")+                                              ,query_="date:'to 2013'"} @?= []  ] 

@@ -29,7 +29,7 @@                           -- Debug.Trace.trace,                           -- module Data.PPrint,                           -- module Hledger.Utils.UTF8IOCompat-                          SystemString,fromSystemString,toSystemString,error',userError',usageError,+                          error',userError',usageError,                           -- the rest need to be done in each module I think                           ) where@@ -66,7 +66,7 @@ import Hledger.Utils.Tree -- import Prelude hiding (readFile,writeFile,appendFile,getContents,putStr,putStrLn) -- import Hledger.Utils.UTF8IOCompat   (readFile,writeFile,appendFile,getContents,putStr,putStrLn)-import Hledger.Utils.UTF8IOCompat (SystemString,fromSystemString,toSystemString,error',userError',usageError)+import Hledger.Utils.UTF8IOCompat (error',userError',usageError)   -- tuples

@@ -49,14 +49,15 @@ -- | A parser of strict text to some type. type SimpleTextParser = Parsec CustomErr Text  -- XXX an "a" argument breaks the CsvRulesParser declaration somehow --- | A parser of text in some monad.+-- | A parser of text that runs in some monad. type TextParser m a = ParsecT CustomErr Text m a --- | A parser of text in some monad, with a journal as state.+-- | A parser of text that runs in some monad, keeping a Journal as state. type JournalParser m a = StateT Journal (ParsecT CustomErr Text m) a --- | A parser of text in some monad, with a journal as state, that can throw a--- "final" parse error that does not backtrack.+-- | A parser of text that runs in some monad, keeping a Journal as+-- state, that can throw an exception to end parsing, preventing+-- further parser backtracking. type ErroringJournalParser m a =   StateT Journal (ParsecT CustomErr Text (ExceptT FinalParseError m)) a 

@@ -21,6 +21,7 @@  lstrip,  rstrip,  chomp,+ singleline,  elideLeft,  elideRight,  formatString,@@ -75,6 +76,10 @@ -- | Remove trailing newlines/carriage returns. chomp :: String -> String chomp = reverse . dropWhile (`elem` "\r\n") . reverse++-- | Remove consequtive line breaks, replacing them with single space+singleline :: String -> String+singleline = unwords . filter (/="") . (map strip) . lines  stripbrackets :: String -> String stripbrackets = dropWhile (`elem` "([") . reverse . dropWhile (`elem` "])") . reverse :: String -> String

@@ -4,229 +4,192 @@ {-# LANGUAGE ScopedTypeVariables #-}  module Hledger.Utils.Test (-   HasCallStack-  ,module EasyTest-  ,runEasytests+   module Test.Tasty+  ,module Test.Tasty.HUnit+  -- ,module QC+  -- ,module SC   ,tests-  ,_tests   ,test-  ,_test-  ,it-  ,_it-  ,is-  ,expectEqPP-  ,expectParse-  ,expectParseE-  ,expectParseError-  ,expectParseErrorE-  ,expectParseEq-  ,expectParseEqE-  ,expectParseEqOn-  ,expectParseEqOnE-  ,expectParseStateOn+  ,assertLeft+  ,assertRight+  ,assertParse+  ,assertParseEq+  ,assertParseEqOn+  ,assertParseError+  ,assertParseE+  ,assertParseEqE+  ,assertParseErrorE+  ,assertParseStateOn ) where -import Control.Exception import Control.Monad.Except (ExceptT, runExceptT) import Control.Monad.State.Strict (StateT, evalStateT, execStateT) #if !(MIN_VERSION_base(4,11,0)) import Data.Monoid ((<>)) #endif-import Data.CallStack-import Data.List+-- import Data.CallStack+import Data.List (isInfixOf) import qualified Data.Text as T-import Safe-import System.Exit+import Test.Tasty hiding (defaultMain)+import Test.Tasty.HUnit+-- import Test.Tasty.QuickCheck as QC+-- import Test.Tasty.SmallCheck as SC import Text.Megaparsec import Text.Megaparsec.Custom--import EasyTest hiding (char, char', tests)  -- reexported-import qualified EasyTest as E               -- used here+  ( CustomErr,+    FinalParseError,+    attachSource,+    customErrorBundlePretty,+    finalErrorBundlePretty,+  )  import Hledger.Utils.Debug (pshow)-import Hledger.Utils.UTF8IOCompat (error')+-- import Hledger.Utils.UTF8IOCompat (error') --- * easytest helpers+-- * tasty helpers --- | Name the given test(s). A readability synonym for easytest's "scope".-test :: T.Text -> E.Test a -> E.Test a-test = E.scope+-- TODO: pretty-print values in failure messages --- | Skip the given test(s), with the same type signature as "test".--- If called in a monadic sequence of tests, also skips following tests.-_test :: T.Text -> E.Test a -> E.Test a-_test _name = (E.skip >>) --- | Name the given test(s). A synonym for "test".-it :: T.Text -> E.Test a -> E.Test a-it = test---- | Skip the given test(s), and any following tests in a monadic sequence.--- A synonym for "_test".-_it :: T.Text -> E.Test a -> E.Test a-_it = _test---- | Name and group a list of tests. Combines easytest's "scope" and "tests".-tests :: T.Text -> [E.Test ()] -> E.Test ()-tests name = E.scope name . E.tests---- | Skip the given list of tests, and any following tests in a monadic sequence,--- with the same type signature as "group".-_tests :: T.Text -> [E.Test ()] -> E.Test ()-_tests _name = (E.skip >>) . E.tests+-- | Name and group a list of tests. Shorter alias for Test.Tasty.HUnit.testGroup.+tests :: String -> [TestTree] -> TestTree+tests = testGroup --- | Run some easytest tests, catching easytest's ExitCode exception,--- returning True if there was a problem.--- With arguments, runs only the scope (or single test) named by the first argument--- (exact, case sensitive).--- If there is a second argument, it should be an integer and will be used--- as the seed for randomness.-runEasytests :: [String] -> E.Test () -> IO Bool-runEasytests args tests = (do-  case args of-    []    -> E.run tests-    [a]   -> E.runOnly (T.pack a) tests-    a:b:_ -> do-      case readMay b :: Maybe Int of-        Nothing   -> error' "the second argument should be an integer (a seed for easytest)"-        Just seed -> E.rerunOnly seed (T.pack a) tests-  return False-  )-  `catch` (\(_::ExitCode) -> return True)+-- | Name an assertion or sequence of assertions. Shorter alias for Test.Tasty.HUnit.testCase.+test :: String -> Assertion -> TestTree+test = testCase --- | Like easytest's expectEq (asserts the second (actual) value equals the first (expected) value)--- but pretty-prints the values in the failure output.-expectEqPP :: (Eq a, Show a, HasCallStack) => a -> a -> E.Test ()-expectEqPP expected actual = if expected == actual then E.ok else E.crash $-  "\nexpected:\n" <> T.pack (pshow expected) <> "\nbut got:\n" <> T.pack (pshow actual) <> "\n"+-- | Assert any Left value.+assertLeft :: (HasCallStack, Eq b, Show b) => Either a b -> Assertion+assertLeft (Left _)  = return ()+assertLeft (Right b) = assertFailure $ "expected Left, got (Right " ++ show b ++ ")" --- | Shorter and flipped version of expectEqPP. The expected value goes last.-is :: (Eq a, Show a, HasCallStack) => a -> a -> Test ()-is = flip expectEqPP+-- | Assert any Right value.+assertRight :: (HasCallStack, Eq a, Show a) => Either a b -> Assertion+assertRight (Right _) = return ()+assertRight (Left a)  = assertFailure $ "expected Right, got (Left " ++ show a ++ ")" --- | Test that this stateful parser runnable in IO successfully parses+-- | Assert that this stateful parser runnable in IO successfully parses -- all of the given input text, showing the parse error if it fails. -- Suitable for hledger's JournalParser parsers.-expectParse :: (Monoid st, Eq a, Show a, HasCallStack) =>-  StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> E.Test ()-expectParse parser input = do-  ep <- E.io (runParserT (evalStateT (parser <* eof) mempty) "" input)-  either (fail.(++"\n").("\nparse error at "++).customErrorBundlePretty)-         (const ok)+assertParse :: (HasCallStack, Eq a, Show a, Monoid st) =>+  StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> Assertion+assertParse parser input = do+  ep <- runParserT (evalStateT (parser <* eof) mempty) "" input+  either (assertFailure.(++"\n").("\nparse error at "++).customErrorBundlePretty)+         (const $ return ())          ep --- Suitable for hledger's ErroringJournalParser parsers.-expectParseE-  :: (Monoid st, Eq a, Show a, HasCallStack)-  => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a-  -> T.Text-  -> E.Test ()-expectParseE parser input = do-  let filepath = ""-  eep <- E.io $ runExceptT $-           runParserT (evalStateT (parser <* eof) mempty) filepath input-  case eep of-    Left finalErr ->-      let prettyErr = finalErrorBundlePretty $ attachSource filepath input finalErr-      in  fail $ "parse error at " <> prettyErr-    Right ep ->-      either (fail.(++"\n").("\nparse error at "++).customErrorBundlePretty)-             (const ok)-             ep+-- | Assert a parser produces an expected value.+assertParseEq :: (HasCallStack, Eq a, Show a, Monoid st) =>+  StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> a -> Assertion+assertParseEq parser input expected = assertParseEqOn parser input id expected --- | Test that this stateful parser runnable in IO fails to parse+-- | Like assertParseEq, but transform the parse result with the given function+-- before comparing it.+assertParseEqOn :: (HasCallStack, Eq b, Show b, Monoid st) =>+  StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> (a -> b) -> b -> Assertion+assertParseEqOn parser input f expected = do+  ep <- runParserT (evalStateT (parser <* eof) mempty) "" input+  either (assertFailure . (++"\n") . ("\nparse error at "++) . customErrorBundlePretty)+         (assertEqual "" expected . f)+         ep++-- | Assert that this stateful parser runnable in IO fails to parse -- the given input text, with a parse error containing the given string.-expectParseError :: (Monoid st, Eq a, Show a, HasCallStack) =>-  StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> String -> E.Test ()-expectParseError parser input errstr = do-  ep <- E.io (runParserT (evalStateT parser mempty) "" input)+assertParseError :: (HasCallStack, Eq a, Show a, Monoid st) =>+  StateT st (ParsecT CustomErr T.Text IO) a -> String -> String -> Assertion+assertParseError parser input errstr = do+  ep <- runParserT (evalStateT parser mempty) "" (T.pack input)   case ep of-    Right v -> fail $ "\nparse succeeded unexpectedly, producing:\n" ++ pshow v ++ "\n"+    Right v -> assertFailure $ "\nparse succeeded unexpectedly, producing:\n" ++ pshow v ++ "\n"     Left e  -> do       let e' = customErrorBundlePretty e       if errstr `isInfixOf` e'-      then ok-      else fail $ "\nparse error is not as expected:\n" ++ e' ++ "\n"+      then return ()+      else assertFailure $ "\nparse error is not as expected:\n" ++ e' ++ "\n" -expectParseErrorE-  :: (Monoid st, Eq a, Show a, HasCallStack)+-- | Run a stateful parser in IO like assertParse, then assert that the+-- final state (the wrapped state, not megaparsec's internal state),+-- transformed by the given function, matches the given expected value.+assertParseStateOn :: (HasCallStack, Eq b, Show b, Monoid st) =>+     StateT st (ParsecT CustomErr T.Text IO) a+  -> T.Text+  -> (st -> b)+  -> b+  -> Assertion+assertParseStateOn parser input f expected = do+  es <- runParserT (execStateT (parser <* eof) mempty) "" input+  case es of+    Left err -> assertFailure $ (++"\n") $ ("\nparse error at "++) $ customErrorBundlePretty err+    Right s  -> assertEqual "" expected $ f s++-- | These "E" variants of the above are suitable for hledger's ErroringJournalParser parsers.+assertParseE+  :: (HasCallStack, Eq a, Show a, Monoid st)   => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a   -> T.Text-  -> String-  -> E.Test ()-expectParseErrorE parser input errstr = do+  -> Assertion+assertParseE parser input = do   let filepath = ""-  eep <- E.io $ runExceptT $ runParserT (evalStateT parser mempty) filepath input+  eep <- runExceptT $+           runParserT (evalStateT (parser <* eof) mempty) filepath input   case eep of-    Left finalErr -> do+    Left finalErr ->       let prettyErr = finalErrorBundlePretty $ attachSource filepath input finalErr-      if errstr `isInfixOf` prettyErr-      then ok-      else fail $ "\nparse error is not as expected:\n" ++ prettyErr ++ "\n"-    Right ep -> case ep of-      Right v -> fail $ "\nparse succeeded unexpectedly, producing:\n" ++ pshow v ++ "\n"-      Left e  -> do-        let e' = customErrorBundlePretty e-        if errstr `isInfixOf` e'-        then ok-        else fail $ "\nparse error is not as expected:\n" ++ e' ++ "\n"---- | Like expectParse, but also test the parse result is an expected value,--- pretty-printing both if it fails.-expectParseEq :: (Monoid st, Eq a, Show a, HasCallStack) =>-  StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> a -> E.Test ()-expectParseEq parser input expected = expectParseEqOn parser input id expected+      in  assertFailure $ "parse error at " <> prettyErr+    Right ep ->+      either (assertFailure.(++"\n").("\nparse error at "++).customErrorBundlePretty)+             (const $ return ())+             ep -expectParseEqE+assertParseEqE   :: (Monoid st, Eq a, Show a, HasCallStack)   => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a   -> T.Text   -> a-  -> E.Test ()-expectParseEqE parser input expected = expectParseEqOnE parser input id expected---- | Like expectParseEq, but transform the parse result with the given function--- before comparing it.-expectParseEqOn :: (Monoid st, Eq b, Show b, HasCallStack) =>-  StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> (a -> b) -> b -> E.Test ()-expectParseEqOn parser input f expected = do-  ep <- E.io $ runParserT (evalStateT (parser <* eof) mempty) "" input-  either (fail . (++"\n") . ("\nparse error at "++) . customErrorBundlePretty)-         (expectEqPP expected . f)-         ep+  -> Assertion+assertParseEqE parser input expected = assertParseEqOnE parser input id expected -expectParseEqOnE-  :: (Monoid st, Eq b, Show b, HasCallStack)+assertParseEqOnE+  :: (HasCallStack, Eq b, Show b, Monoid st)   => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a   -> T.Text   -> (a -> b)   -> b-  -> E.Test ()-expectParseEqOnE parser input f expected = do+  -> Assertion+assertParseEqOnE parser input f expected = do   let filepath = ""-  eep <- E.io $ runExceptT $-           runParserT (evalStateT (parser <* eof) mempty) filepath input+  eep <- runExceptT $ runParserT (evalStateT (parser <* eof) mempty) filepath input   case eep of     Left finalErr ->       let prettyErr = finalErrorBundlePretty $ attachSource filepath input finalErr-      in  fail $ "parse error at " <> prettyErr+      in  assertFailure $ "parse error at " <> prettyErr     Right ep ->-      either (fail . (++"\n") . ("\nparse error at "++) . customErrorBundlePretty)-             (expectEqPP expected . f)+      either (assertFailure . (++"\n") . ("\nparse error at "++) . customErrorBundlePretty)+             (assertEqual "" expected . f)              ep --- | Run a stateful parser in IO like expectParse, then compare the--- final state (the wrapped state, not megaparsec's internal state),--- transformed by the given function, with the given expected value.-expectParseStateOn :: (HasCallStack, Monoid st, Eq b, Show b) =>-     StateT st (ParsecT CustomErr T.Text IO) a+assertParseErrorE+  :: (Monoid st, Eq a, Show a, HasCallStack)+  => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a   -> T.Text-  -> (st -> b)-  -> b-  -> E.Test ()-expectParseStateOn parser input f expected = do-  es <- E.io $ runParserT (execStateT (parser <* eof) mempty) "" input-  case es of-    Left err -> fail $ (++"\n") $ ("\nparse error at "++) $ customErrorBundlePretty err-    Right s  -> expectEqPP expected $ f s+  -> String+  -> Assertion+assertParseErrorE parser input errstr = do+  let filepath = ""+  eep <- runExceptT $ runParserT (evalStateT parser mempty) filepath input+  case eep of+    Left finalErr -> do+      let prettyErr = finalErrorBundlePretty $ attachSource filepath input finalErr+      if errstr `isInfixOf` prettyErr+      then return ()+      else assertFailure $ "\nparse error is not as expected:\n" ++ prettyErr ++ "\n"+    Right ep -> case ep of+      Right v -> assertFailure $ "\nparse succeeded unexpectedly, producing:\n" ++ pshow v ++ "\n"+      Left e  -> do+        let e' = customErrorBundlePretty e+        if errstr `isInfixOf` e'+        then return ()+        else assertFailure $ "\nparse error is not as expected:\n" ++ e' ++ "\n"

@@ -421,13 +421,12 @@   tests_Text = tests "Text" [-   tests "quoteIfSpaced" [-     quoteIfSpaced "a'a" `is` "a'a"-    ,quoteIfSpaced "a\"a" `is` "a\"a"-    ,quoteIfSpaced "a a" `is` "\"a a\""-    ,quoteIfSpaced "mimi's cafe" `is` "\"mimi's cafe\""-    ,quoteIfSpaced "\"alex\" cafe" `is` "\"\\\"alex\\\" cafe\""-    ,quoteIfSpaced "le'shan's cafe" `is` "\"le'shan's cafe\""-    ,quoteIfSpaced "\"be'any's\" cafe" `is` "\"\\\"be'any's\\\" cafe\""-    ]+   test "quoteIfSpaced" $ do+     quoteIfSpaced "a'a" @?= "a'a"+     quoteIfSpaced "a\"a" @?= "a\"a"+     quoteIfSpaced "a a" @?= "\"a a\""+     quoteIfSpaced "mimi's cafe" @?= "\"mimi's cafe\""+     quoteIfSpaced "\"alex\" cafe" @?= "\"\\\"alex\\\" cafe\""+     quoteIfSpaced "le'shan's cafe" @?= "\"le'shan's cafe\""+     quoteIfSpaced "\"be'any's\" cafe" @?= "\"\\\"be'any's\\\" cafe\""   ]

@@ -15,6 +15,9 @@ do the right thing, so this file is a no-op and on its way to being removed. Not carefully tested. +2019/10/20 update: all packages have base>=4.9 which corresponds to GHC v8.0.1+and higher. Tear this file apart!+ -} -- TODO obsolete ? @@ -29,9 +32,6 @@   hPutStr,   hPutStrLn,   ---  SystemString,-  fromSystemString,-  toSystemString,   error',   userError',   usageError,@@ -85,37 +85,19 @@ -- bs_hPutStr        = B8.hPut -- bs_hPutStrLn h bs = B8.hPut h bs >> B8.hPut h (B.singleton 0x0a) ---- | A string received from or being passed to the operating system, such--- as a file path, command-line argument, or environment variable name or--- value. With GHC versions before 7.2 on some platforms (posix) these are--- typically encoded. When converting, we assume the encoding is UTF-8 (cf--- <http://www.dwheeler.com/essays/fixing-unix-linux-filenames.html#UTF8>).-type SystemString = String---- | Convert a system string to an ordinary string, decoding from UTF-8 if--- it appears to be UTF8-encoded and GHC version is less than 7.2.-fromSystemString :: SystemString -> String-fromSystemString = id---- | Convert a unicode string to a system string, encoding with UTF-8 if--- we are on a posix platform with GHC < 7.2.-toSystemString :: String -> SystemString-toSystemString = id- -- | A SystemString-aware version of error. error' :: String -> a error' = #if __GLASGOW_HASKELL__ < 800 -- (easier than if base < 4.9)-  error . toSystemString+  error #else-  errorWithoutStackTrace . toSystemString+  errorWithoutStackTrace #endif  -- | A SystemString-aware version of userError. userError' :: String -> IOError-userError' = userError . toSystemString+userError' = userError  -- | A SystemString-aware version of error that adds a usage hint. usageError :: String -> a

@@ -4,17 +4,17 @@ -- -- see: https://github.com/sol/hpack ----- hash: 567ed725b211714a0f6db5e17a68d670789c7e603020b42d6b8f18e7af5ceb63+-- hash: 3228f8dbb178d427e76291b5e60b3dee1eb4d2e5f9ab803ce8e3fe85e79f25ad  name:           hledger-lib-version:        1.15.2+version:        1.16 synopsis:       Core data types, parsers and functionality for the hledger accounting tools description:    This is a reusable library containing hledger's core functionality.                 .                 hledger is a cross-platform program for tracking money, time, or                 any other commodity, using double-entry accounting and a simple,                 editable file format. It is inspired by and largely compatible-                with ledger(1).  hledger provides command-line, curses and web+                with ledger(1).  hledger provides command-line, terminal and web                 interfaces, and aims to be a reliable, practical tool for daily                 use. category:       Finance@@ -25,11 +25,13 @@ maintainer:     Simon Michael <simon@joyful.com> license:        GPL-3 license-file:   LICENSE-tested-with:    GHC==7.10.3, GHC==8.0.2, GHC==8.2.2, GHC==8.4.3, GHC==8.6.5+tested-with:    GHC==8.0.2, GHC==8.2.2, GHC==8.4.3, GHC==8.6.5, GHC==8.8.1 build-type:     Simple extra-source-files:     CHANGES.md     README+    test/unittest.hs+    test/doctests.hs     hledger_csv.5     hledger_csv.txt     hledger_csv.info@@ -107,8 +109,8 @@     , Glob >=0.9     , ansi-terminal >=0.6.2.3     , array-    , base >=4.8 && <4.13-    , base-compat-batteries >=0.10.1 && <0.11+    , base >=4.9 && <4.14+    , base-compat-batteries >=0.10.1 && <0.12     , blaze-markup >=0.5.1     , bytestring     , call-stack@@ -119,7 +121,6 @@     , data-default >=0.5     , deepseq     , directory-    , easytest >=0.2.1 && <0.3     , extra >=1.6.3     , fgl >=5.5.4.0     , file-embed >=0.0.10@@ -136,6 +137,8 @@     , safe >=0.2     , split >=0.1     , tabular >=0.2+    , tasty >=1.2.3+    , tasty-hunit >=0.10.0.2     , template-haskell     , text >=1.2     , time >=1.5@@ -143,64 +146,11 @@     , transformers >=0.2     , uglymemo     , utf8-string >=0.3.5-  if (!impl(ghc >= 8.0))-    build-depends:-        semigroups ==0.18.*   default-language: Haskell2010 -test-suite doctests+test-suite doctest   type: exitcode-stdio-1.0   main-is: doctests.hs-  other-modules:-      Hledger-      Hledger.Data-      Hledger.Data.Account-      Hledger.Data.AccountName-      Hledger.Data.Amount-      Hledger.Data.Commodity-      Hledger.Data.Dates-      Hledger.Data.Journal-      Hledger.Data.Ledger-      Hledger.Data.Period-      Hledger.Data.PeriodicTransaction-      Hledger.Data.Posting-      Hledger.Data.RawOptions-      Hledger.Data.StringFormat-      Hledger.Data.Timeclock-      Hledger.Data.Transaction-      Hledger.Data.TransactionModifier-      Hledger.Data.Types-      Hledger.Data.Valuation-      Hledger.Query-      Hledger.Read-      Hledger.Read.Common-      Hledger.Read.CsvReader-      Hledger.Read.JournalReader-      Hledger.Read.TimeclockReader-      Hledger.Read.TimedotReader-      Hledger.Reports-      Hledger.Reports.AccountTransactionsReport-      Hledger.Reports.BalanceReport-      Hledger.Reports.BudgetReport-      Hledger.Reports.EntriesReport-      Hledger.Reports.MultiBalanceReport-      Hledger.Reports.PostingsReport-      Hledger.Reports.ReportOptions-      Hledger.Reports.ReportTypes-      Hledger.Reports.TransactionsReport-      Hledger.Utils-      Hledger.Utils.Color-      Hledger.Utils.Debug-      Hledger.Utils.Parse-      Hledger.Utils.Regex-      Hledger.Utils.String-      Hledger.Utils.Test-      Hledger.Utils.Text-      Hledger.Utils.Tree-      Hledger.Utils.UTF8IOCompat-      Text.Megaparsec.Custom-      Text.Tabular.AsciiWide-      Paths_hledger_lib   hs-source-dirs:       ./.       test@@ -210,8 +160,8 @@     , Glob >=0.7     , ansi-terminal >=0.6.2.3     , array-    , base >=4.8 && <4.13-    , base-compat-batteries >=0.10.1 && <0.11+    , base >=4.9 && <4.14+    , base-compat-batteries >=0.10.1 && <0.12     , blaze-markup >=0.5.1     , bytestring     , call-stack@@ -223,7 +173,6 @@     , deepseq     , directory     , doctest >=0.16-    , easytest >=0.2.1 && <0.3     , extra >=1.6.3     , fgl >=5.5.4.0     , file-embed >=0.0.10@@ -240,6 +189,8 @@     , safe >=0.2     , split >=0.1     , tabular >=0.2+    , tasty >=1.2.3+    , tasty-hunit >=0.10.0.2     , template-haskell     , text >=1.2     , time >=1.5@@ -247,66 +198,14 @@     , transformers >=0.2     , uglymemo     , utf8-string >=0.3.5-  if (!impl(ghc >= 8.0))-    build-depends:-        semigroups ==0.18.*+  buildable: False   if (impl(ghc < 8.2))     buildable: False   default-language: Haskell2010 -test-suite easytests+test-suite unittest   type: exitcode-stdio-1.0-  main-is: easytests.hs-  other-modules:-      Hledger-      Hledger.Data-      Hledger.Data.Account-      Hledger.Data.AccountName-      Hledger.Data.Amount-      Hledger.Data.Commodity-      Hledger.Data.Dates-      Hledger.Data.Journal-      Hledger.Data.Ledger-      Hledger.Data.Period-      Hledger.Data.PeriodicTransaction-      Hledger.Data.Posting-      Hledger.Data.RawOptions-      Hledger.Data.StringFormat-      Hledger.Data.Timeclock-      Hledger.Data.Transaction-      Hledger.Data.TransactionModifier-      Hledger.Data.Types-      Hledger.Data.Valuation-      Hledger.Query-      Hledger.Read-      Hledger.Read.Common-      Hledger.Read.CsvReader-      Hledger.Read.JournalReader-      Hledger.Read.TimeclockReader-      Hledger.Read.TimedotReader-      Hledger.Reports-      Hledger.Reports.AccountTransactionsReport-      Hledger.Reports.BalanceReport-      Hledger.Reports.BudgetReport-      Hledger.Reports.EntriesReport-      Hledger.Reports.MultiBalanceReport-      Hledger.Reports.PostingsReport-      Hledger.Reports.ReportOptions-      Hledger.Reports.ReportTypes-      Hledger.Reports.TransactionsReport-      Hledger.Utils-      Hledger.Utils.Color-      Hledger.Utils.Debug-      Hledger.Utils.Parse-      Hledger.Utils.Regex-      Hledger.Utils.String-      Hledger.Utils.Test-      Hledger.Utils.Text-      Hledger.Utils.Tree-      Hledger.Utils.UTF8IOCompat-      Text.Megaparsec.Custom-      Text.Tabular.AsciiWide-      Paths_hledger_lib+  main-is: unittest.hs   hs-source-dirs:       ./.       test@@ -316,8 +215,8 @@     , Glob >=0.9     , ansi-terminal >=0.6.2.3     , array-    , base >=4.8 && <4.13-    , base-compat-batteries >=0.10.1 && <0.11+    , base >=4.9 && <4.14+    , base-compat-batteries >=0.10.1 && <0.12     , blaze-markup >=0.5.1     , bytestring     , call-stack@@ -328,7 +227,6 @@     , data-default >=0.5     , deepseq     , directory-    , easytest >=0.2.1 && <0.3     , extra >=1.6.3     , fgl >=5.5.4.0     , file-embed >=0.0.10@@ -346,6 +244,8 @@     , safe >=0.2     , split >=0.1     , tabular >=0.2+    , tasty >=1.2.3+    , tasty-hunit >=0.10.0.2     , template-haskell     , text >=1.2     , time >=1.5@@ -353,7 +253,5 @@     , transformers >=0.2     , uglymemo     , utf8-string >=0.3.5-  if (!impl(ghc >= 8.0))-    build-depends:-        semigroups ==0.18.*+  buildable: True   default-language: Haskell2010

@@ -1,364 +1,979 @@--.TH "hledger_csv" "5" "September 2019" "hledger 1.15.2" "hledger User Manuals"----.SH NAME-.PP-CSV - how hledger reads CSV data, and the CSV rules file format-.SH DESCRIPTION-.PP-hledger can read CSV (comma-separated value) files as if they were-journal files, automatically converting each CSV record into a-transaction.-(To learn about \f[I]writing\f[R] CSV, see CSV output.)-.PP-Converting CSV to transactions requires some special conversion rules.-These do several things:-.IP \[bu] 2-they describe the layout and format of the CSV data-.IP \[bu] 2-they can customize the generated journal entries using a simple-templating language-.IP \[bu] 2-they can add refinements based on patterns in the CSV data, eg-categorizing transactions with more detailed account names.-.PP-When reading a CSV file named \f[C]FILE.csv\f[R], hledger looks for a-conversion rules file named \f[C]FILE.csv.rules\f[R] in the same-directory.-You can override this with the \f[C]--rules-file\f[R] option.-If the rules file does not exist, hledger will auto-create one with some-example rules, which you\[aq]ll need to adjust.-.PP-At minimum, the rules file must identify the date and amount fields.-It\[aq]s often necessary to specify the date format, and the number of-header lines to skip, also.-Eg:-.IP-.nf-\f[C]-fields date, _, _, amount-date-format  %d/%m/%Y-skip 1-\f[R]-.fi-.PP-A more complete example:-.IP-.nf-\f[C]-# hledger CSV rules for amazon.com order history--# sample:-# \[dq]Date\[dq],\[dq]Type\[dq],\[dq]To/From\[dq],\[dq]Name\[dq],\[dq]Status\[dq],\[dq]Amount\[dq],\[dq]Fees\[dq],\[dq]Transaction ID\[dq]-# \[dq]Jul 29, 2012\[dq],\[dq]Payment\[dq],\[dq]To\[dq],\[dq]Adapteva, Inc.\[dq],\[dq]Completed\[dq],\[dq]$25.00\[dq],\[dq]$0.00\[dq],\[dq]17LA58JSK6PRD4HDGLNJQPI1PB9N8DKPVHL\[dq]--# skip one header line-skip 1--# name the csv fields (and assign the transaction\[aq]s date, amount and code)-fields date, _, toorfrom, name, amzstatus, amount, fees, code--# how to parse the date-date-format %b %-d, %Y--# combine two fields to make the description-description %toorfrom %name--# save these fields as tags-comment     status:%amzstatus, fees:%fees--# set the base account for all transactions-account1    assets:amazon--# flip the sign on the amount-amount      -%amount-\f[R]-.fi-.PP-For more examples, see Convert CSV files.-.SH CSV RULES-.PP-The following seven kinds of rule can appear in the rules file, in any-order.-Blank lines and lines beginning with \f[C]#\f[R] or \f[C];\f[R] are-ignored.-.SS skip-.PP-\f[C]skip\f[R]\f[I]\f[CI]N\f[I]\f[R]-.PP-Skip this number of CSV records at the beginning.-You\[aq]ll need this whenever your CSV data contains header lines.-Eg:-.IP-.nf-\f[C]-# ignore the first CSV line-skip 1-\f[R]-.fi-.SS date-format-.PP-\f[C]date-format\f[R]\f[I]\f[CI]DATEFMT\f[I]\f[R]-.PP-When your CSV date fields are not formatted like \f[C]YYYY/MM/DD\f[R]-(or \f[C]YYYY-MM-DD\f[R] or \f[C]YYYY.MM.DD\f[R]), you\[aq]ll need to-specify the format.-DATEFMT is a strptime-like date parsing pattern, which must parse the-date field values completely.-Examples:-.IP-.nf-\f[C]-# for dates like \[dq]11/06/2013\[dq]:-date-format %m/%d/%Y-\f[R]-.fi-.IP-.nf-\f[C]-# for dates like \[dq]6/11/2013\[dq] (note the - to make leading zeros optional):-date-format %-d/%-m/%Y-\f[R]-.fi-.IP-.nf-\f[C]-# for dates like \[dq]2013-Nov-06\[dq]:-date-format %Y-%h-%d-\f[R]-.fi-.IP-.nf-\f[C]-# for dates like \[dq]11/6/2013 11:32 PM\[dq]:-date-format %-m/%-d/%Y %l:%M %p-\f[R]-.fi-.SS field list-.PP-\f[C]fields\f[R]\f[I]\f[CI]FIELDNAME1\f[I]\f[R],-\f[I]\f[CI]FIELDNAME2\f[I]\f[R]...-.PP-This (a) names the CSV fields, in order (names may not contain-whitespace; uninteresting names may be left blank), and (b) assigns them-to journal entry fields if you use any of these standard field names:-\f[C]date\f[R], \f[C]date2\f[R], \f[C]status\f[R], \f[C]code\f[R],-\f[C]description\f[R], \f[C]comment\f[R], \f[C]account1\f[R],-\f[C]account2\f[R], \f[C]amount\f[R], \f[C]amount-in\f[R],-\f[C]amount-out\f[R], \f[C]currency\f[R], \f[C]balance\f[R],-\f[C]balance1\f[R], \f[C]balance2\f[R].-Eg:-.IP-.nf-\f[C]-# use the 1st, 2nd and 4th CSV fields as the entry\[aq]s date, description and amount,-# and give the 7th and 8th fields meaningful names for later reference:-#-# CSV field:-#      1     2            3 4       5 6 7          8-# entry field:-fields date, description, , amount, , , somefield, anotherfield-\f[R]-.fi-.SS field assignment-.PP-\f[I]\f[CI]ENTRYFIELDNAME\f[I]\f[R] \f[I]\f[CI]FIELDVALUE\f[I]\f[R]-.PP-This sets a journal entry field (one of the standard names above) to the-given text value, which can include CSV field values interpolated by-name (\f[C]%CSVFIELDNAME\f[R]) or 1-based position (\f[C]%N\f[R]).-Eg:-.IP-.nf-\f[C]-# set the amount to the 4th CSV field with \[dq]USD \[dq] prepended-amount USD %4-\f[R]-.fi-.IP-.nf-\f[C]-# combine three fields to make a comment (containing two tags)-comment note: %somefield - %anotherfield, date: %1-\f[R]-.fi-.PP-Field assignments can be used instead of or in addition to a field list.-.PP-Note, interpolation strips any outer whitespace, so a CSV value like-\f[C]\[dq] 1 \[dq]\f[R] becomes \f[C]1\f[R] when interpolated (#1051).-.SS conditional block-.PP-\f[C]if\f[R] \f[I]\f[CI]PATTERN\f[I]\f[R]-.PD 0-.P-.PD-\ \ \ \ \f[I]\f[CI]FIELDASSIGNMENTS\f[I]\f[R]...-.PP-\f[C]if\f[R]-.PD 0-.P-.PD-\f[I]\f[CI]PATTERN\f[I]\f[R]-.PD 0-.P-.PD-\f[I]\f[CI]PATTERN\f[I]\f[R]...-.PD 0-.P-.PD-\ \ \ \ \f[I]\f[CI]FIELDASSIGNMENTS\f[I]\f[R]...-.PP-This applies one or more field assignments, only to those CSV records-matched by one of the PATTERNs.-The patterns are case-insensitive regular expressions which match-anywhere within the whole CSV record (it\[aq]s not yet possible to match-within a specific field).-When there are multiple patterns they can be written on separate lines,-unindented.-The field assignments are on separate lines indented by at least one-space.-Examples:-.IP-.nf-\f[C]-# if the CSV record contains \[dq]groceries\[dq], set account2 to \[dq]expenses:groceries\[dq]-if groceries- account2 expenses:groceries-\f[R]-.fi-.IP-.nf-\f[C]-# if the CSV record contains any of these patterns, set account2 and comment as shown-if-monthly service fee-atm transaction fee-banking thru software- account2 expenses:business:banking- comment  XXX deductible ? check it-\f[R]-.fi-.SS include-.PP-\f[C]include\f[R]\f[I]\f[CI]RULESFILE\f[I]\f[R]-.PP-Include another rules file at this point.-\f[C]RULESFILE\f[R] is either an absolute file path or a path relative-to the current file\[aq]s directory.-Eg:-.IP-.nf-\f[C]-# rules reused with several CSV files-include common.rules-\f[R]-.fi-.SS newest-first-.PP-\f[C]newest-first\f[R]-.PP-Consider adding this rule if all of the following are true: you might be-processing just one day of data, your CSV records are in reverse-chronological order (newest first), and you care about preserving the-order of same-day transactions.-It usually isn\[aq]t needed, because hledger autodetects the CSV order,-but when all CSV records have the same date it will assume they are-oldest first.-.SH CSV TIPS-.SS CSV ordering-.PP-The generated journal entries will be sorted by date.-The order of same-day entries will be preserved (except in the special-case where you might need \f[C]newest-first\f[R], see above).-.SS CSV accounts-.PP-Each journal entry will have two postings, to \f[C]account1\f[R] and-\f[C]account2\f[R] respectively.-It\[aq]s not yet possible to generate entries with more than two-postings.-It\[aq]s conventional and recommended to use \f[C]account1\f[R] for the-account whose CSV we are reading.-.SS CSV amounts-.PP-A transaction amount must be set, in one of these ways:-.IP \[bu] 2-with an \f[C]amount\f[R] field assignment, which sets the first-posting\[aq]s amount-.IP \[bu] 2-(When the CSV has debit and credit amounts in separate fields:)-.PD 0-.P-.PD-with field assignments for the \f[C]amount-in\f[R] and-\f[C]amount-out\f[R] pseudo fields (both of them).-Whichever one has a value will be used, with appropriate sign.-If both contain a value, it might not work so well.-.IP \[bu] 2-or implicitly by means of a balance assignment (see below).-.PP-There is some special handling for sign in amounts:-.IP \[bu] 2-If an amount value is parenthesised, it will be de-parenthesised and-sign-flipped.-.IP \[bu] 2-If an amount value begins with a double minus sign, those will cancel-out and be removed.-.PP-If the currency/commodity symbol is provided as a separate CSV field,-assign it to the \f[C]currency\f[R] pseudo field; the symbol will be-prepended to the amount (TODO: when there is an amount).-Or, you can use an \f[C]amount\f[R] field assignment for more control,-eg:-.IP-.nf-\f[C]-fields date,description,currency,amount-amount %amount %currency-\f[R]-.fi-.SS CSV balance assertions/assignments-.PP-If the CSV includes a running balance, you can assign that to one of the-pseudo fields \f[C]balance\f[R] (or \f[C]balance1\f[R]) or-\f[C]balance2\f[R].-This will generate a balance assertion (or if the amount is left empty,-a balance assignment), on the first or second posting, whenever the-running balance field is non-empty.-(TODO: #1000)-.SS Reading multiple CSV files-.PP-You can read multiple CSV files at once using multiple \f[C]-f\f[R]-arguments on the command line, and hledger will look for a-correspondingly-named rules file for each.-Note if you use the \f[C]--rules-file\f[R] option, this one rules file-will be used for all the CSV files being read.-.SS Valid CSV-.PP-hledger follows RFC 4180, with the addition of a customisable separator-character.-.PP-Some things to note:-.PP-When quoting fields,-.IP \[bu] 2-you must use double quotes, not single quotes-.IP \[bu] 2-spaces outside the quotes are not allowed.---.SH "REPORTING BUGS"-Report bugs at http://bugs.hledger.org-(or on the #hledger IRC channel or hledger mail list)--.SH AUTHORS-Simon Michael <simon@joyful.com> and contributors--.SH COPYRIGHT--Copyright (C) 2007-2016 Simon Michael.+.\"t++.TH "hledger_csv" "5" "December 2019" "hledger 1.16" "hledger User Manuals"++++.SH NAME+.PP+CSV - how hledger reads CSV data, and the CSV rules file format+.SH DESCRIPTION+.PP+hledger can read CSV (comma-separated value, or character-separated+value) files as if they were journal files, automatically converting+each CSV record into a transaction.+(To learn about \f[I]writing\f[R] CSV, see CSV output.)+.PP+We describe each CSV file\[aq]s format with a corresponding \f[I]rules+file\f[R].+By default this is named like the CSV file with a \f[C].rules\f[R]+extension added.+Eg when reading \f[C]FILE.csv\f[R], hledger also looks for+\f[C]FILE.csv.rules\f[R] in the same directory.+You can specify a different rules file with the \f[C]--rules-file\f[R]+option.+If a rules file is not found, hledger will create a sample rules file,+which you\[aq]ll need to adjust.+.PP+This file contains rules describing the CSV data (header line, fields+layout, date format etc.), and how to construct hledger journal entries+(transactions) from it.+Often there will also be a list of conditional rules for categorising+transactions based on their descriptions.+Here\[aq]s an overview of the CSV rules; these are described more fully+below, after the examples:+.PP+.TS+tab(@);+l l.+T{+\f[B]\f[CB]skip\f[B]\f[R]+T}@T{+skip one or more header lines or matched CSV records+T}+T{+\f[B]\f[CB]fields\f[B]\f[R]+T}@T{+name CSV fields, assign them to hledger fields+T}+T{+\f[B]field assignment\f[R]+T}@T{+assign a value to one hledger field, with interpolation+T}+T{+\f[B]\f[CB]if\f[B]\f[R]+T}@T{+apply some rules to matched CSV records+T}+T{+\f[B]\f[CB]end\f[B]\f[R]+T}@T{+skip the remaining CSV records+T}+T{+\f[B]\f[CB]date-format\f[B]\f[R]+T}@T{+describe the format of CSV dates+T}+T{+\f[B]\f[CB]newest-first\f[B]\f[R]+T}@T{+disambiguate record order when there\[aq]s only one date+T}+T{+\f[B]\f[CB]include\f[B]\f[R]+T}@T{+inline another CSV rules file+T}+.TE+.PP+There\[aq]s also a Convert CSV files tutorial on hledger.org.+.SH EXAMPLES+.PP+Here are some sample hledger CSV rules files.+See also the full collection at:+.PD 0+.P+.PD+https://github.com/simonmichael/hledger/tree/master/examples/csv+.SS Basic+.PP+At minimum, the rules file must identify the date and amount fields, and+often it also specifies the date format and how many header lines there+are.+Here\[aq]s a simple CSV file and a rules file for it:+.IP+.nf+\f[C]+Date, Description, Id, Amount+12/11/2019, Foo, 123, 10.23+\f[R]+.fi+.IP+.nf+\f[C]+# basic.csv.rules+skip         1+fields       date, description, _, amount+date-format  %d/%m/%Y+\f[R]+.fi+.IP+.nf+\f[C]+$ hledger print -f basic.csv+2019/11/12 Foo+    expenses:unknown           10.23+    income:unknown            -10.23+\f[R]+.fi+.PP+Default account names are chosen, since we didn\[aq]t set them.+.SS Bank of Ireland+.PP+Here\[aq]s a CSV with two amount fields (Debit and Credit), and a+balance field, which we can use to add balance assertions, which is not+necessary but provides extra error checking:+.IP+.nf+\f[C]+Date,Details,Debit,Credit,Balance+07/12/2012,LODGMENT       529898,,10.0,131.21+07/12/2012,PAYMENT,5,,126+\f[R]+.fi+.IP+.nf+\f[C]+# bankofireland-checking.csv.rules++# skip the header line+skip++# name the csv fields, and assign some of them as journal entry fields+fields  date, description, amount-out, amount-in, balance++# We generate balance assertions by assigning to \[dq]balance\[dq]+# above, but you may sometimes need to remove these because:+#+# - the CSV balance differs from the true balance, +#   by up to 0.0000000000005 in my experience+#+# - it is sometimes calculated based on non-chronological ordering,+#   eg when multiple transactions clear on the same day++# date is in UK/Ireland format+date-format  %d/%m/%Y++# set the currency+currency  EUR++# set the base account for all txns+account1  assets:bank:boi:checking+\f[R]+.fi+.IP+.nf+\f[C]+$ hledger -f bankofireland-checking.csv print+2012/12/07 LODGMENT       529898+    assets:bank:boi:checking         EUR10.0 = EUR131.2+    income:unknown                  EUR-10.0++2012/12/07 PAYMENT+    assets:bank:boi:checking         EUR-5.0 = EUR126.0+    expenses:unknown                  EUR5.0+\f[R]+.fi+.PP+The balance assertions don\[aq]t raise an error above, because we\[aq]re+reading directly from CSV, but they will be checked if these entries are+imported into a journal file.+.SS Amazon+.PP+Here we convert amazon.com order history, and use an if block to+generate a third posting if there\[aq]s a fee.+(In practice you\[aq]d probably get this data from your bank instead,+but it\[aq]s an example.)+.IP+.nf+\f[C]+\[dq]Date\[dq],\[dq]Type\[dq],\[dq]To/From\[dq],\[dq]Name\[dq],\[dq]Status\[dq],\[dq]Amount\[dq],\[dq]Fees\[dq],\[dq]Transaction ID\[dq]+\[dq]Jul 29, 2012\[dq],\[dq]Payment\[dq],\[dq]To\[dq],\[dq]Foo.\[dq],\[dq]Completed\[dq],\[dq]$20.00\[dq],\[dq]$0.00\[dq],\[dq]16000000000000DGLNJPI1P9B8DKPVHL\[dq]+\[dq]Jul 30, 2012\[dq],\[dq]Payment\[dq],\[dq]To\[dq],\[dq]Adapteva, Inc.\[dq],\[dq]Completed\[dq],\[dq]$25.00\[dq],\[dq]$1.00\[dq],\[dq]17LA58JSKRD4HDGLNJPI1P9B8DKPVHL\[dq]+\f[R]+.fi+.IP+.nf+\f[C]+# amazon-orders.csv.rules++# skip one header line+skip 1++# name the csv fields, and assign the transaction\[aq]s date, amount and code.+# Avoided the \[dq]status\[dq] and \[dq]amount\[dq] hledger field names to prevent confusion.+fields date, _, toorfrom, name, amzstatus, amzamount, fees, code++# how to parse the date+date-format %b %-d, %Y++# combine two fields to make the description+description %toorfrom %name++# save the status as a tag+comment     status:%amzstatus++# set the base account for all transactions+account1    assets:amazon+# leave amount1 blank so it can balance the other(s).+# I\[aq]m assuming amzamount excludes the fees, don\[aq]t remember++# set a generic account2+account2    expenses:misc+amount2     %amzamount+# and maybe refine it further:+#include categorisation.rules++# add a third posting for fees, but only if they are non-zero.+# Commas in the data makes counting fields hard, so count from the right instead.+# (Regex translation: \[dq]a field containing a non-zero dollar amount, +# immediately before the 1 right-most fields\[dq])+if ,\[rs]$[1-9][.0-9]+(,[\[ha],]*){1}$+ account3    expenses:fees+ amount3     %fees+\f[R]+.fi+.IP+.nf+\f[C]+$ hledger -f amazon-orders.csv print+2012/07/29 (16000000000000DGLNJPI1P9B8DKPVHL) To Foo.  ; status:Completed+    assets:amazon+    expenses:misc          $20.00++2012/07/30 (17LA58JSKRD4HDGLNJPI1P9B8DKPVHL) To Adapteva, Inc.  ; status:Completed+    assets:amazon+    expenses:misc          $25.00+    expenses:fees           $1.00+\f[R]+.fi+.SS Paypal+.PP+Here\[aq]s a real-world rules file for (customised) Paypal CSV, with+some Paypal-specific rules, and a second rules file included:+.IP+.nf+\f[C]+\[dq]Date\[dq],\[dq]Time\[dq],\[dq]TimeZone\[dq],\[dq]Name\[dq],\[dq]Type\[dq],\[dq]Status\[dq],\[dq]Currency\[dq],\[dq]Gross\[dq],\[dq]Fee\[dq],\[dq]Net\[dq],\[dq]From Email Address\[dq],\[dq]To Email Address\[dq],\[dq]Transaction ID\[dq],\[dq]Item Title\[dq],\[dq]Item ID\[dq],\[dq]Reference Txn ID\[dq],\[dq]Receipt ID\[dq],\[dq]Balance\[dq],\[dq]Note\[dq]+\[dq]10/01/2019\[dq],\[dq]03:46:20\[dq],\[dq]PDT\[dq],\[dq]Calm Radio\[dq],\[dq]Subscription Payment\[dq],\[dq]Completed\[dq],\[dq]USD\[dq],\[dq]-6.99\[dq],\[dq]0.00\[dq],\[dq]-6.99\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]memberships\[at]calmradio.com\[dq],\[dq]60P57143A8206782E\[dq],\[dq]MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month\[dq],\[dq]\[dq],\[dq]I-R8YLY094FJYR\[dq],\[dq]\[dq],\[dq]-6.99\[dq],\[dq]\[dq]+\[dq]10/01/2019\[dq],\[dq]03:46:20\[dq],\[dq]PDT\[dq],\[dq]\[dq],\[dq]Bank Deposit to PP Account \[dq],\[dq]Pending\[dq],\[dq]USD\[dq],\[dq]6.99\[dq],\[dq]0.00\[dq],\[dq]6.99\[dq],\[dq]\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]0TU1544T080463733\[dq],\[dq]\[dq],\[dq]\[dq],\[dq]60P57143A8206782E\[dq],\[dq]\[dq],\[dq]0.00\[dq],\[dq]\[dq]+\[dq]10/01/2019\[dq],\[dq]08:57:01\[dq],\[dq]PDT\[dq],\[dq]Patreon\[dq],\[dq]PreApproved Payment Bill User Payment\[dq],\[dq]Completed\[dq],\[dq]USD\[dq],\[dq]-7.00\[dq],\[dq]0.00\[dq],\[dq]-7.00\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]support\[at]patreon.com\[dq],\[dq]2722394R5F586712G\[dq],\[dq]Patreon* Membership\[dq],\[dq]\[dq],\[dq]B-0PG93074E7M86381M\[dq],\[dq]\[dq],\[dq]-7.00\[dq],\[dq]\[dq]+\[dq]10/01/2019\[dq],\[dq]08:57:01\[dq],\[dq]PDT\[dq],\[dq]\[dq],\[dq]Bank Deposit to PP Account \[dq],\[dq]Pending\[dq],\[dq]USD\[dq],\[dq]7.00\[dq],\[dq]0.00\[dq],\[dq]7.00\[dq],\[dq]\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]71854087RG994194F\[dq],\[dq]Patreon* Membership\[dq],\[dq]\[dq],\[dq]2722394R5F586712G\[dq],\[dq]\[dq],\[dq]0.00\[dq],\[dq]\[dq]+\[dq]10/19/2019\[dq],\[dq]03:02:12\[dq],\[dq]PDT\[dq],\[dq]Wikimedia Foundation, Inc.\[dq],\[dq]Subscription Payment\[dq],\[dq]Completed\[dq],\[dq]USD\[dq],\[dq]-2.00\[dq],\[dq]0.00\[dq],\[dq]-2.00\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]tle\[at]wikimedia.org\[dq],\[dq]K9U43044RY432050M\[dq],\[dq]Monthly donation to the Wikimedia Foundation\[dq],\[dq]\[dq],\[dq]I-R5C3YUS3285L\[dq],\[dq]\[dq],\[dq]-2.00\[dq],\[dq]\[dq]+\[dq]10/19/2019\[dq],\[dq]03:02:12\[dq],\[dq]PDT\[dq],\[dq]\[dq],\[dq]Bank Deposit to PP Account \[dq],\[dq]Pending\[dq],\[dq]USD\[dq],\[dq]2.00\[dq],\[dq]0.00\[dq],\[dq]2.00\[dq],\[dq]\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]3XJ107139A851061F\[dq],\[dq]\[dq],\[dq]\[dq],\[dq]K9U43044RY432050M\[dq],\[dq]\[dq],\[dq]0.00\[dq],\[dq]\[dq]+\[dq]10/22/2019\[dq],\[dq]05:07:06\[dq],\[dq]PDT\[dq],\[dq]Noble Benefactor\[dq],\[dq]Subscription Payment\[dq],\[dq]Completed\[dq],\[dq]USD\[dq],\[dq]10.00\[dq],\[dq]-0.59\[dq],\[dq]9.41\[dq],\[dq]noble\[at]bene.fac.tor\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]6L8L1662YP1334033\[dq],\[dq]Joyful Systems\[dq],\[dq]\[dq],\[dq]I-KC9VBGY2GWDB\[dq],\[dq]\[dq],\[dq]9.41\[dq],\[dq]\[dq]+\f[R]+.fi+.IP+.nf+\f[C]+# paypal-custom.csv.rules++# Tips:+# Export from Activity -> Statements -> Custom -> Activity download+# Suggested transaction type: \[dq]Balance affecting\[dq]+# Paypal\[aq]s default fields in 2018 were:+# \[dq]Date\[dq],\[dq]Time\[dq],\[dq]TimeZone\[dq],\[dq]Name\[dq],\[dq]Type\[dq],\[dq]Status\[dq],\[dq]Currency\[dq],\[dq]Gross\[dq],\[dq]Fee\[dq],\[dq]Net\[dq],\[dq]From Email Address\[dq],\[dq]To Email Address\[dq],\[dq]Transaction ID\[dq],\[dq]Shipping Address\[dq],\[dq]Address Status\[dq],\[dq]Item Title\[dq],\[dq]Item ID\[dq],\[dq]Shipping and Handling Amount\[dq],\[dq]Insurance Amount\[dq],\[dq]Sales Tax\[dq],\[dq]Option 1 Name\[dq],\[dq]Option 1 Value\[dq],\[dq]Option 2 Name\[dq],\[dq]Option 2 Value\[dq],\[dq]Reference Txn ID\[dq],\[dq]Invoice Number\[dq],\[dq]Custom Number\[dq],\[dq]Quantity\[dq],\[dq]Receipt ID\[dq],\[dq]Balance\[dq],\[dq]Address Line 1\[dq],\[dq]Address Line 2/District/Neighborhood\[dq],\[dq]Town/City\[dq],\[dq]State/Province/Region/County/Territory/Prefecture/Republic\[dq],\[dq]Zip/Postal Code\[dq],\[dq]Country\[dq],\[dq]Contact Phone Number\[dq],\[dq]Subject\[dq],\[dq]Note\[dq],\[dq]Country Code\[dq],\[dq]Balance Impact\[dq]+# This rules file assumes the following more detailed fields, configured in \[dq]Customize report fields\[dq]:+# \[dq]Date\[dq],\[dq]Time\[dq],\[dq]TimeZone\[dq],\[dq]Name\[dq],\[dq]Type\[dq],\[dq]Status\[dq],\[dq]Currency\[dq],\[dq]Gross\[dq],\[dq]Fee\[dq],\[dq]Net\[dq],\[dq]From Email Address\[dq],\[dq]To Email Address\[dq],\[dq]Transaction ID\[dq],\[dq]Item Title\[dq],\[dq]Item ID\[dq],\[dq]Reference Txn ID\[dq],\[dq]Receipt ID\[dq],\[dq]Balance\[dq],\[dq]Note\[dq]++fields date, time, timezone, description_, type, status_, currency, grossamount, feeamount, netamount, fromemail, toemail, code, itemtitle, itemid, referencetxnid, receiptid, balance, note++skip  1++date-format  %-m/%-d/%Y++# ignore some paypal events+if+In Progress+Temporary Hold+Update to + skip++# add more fields to the description+description %description_ %itemtitle ++# save some other fields as tags+comment  itemid:%itemid, fromemail:%fromemail, toemail:%toemail, time:%time, type:%type, status:%status_++# convert to short currency symbols+# Note: in conditional block regexps, the line of csv being matched is+# a synthetic one: the unquoted field values, with commas between them.+if ,USD,+ currency $+if ,EUR,+ currency E+if ,GBP,+ currency P++# generate postings++# the first posting will be the money leaving/entering my paypal account+# (negative means leaving my account, in all amount fields)+account1 assets:online:paypal+amount1  %netamount++# the second posting will be money sent to/received from other party+# (account2 is set below)+amount2  -%grossamount++# if there\[aq]s a fee (9th field), add a third posting for the money taken by paypal.+# TODO: This regexp fails when fields contain a comma (generates a third posting with zero amount)+if \[ha]([\[ha],]+,){8}[\[ha]0]+ account3 expenses:banking:paypal+ amount3  -%feeamount+ comment3 business:++# choose an account for the second posting++# override the default account names:+# if amount (8th field) is positive, it\[aq]s income (a debit)+if \[ha]([\[ha],]+,){7}[0-9]+ account2 income:unknown+# if negative, it\[aq]s an expense (a credit)+if \[ha]([\[ha],]+,){7}-+ account2 expenses:unknown++# apply common rules for setting account2 & other tweaks+include common.rules++# apply some overrides specific to this csv++# Transfers from/to bank. These are usually marked Pending, +# which can be disregarded in this case.+if +Bank Account+Bank Deposit to PP Account+ description %type for %referencetxnid %itemtitle+ account2 assets:bank:wf:pchecking+ account1 assets:online:paypal++# Currency conversions+if Currency Conversion+ account2 equity:currency conversion+\f[R]+.fi+.IP+.nf+\f[C]+# common.rules++if+darcs+noble benefactor+ account2 revenues:foss donations:darcshub+ comment2 business:++if+Calm Radio+ account2 expenses:online:apps++if+electronic frontier foundation+Patreon+wikimedia+Advent of Code+ account2 expenses:dues++if Google+ account2 expenses:online:apps+ description google | music+\f[R]+.fi+.IP+.nf+\f[C]+$ hledger -f paypal-custom.csv  print+2019/10/01 (60P57143A8206782E) Calm Radio MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month  ; itemid:, fromemail:simon\[at]joyful.com, toemail:memberships\[at]calmradio.com, time:03:46:20, type:Subscription Payment, status:Completed+    assets:online:paypal          $-6.99 = $-6.99+    expenses:online:apps           $6.99++2019/10/01 (0TU1544T080463733) Bank Deposit to PP Account for 60P57143A8206782E  ; itemid:, fromemail:, toemail:simon\[at]joyful.com, time:03:46:20, type:Bank Deposit to PP Account, status:Pending+    assets:online:paypal               $6.99 = $0.00+    assets:bank:wf:pchecking          $-6.99++2019/10/01 (2722394R5F586712G) Patreon Patreon* Membership  ; itemid:, fromemail:simon\[at]joyful.com, toemail:support\[at]patreon.com, time:08:57:01, type:PreApproved Payment Bill User Payment, status:Completed+    assets:online:paypal          $-7.00 = $-7.00+    expenses:dues                  $7.00++2019/10/01 (71854087RG994194F) Bank Deposit to PP Account for 2722394R5F586712G Patreon* Membership  ; itemid:, fromemail:, toemail:simon\[at]joyful.com, time:08:57:01, type:Bank Deposit to PP Account, status:Pending+    assets:online:paypal               $7.00 = $0.00+    assets:bank:wf:pchecking          $-7.00++2019/10/19 (K9U43044RY432050M) Wikimedia Foundation, Inc. Monthly donation to the Wikimedia Foundation  ; itemid:, fromemail:simon\[at]joyful.com, toemail:tle\[at]wikimedia.org, time:03:02:12, type:Subscription Payment, status:Completed+    assets:online:paypal             $-2.00 = $-2.00+    expenses:dues                     $2.00+    expenses:banking:paypal      ; business:++2019/10/19 (3XJ107139A851061F) Bank Deposit to PP Account for K9U43044RY432050M  ; itemid:, fromemail:, toemail:simon\[at]joyful.com, time:03:02:12, type:Bank Deposit to PP Account, status:Pending+    assets:online:paypal               $2.00 = $0.00+    assets:bank:wf:pchecking          $-2.00++2019/10/22 (6L8L1662YP1334033) Noble Benefactor Joyful Systems  ; itemid:, fromemail:noble\[at]bene.fac.tor, toemail:simon\[at]joyful.com, time:05:07:06, type:Subscription Payment, status:Completed+    assets:online:paypal                       $9.41 = $9.41+    revenues:foss donations:darcshub         $-10.00  ; business:+    expenses:banking:paypal                    $0.59  ; business:+\f[R]+.fi+.SH CSV RULES+.PP+The following kinds of rule can appear in the rules file, in any order.+Blank lines and lines beginning with \f[C]#\f[R] or \f[C];\f[R] are+ignored.+.SS \f[C]skip\f[R]+.IP+.nf+\f[C]+skip N+\f[R]+.fi+.PP+The word \[dq]skip\[dq] followed by a number (or no number, meaning 1)+tells hledger to ignore this many non-empty lines preceding the CSV+data.+(Empty/blank lines are skipped automatically.) You\[aq]ll need this+whenever your CSV data contains header lines.+.PP+It also has a second purpose: it can be used inside if blocks to ignore+certain CSV records (described below).+.SS \f[C]fields\f[R]+.IP+.nf+\f[C]+fields FIELDNAME1, FIELDNAME2, ...+\f[R]+.fi+.PP+A fields list (the word \[dq]fields\[dq] followed by comma-separated+field names) is the quick way to assign CSV field values to hledger+fields.+It does two things:+.IP "1." 3+it names the CSV fields.+This is optional, but can be convenient later for interpolating them.+.IP "2." 3+when you use a standard hledger field name, it assigns the CSV value to+that part of the hledger transaction.+.PP+Here\[aq]s an example that says \[dq]use the 1st, 2nd and 4th fields as+the transaction\[aq]s date, description and amount; name the last two+fields for later reference; and ignore the others\[dq]:+.IP+.nf+\f[C]+fields date, description, , amount, , , somefield, anotherfield+\f[R]+.fi+.PP+Field names may not contain whitespace.+Fields you don\[aq]t care about can be left unnamed.+Currently there must be least two items (there must be at least one+comma).+.PP+Here are the standard hledger field/pseudo-field names.+For more about the transaction parts they refer to, see the manual for+hledger\[aq]s journal format.+.SS Transaction field names+.PP+\f[C]date\f[R], \f[C]date2\f[R], \f[C]status\f[R], \f[C]code\f[R],+\f[C]description\f[R], \f[C]comment\f[R] can be used to form the+transaction\[aq]s first line.+.SS Posting field names+.PP+\f[C]accountN\f[R], where N is 1 to 9, generates a posting, with that+account name.+Most often there are two postings, so you\[aq]ll want to set+\f[C]account1\f[R] and \f[C]account2\f[R].+If a posting\[aq]s account name is left unset but its amount is set, a+default account name will be chosen (like expenses:unknown or+income:unknown).+.PP+\f[C]amountN\f[R] sets posting N\[aq]s amount.+Or, \f[C]amount\f[R] with no N sets posting 1\[aq]s.+If the CSV has debits and credits in separate fields, use+\f[C]amountN-in\f[R] and \f[C]amountN-out\f[R] instead.+Or \f[C]amount-in\f[R] and \f[C]amount-out\f[R] with no N for posting 1.+.PP+For convenience and backwards compatibility, if you set the amount of+posting 1 only, a second posting with the negative amount will be+generated automatically.+(Unless the account name is parenthesised indicating an unbalanced+posting.)+.PP+If the CSV has the currency symbol in a separate field, you can use+\f[C]currencyN\f[R] to prepend it to posting N\[aq]s amount.+\f[C]currency\f[R] with no N affects ALL postings.+.PP+\f[C]balanceN\f[R] sets a balance assertion amount (or if the posting+amount is left empty, a balance assignment).+.PP+Finally, \f[C]commentN\f[R] sets a comment on the Nth posting.+Comments can also contain tags, as usual.+.PP+See TIPS below for more about setting amounts and currency.+.SS field assignment+.IP+.nf+\f[C]+HLEDGERFIELDNAME FIELDVALUE+\f[R]+.fi+.PP+Instead of or in addition to a fields list, you can use a \[dq]field+assignment\[dq] rule to set the value of a single hledger field, by+writing its name (any of the standard hledger field names above)+followed by a text value.+The value may contain interpolated CSV fields, referenced by their+1-based position in the CSV record (\f[C]%N\f[R]), or by the name they+were given in the fields list (\f[C]%CSVFIELDNAME\f[R]).+Some examples:+.IP+.nf+\f[C]+# set the amount to the 4th CSV field, with \[dq] USD\[dq] appended+amount %4 USD++# combine three fields to make a comment, containing note: and date: tags+comment note: %somefield - %anotherfield, date: %1+\f[R]+.fi+.PP+Interpolation strips outer whitespace (so a CSV value like+\f[C]\[dq] 1 \[dq]\f[R] becomes \f[C]1\f[R] when interpolated) (#1051).+See TIPS below for more about referencing other fields.+.SS \f[C]if\f[R]+.IP+.nf+\f[C]+if PATTERN+ RULE++if+PATTERN+PATTERN+PATTERN+ RULE+ RULE+\f[R]+.fi+.PP+Conditional blocks (\[dq]if blocks\[dq]) are a block of rules that are+applied only to CSV records which match certain patterns.+They are often used for customising account names based on transaction+descriptions.+.PP+A single pattern can be written on the same line as the \[dq]if\[dq]; or+multiple patterns can be written on the following lines, non-indented.+Multiple patterns are OR\[aq]d (any one of them can match).+Patterns are case-insensitive regular expressions which try to match+anywhere within the whole CSV record (POSIX extended regular expressions+with some additions, see+https://hledger.org/hledger.html#regular-expressions).+Note the CSV record they see is close to, but not identical to, the one+in the CSV file; enclosing double quotes will be removed, and the+separator character is always comma.+.PP+It\[aq]s not yet easy to match within a specific field.+If the data does not contain commas, you can hack it with a regular+expression like:+.IP+.nf+\f[C]+# match \[dq]foo\[dq] in the fourth field+if \[ha]([\[ha],]*,){3}foo+\f[R]+.fi+.PP+After the patterns there should be one or more rules to apply, all+indented by at least one space.+Three kinds of rule are allowed in conditional blocks:+.IP \[bu] 2+field assignments (to set a hledger field)+.IP \[bu] 2+skip (to skip the matched CSV record)+.IP \[bu] 2+end (to skip all remaining CSV records).+.PP+Examples:+.IP+.nf+\f[C]+# if the CSV record contains \[dq]groceries\[dq], set account2 to \[dq]expenses:groceries\[dq]+if groceries+ account2 expenses:groceries+\f[R]+.fi+.IP+.nf+\f[C]+# if the CSV record contains any of these patterns, set account2 and comment as shown+if+monthly service fee+atm transaction fee+banking thru software+ account2 expenses:business:banking+ comment  XXX deductible ? check it+\f[R]+.fi+.SS \f[C]end\f[R]+.PP+This rule can be used inside if blocks (only), to make hledger stop+reading this CSV file and move on to the next input file, or to command+execution.+Eg:+.IP+.nf+\f[C]+# ignore everything following the first empty record+if ,,,,+ end+\f[R]+.fi+.SS \f[C]date-format\f[R]+.IP+.nf+\f[C]+date-format DATEFMT+\f[R]+.fi+.PP+This is a helper for the \f[C]date\f[R] (and \f[C]date2\f[R]) fields.+If your CSV dates are not formatted like \f[C]YYYY-MM-DD\f[R],+\f[C]YYYY/MM/DD\f[R] or \f[C]YYYY.MM.DD\f[R], you\[aq]ll need to add a+date-format rule describing them with a strptime date parsing pattern,+which must parse the CSV date value completely.+Some examples:+.IP+.nf+\f[C]+# MM/DD/YY+date-format %m/%d/%y+\f[R]+.fi+.IP+.nf+\f[C]+# D/M/YYYY+# The - makes leading zeros optional.+date-format %-d/%-m/%Y+\f[R]+.fi+.IP+.nf+\f[C]+# YYYY-Mmm-DD+date-format %Y-%h-%d+\f[R]+.fi+.IP+.nf+\f[C]+# M/D/YYYY HH:MM AM some other junk+# Note the time and junk must be fully parsed, though only the date is used.+date-format %-m/%-d/%Y %l:%M %p some other junk+\f[R]+.fi+.PP+For the supported strptime syntax, see:+.PD 0+.P+.PD+https://hackage.haskell.org/package/time/docs/Data-Time-Format.html#v:formatTime+.SS \f[C]newest-first\f[R]+.PP+hledger always sorts the generated transactions by date.+Transactions on the same date should appear in the same order as their+CSV records, as hledger can usually auto-detect whether the CSV\[aq]s+normal order is oldest first or newest first.+But if all of the following are true:+.IP \[bu] 2+the CSV might sometimes contain just one day of data (all records having+the same date)+.IP \[bu] 2+the CSV records are normally in reverse chronological order (newest at+the top)+.IP \[bu] 2+and you care about preserving the order of same-day transactions+.PP+then, you should add the \f[C]newest-first\f[R] rule as a hint.+Eg:+.IP+.nf+\f[C]+# tell hledger explicitly that the CSV is normally newest first+newest-first+\f[R]+.fi+.SS \f[C]include\f[R]+.IP+.nf+\f[C]+include RULESFILE+\f[R]+.fi+.PP+This includes the contents of another CSV rules file at this point.+\f[C]RULESFILE\f[R] is an absolute file path or a path relative to the+current file\[aq]s directory.+This can be useful for sharing common rules between several rules files,+eg:+.IP+.nf+\f[C]+# someaccount.csv.rules++## someaccount-specific rules+fields   date,description,amount+account1 assets:someaccount+account2 expenses:misc++## common rules+include categorisation.rules+\f[R]+.fi+.SH TIPS+.SS Valid CSV+.PP+hledger accepts CSV conforming to RFC 4180.+When CSV values are enclosed in quotes, note:+.IP \[bu] 2+they must be double quotes (not single quotes)+.IP \[bu] 2+spaces outside the quotes are not allowed+.SS Other separator characters+.PP+With the \f[C]--separator \[aq]CHAR\[aq]\f[R] option (experimental),+hledger will expect the separator to be CHAR instead of a comma.+Ie it will read other \[dq]Character Separated Values\[dq] formats, such+as TSV (Tab Separated Values).+Note: on the command line, use a real tab character in quotes, not Eg:+.IP+.nf+\f[C]+$ hledger -f foo.tsv --separator \[aq]  \[aq] print+\f[R]+.fi+.SS Reading multiple CSV files+.PP+If you use multiple \f[C]-f\f[R] options to read multiple CSV files at+once, hledger will look for a correspondingly-named rules file for each+CSV file.+But if you use the \f[C]--rules-file\f[R] option, that rules file will+be used for all the CSV files.+.SS Valid transactions+.PP+After reading a CSV file, hledger post-processes and validates the+generated journal entries as it would for a journal file - balancing+them, applying balance assignments, and canonicalising amount styles.+Any errors at this stage will be reported in the usual way, displaying+the problem entry.+.PP+There is one exception: balance assertions, if you have generated them,+will not be checked, since normally these will work only when the CSV+data is part of the main journal.+If you do need to check balance assertions generated from CSV right+away, pipe into another hledger:+.IP+.nf+\f[C]+$ hledger -f file.csv print | hledger -f- print+\f[R]+.fi+.SS Deduplicating, importing+.PP+When you download a CSV file periodically, eg to get your latest bank+transactions, the new file may overlap with the old one, containing some+of the same records.+.PP+The import command will (a) detect the new transactions, and (b) append+just those transactions to your main journal.+It is idempotent, so you don\[aq]t have to remember how many times you+ran it or with which version of the CSV.+(It keeps state in a hidden \f[C].latest.FILE.csv\f[R] file.) This is+the easiest way to import CSV data.+Eg:+.IP+.nf+\f[C]+# download the latest CSV files, then run this command.+# Note, no -f flags needed here.+$ hledger import *.csv [--dry]+\f[R]+.fi+.PP+This method works for most CSV files.+(Where records have a stable chronological order, and new records appear+only at the new end.)+.PP+A number of other tools and workflows, hledger-specific and otherwise,+exist for converting, deduplicating, classifying and managing CSV data.+See:+.IP \[bu] 2+https://hledger.org -> sidebar -> real world setups+.IP \[bu] 2+https://plaintextaccounting.org -> data import/conversion+.SS Setting amounts+.PP+A posting amount can be set in one of these ways:+.IP \[bu] 2+by assigning (with a fields list or field assigment) to+\f[C]amountN\f[R] (posting N\[aq]s amount) or \f[C]amount\f[R] (posting+1\[aq]s amount)+.IP \[bu] 2+by assigning to \f[C]amountN-in\f[R] and \f[C]amountN-out\f[R] (or+\f[C]amount-in\f[R] and \f[C]amount-out\f[R]).+For each CSV record, whichever of these has a non-zero value will be+used, with appropriate sign.+If both contain a non-zero value, this may not work.+.IP \[bu] 2+by assigning to \f[C]balanceN\f[R] (or \f[C]balance\f[R]) instead of the+above, setting the amount indirectly via a balance assignment.+If you do this the default account name may be wrong, so you should set+that explicitly.+.PP+There is some special handling for an amount\[aq]s sign:+.IP \[bu] 2+If an amount value is parenthesised, it will be de-parenthesised and+sign-flipped.+.IP \[bu] 2+If an amount value begins with a double minus sign, those cancel out and+are removed.+.IP \[bu] 2+If an amount value begins with a plus sign, that will be removed+.SS Setting currency/commodity+.PP+If the currency/commodity symbol is included in the CSV\[aq]s amount+field(s), you don\[aq]t have to do anything special.+.PP+If the currency is provided as a separate CSV field, you can either:+.IP \[bu] 2+assign that to \f[C]currency\f[R], which adds it to all posting amounts.+The symbol will prepended to the amount quantity (on the left side).+If you write a trailing space after the symbol, there will be a space+between symbol and amount (an exception to the usual whitespace+stripping).+.IP \[bu] 2+or assign it to \f[C]currencyN\f[R] which adds it to posting N\[aq]s+amount only.+.IP \[bu] 2+or for more control, construct the amount from symbol and quantity using+field assignment, eg:+.RS 2+.IP+.nf+\f[C]+fields date,description,currency,quantity+# add currency symbol on the right:+amount %quantity %currency+\f[R]+.fi+.RE+.SS Referencing other fields+.PP+In field assignments, you can interpolate only CSV fields, not hledger+fields.+In the example below, there\[aq]s both a CSV field and a hledger field+named amount1, but %amount1 always means the CSV field, not the hledger+field:+.IP+.nf+\f[C]+# Name the third CSV field \[dq]amount1\[dq]+fields date,description,amount1++# Set hledger\[aq]s amount1 to the CSV amount1 field followed by USD+amount1 %amount1 USD++# Set comment to the CSV amount1 (not the amount1 assigned above)+comment %amount1+\f[R]+.fi+.PP+Here, since there\[aq]s no CSV amount1 field, %amount1 will produce a+literal \[dq]amount1\[dq]:+.IP+.nf+\f[C]+fields date,description,csvamount+amount1 %csvamount USD+# Can\[aq]t interpolate amount1 here+comment %amount1+\f[R]+.fi+.PP+When there are multiple field assignments to the same hledger field,+only the last one takes effect.+Here, comment\[aq]s value will be be B, or C if \[dq]something\[dq] is+matched, but never A:+.IP+.nf+\f[C]+comment A+comment B+if something+ comment C+\f[R]+.fi+.SS How CSV rules are evaluated+.PP+Here\[aq]s how to think of CSV rules being evaluated (if you really need+to).+First,+.IP \[bu] 2+\f[C]include\f[R] - all includes are inlined, from top to bottom, depth+first.+(At each include point the file is inlined and scanned for further+includes, recursively, before proceeding.)+.PP+Then \[dq]global\[dq] rules are evaluated, top to bottom.+If a rule is repeated, the last one wins:+.IP \[bu] 2+\f[C]skip\f[R] (at top level)+.IP \[bu] 2+\f[C]date-format\f[R]+.IP \[bu] 2+\f[C]newest-first\f[R]+.IP \[bu] 2+\f[C]fields\f[R] - names the CSV fields, optionally sets up initial+assignments to hledger fields+.PP+Then for each CSV record in turn:+.IP \[bu] 2+test all \f[C]if\f[R] blocks.+If any of them contain a \f[C]end\f[R] rule, skip all remaining CSV+records.+Otherwise if any of them contain a \f[C]skip\f[R] rule, skip that many+CSV records.+If there are multiple matched \f[C]skip\f[R] rules, the first one wins.+.IP \[bu] 2+collect all field assignments at top level and in matched \f[C]if\f[R]+blocks.+When there are multiple assignments for a field, keep only the last one.+.IP \[bu] 2+compute a value for each hledger field - either the one that was+assigned to it (and interpolate the %CSVFIELDNAME references), or a+default+.IP \[bu] 2+generate a synthetic hledger transaction from these values.+.PP+This is all part of the CSV reader, one of several readers hledger can+use to parse input files.+When all files have been read successfully, the transactions are passed+as input to whichever hledger command the user specified.+++.SH "REPORTING BUGS"+Report bugs at http://bugs.hledger.org+(or on the #hledger IRC channel or hledger mail list)++.SH AUTHORS+Simon Michael <simon@joyful.com> and contributors++.SH COPYRIGHT++Copyright (C) 2007-2019 Simon Michael. .br Released under GNU GPL v3 or later. 

@@ -1,384 +1,944 @@ This is hledger_csv.info, produced by makeinfo version 6.5 from stdin.  -File: hledger_csv.info,  Node: Top,  Next: CSV RULES,  Up: (dir)--hledger_csv(5) hledger 1.15.2-*****************************--hledger can read CSV (comma-separated value) files as if they were-journal files, automatically converting each CSV record into a-transaction.  (To learn about _writing_ CSV, see CSV output.)--   Converting CSV to transactions requires some special conversion-rules.  These do several things:--   * they describe the layout and format of the CSV data-   * they can customize the generated journal entries using a simple-     templating language-   * they can add refinements based on patterns in the CSV data, eg-     categorizing transactions with more detailed account names.--   When reading a CSV file named 'FILE.csv', hledger looks for a-conversion rules file named 'FILE.csv.rules' in the same directory.  You-can override this with the '--rules-file' option.  If the rules file-does not exist, hledger will auto-create one with some example rules,-which you'll need to adjust.--   At minimum, the rules file must identify the date and amount fields.-It's often necessary to specify the date format, and the number of-header lines to skip, also.  Eg:--fields date, _, _, amount-date-format  %d/%m/%Y-skip 1--   A more complete example:--# hledger CSV rules for amazon.com order history--# sample:-# "Date","Type","To/From","Name","Status","Amount","Fees","Transaction ID"-# "Jul 29, 2012","Payment","To","Adapteva, Inc.","Completed","$25.00","$0.00","17LA58JSK6PRD4HDGLNJQPI1PB9N8DKPVHL"--# skip one header line-skip 1--# name the csv fields (and assign the transaction's date, amount and code)-fields date, _, toorfrom, name, amzstatus, amount, fees, code--# how to parse the date-date-format %b %-d, %Y--# combine two fields to make the description-description %toorfrom %name--# save these fields as tags-comment     status:%amzstatus, fees:%fees--# set the base account for all transactions-account1    assets:amazon--# flip the sign on the amount-amount      -%amount--   For more examples, see Convert CSV files.--* Menu:--* CSV RULES::-* CSV TIPS::---File: hledger_csv.info,  Node: CSV RULES,  Next: CSV TIPS,  Prev: Top,  Up: Top--1 CSV RULES-***********--The following seven kinds of rule can appear in the rules file, in any-order.  Blank lines and lines beginning with '#' or ';' are ignored.--* Menu:--* skip::-* date-format::-* field list::-* field assignment::-* conditional block::-* include::-* newest-first::---File: hledger_csv.info,  Node: skip,  Next: date-format,  Up: CSV RULES--1.1 skip-========--'skip'_'N'_--   Skip this number of CSV records at the beginning.  You'll need this-whenever your CSV data contains header lines.  Eg:--# ignore the first CSV line-skip 1---File: hledger_csv.info,  Node: date-format,  Next: field list,  Prev: skip,  Up: CSV RULES--1.2 date-format-===============--'date-format'_'DATEFMT'_--   When your CSV date fields are not formatted like 'YYYY/MM/DD' (or-'YYYY-MM-DD' or 'YYYY.MM.DD'), you'll need to specify the format.-DATEFMT is a strptime-like date parsing pattern, which must parse the-date field values completely.  Examples:--# for dates like "11/06/2013":-date-format %m/%d/%Y--# for dates like "6/11/2013" (note the - to make leading zeros optional):-date-format %-d/%-m/%Y--# for dates like "2013-Nov-06":-date-format %Y-%h-%d--# for dates like "11/6/2013 11:32 PM":-date-format %-m/%-d/%Y %l:%M %p---File: hledger_csv.info,  Node: field list,  Next: field assignment,  Prev: date-format,  Up: CSV RULES--1.3 field list-==============--'fields'_'FIELDNAME1'_, _'FIELDNAME2'_...--   This (a) names the CSV fields, in order (names may not contain-whitespace; uninteresting names may be left blank), and (b) assigns them-to journal entry fields if you use any of these standard field names:-'date', 'date2', 'status', 'code', 'description', 'comment', 'account1',-'account2', 'amount', 'amount-in', 'amount-out', 'currency', 'balance',-'balance1', 'balance2'.  Eg:--# use the 1st, 2nd and 4th CSV fields as the entry's date, description and amount,-# and give the 7th and 8th fields meaningful names for later reference:-#-# CSV field:-#      1     2            3 4       5 6 7          8-# entry field:-fields date, description, , amount, , , somefield, anotherfield---File: hledger_csv.info,  Node: field assignment,  Next: conditional block,  Prev: field list,  Up: CSV RULES--1.4 field assignment-====================--_'ENTRYFIELDNAME'_ _'FIELDVALUE'_--   This sets a journal entry field (one of the standard names above) to-the given text value, which can include CSV field values interpolated by-name ('%CSVFIELDNAME') or 1-based position ('%N').  Eg:--# set the amount to the 4th CSV field with "USD " prepended-amount USD %4--# combine three fields to make a comment (containing two tags)-comment note: %somefield - %anotherfield, date: %1--   Field assignments can be used instead of or in addition to a field-list.--   Note, interpolation strips any outer whitespace, so a CSV value like-'" 1 "' becomes '1' when interpolated (#1051).---File: hledger_csv.info,  Node: conditional block,  Next: include,  Prev: field assignment,  Up: CSV RULES--1.5 conditional block-=====================--'if' _'PATTERN'_-    _'FIELDASSIGNMENTS'_...--   'if'-_'PATTERN'_-_'PATTERN'_...-    _'FIELDASSIGNMENTS'_...--   This applies one or more field assignments, only to those CSV records-matched by one of the PATTERNs.  The patterns are case-insensitive-regular expressions which match anywhere within the whole CSV record-(it's not yet possible to match within a specific field).  When there-are multiple patterns they can be written on separate lines, unindented.-The field assignments are on separate lines indented by at least one-space.  Examples:--# if the CSV record contains "groceries", set account2 to "expenses:groceries"-if groceries- account2 expenses:groceries--# if the CSV record contains any of these patterns, set account2 and comment as shown-if-monthly service fee-atm transaction fee-banking thru software- account2 expenses:business:banking- comment  XXX deductible ? check it---File: hledger_csv.info,  Node: include,  Next: newest-first,  Prev: conditional block,  Up: CSV RULES--1.6 include-===========--'include'_'RULESFILE'_--   Include another rules file at this point.  'RULESFILE' is either an-absolute file path or a path relative to the current file's directory.-Eg:--# rules reused with several CSV files-include common.rules---File: hledger_csv.info,  Node: newest-first,  Prev: include,  Up: CSV RULES--1.7 newest-first-================--'newest-first'--   Consider adding this rule if all of the following are true: you might-be processing just one day of data, your CSV records are in reverse-chronological order (newest first), and you care about preserving the-order of same-day transactions.  It usually isn't needed, because-hledger autodetects the CSV order, but when all CSV records have the-same date it will assume they are oldest first.---File: hledger_csv.info,  Node: CSV TIPS,  Prev: CSV RULES,  Up: Top--2 CSV TIPS-**********--* Menu:--* CSV ordering::-* CSV accounts::-* CSV amounts::-* CSV balance assertions/assignments::-* Reading multiple CSV files::-* Valid CSV::---File: hledger_csv.info,  Node: CSV ordering,  Next: CSV accounts,  Up: CSV TIPS--2.1 CSV ordering-================--The generated journal entries will be sorted by date.  The order of-same-day entries will be preserved (except in the special case where you-might need 'newest-first', see above).---File: hledger_csv.info,  Node: CSV accounts,  Next: CSV amounts,  Prev: CSV ordering,  Up: CSV TIPS--2.2 CSV accounts-================--Each journal entry will have two postings, to 'account1' and 'account2'-respectively.  It's not yet possible to generate entries with more than-two postings.  It's conventional and recommended to use 'account1' for-the account whose CSV we are reading.---File: hledger_csv.info,  Node: CSV amounts,  Next: CSV balance assertions/assignments,  Prev: CSV accounts,  Up: CSV TIPS--2.3 CSV amounts-===============--A transaction amount must be set, in one of these ways:--   * with an 'amount' field assignment, which sets the first posting's-     amount--   * (When the CSV has debit and credit amounts in separate fields:)-     with field assignments for the 'amount-in' and 'amount-out' pseudo-     fields (both of them).  Whichever one has a value will be used,-     with appropriate sign.  If both contain a value, it might not work-     so well.--   * or implicitly by means of a balance assignment (see below).--   There is some special handling for sign in amounts:--   * If an amount value is parenthesised, it will be de-parenthesised-     and sign-flipped.-   * If an amount value begins with a double minus sign, those will-     cancel out and be removed.--   If the currency/commodity symbol is provided as a separate CSV field,-assign it to the 'currency' pseudo field; the symbol will be prepended-to the amount (TODO: when there is an amount).  Or, you can use an-'amount' field assignment for more control, eg:--fields date,description,currency,amount-amount %amount %currency---File: hledger_csv.info,  Node: CSV balance assertions/assignments,  Next: Reading multiple CSV files,  Prev: CSV amounts,  Up: CSV TIPS--2.4 CSV balance assertions/assignments-======================================--If the CSV includes a running balance, you can assign that to one of the-pseudo fields 'balance' (or 'balance1') or 'balance2'.  This will-generate a balance assertion (or if the amount is left empty, a balance-assignment), on the first or second posting, whenever the running-balance field is non-empty.  (TODO: #1000)---File: hledger_csv.info,  Node: Reading multiple CSV files,  Next: Valid CSV,  Prev: CSV balance assertions/assignments,  Up: CSV TIPS--2.5 Reading multiple CSV files-==============================--You can read multiple CSV files at once using multiple '-f' arguments on-the command line, and hledger will look for a correspondingly-named-rules file for each.  Note if you use the '--rules-file' option, this-one rules file will be used for all the CSV files being read.---File: hledger_csv.info,  Node: Valid CSV,  Prev: Reading multiple CSV files,  Up: CSV TIPS--2.6 Valid CSV-=============--hledger follows RFC 4180, with the addition of a customisable separator-character.--   Some things to note:--   When quoting fields,--   * you must use double quotes, not single quotes-   * spaces outside the quotes are not allowed.---Tag Table:-Node: Top72-Node: CSV RULES2165-Ref: #csv-rules2273-Node: skip2536-Ref: #skip2630-Node: date-format2802-Ref: #date-format2929-Node: field list3479-Ref: #field-list3616-Node: field assignment4346-Ref: #field-assignment4501-Node: conditional block5125-Ref: #conditional-block5279-Node: include6175-Ref: #include6305-Node: newest-first6536-Ref: #newest-first6650-Node: CSV TIPS7061-Ref: #csv-tips7155-Node: CSV ordering7299-Ref: #csv-ordering7417-Node: CSV accounts7598-Ref: #csv-accounts7736-Node: CSV amounts7990-Ref: #csv-amounts8148-Node: CSV balance assertions/assignments9228-Ref: #csv-balance-assertionsassignments9446-Node: Reading multiple CSV files9767-Ref: #reading-multiple-csv-files9967-Node: Valid CSV10241-Ref: #valid-csv10364+File: hledger_csv.info,  Node: Top,  Next: EXAMPLES,  Up: (dir)++hledger_csv(5) hledger 1.16+***************************++hledger can read CSV (comma-separated value, or character-separated+value) files as if they were journal files, automatically converting+each CSV record into a transaction.  (To learn about _writing_ CSV, see+CSV output.)++   We describe each CSV file's format with a corresponding _rules file_.+By default this is named like the CSV file with a '.rules' extension+added.  Eg when reading 'FILE.csv', hledger also looks for+'FILE.csv.rules' in the same directory.  You can specify a different+rules file with the '--rules-file' option.  If a rules file is not+found, hledger will create a sample rules file, which you'll need to+adjust.++   This file contains rules describing the CSV data (header line, fields+layout, date format etc.), and how to construct hledger journal entries+(transactions) from it.  Often there will also be a list of conditional+rules for categorising transactions based on their descriptions.  Here's+an overview of the CSV rules; these are described more fully below,+after the examples:++*'skip'*           skip one or more header lines or matched CSV records+*'fields'*         name CSV fields, assign them to hledger fields+*field             assign a value to one hledger field, with interpolation+assignment*+*'if'*             apply some rules to matched CSV records+*'end'*            skip the remaining CSV records+*'date-format'*    describe the format of CSV dates+*'newest-first'*   disambiguate record order when there's only one date+*'include'*        inline another CSV rules file++   There's also a Convert CSV files tutorial on hledger.org.++* Menu:++* EXAMPLES::+* CSV RULES::+* TIPS::+++File: hledger_csv.info,  Node: EXAMPLES,  Next: CSV RULES,  Prev: Top,  Up: Top++1 EXAMPLES+**********++Here are some sample hledger CSV rules files.  See also the full+collection at:+https://github.com/simonmichael/hledger/tree/master/examples/csv++* Menu:++* Basic::+* Bank of Ireland::+* Amazon::+* Paypal::+++File: hledger_csv.info,  Node: Basic,  Next: Bank of Ireland,  Up: EXAMPLES++1.1 Basic+=========++At minimum, the rules file must identify the date and amount fields, and+often it also specifies the date format and how many header lines there+are.  Here's a simple CSV file and a rules file for it:++Date, Description, Id, Amount+12/11/2019, Foo, 123, 10.23++# basic.csv.rules+skip         1+fields       date, description, _, amount+date-format  %d/%m/%Y++$ hledger print -f basic.csv+2019/11/12 Foo+    expenses:unknown           10.23+    income:unknown            -10.23++   Default account names are chosen, since we didn't set them.+++File: hledger_csv.info,  Node: Bank of Ireland,  Next: Amazon,  Prev: Basic,  Up: EXAMPLES++1.2 Bank of Ireland+===================++Here's a CSV with two amount fields (Debit and Credit), and a balance+field, which we can use to add balance assertions, which is not+necessary but provides extra error checking:++Date,Details,Debit,Credit,Balance+07/12/2012,LODGMENT       529898,,10.0,131.21+07/12/2012,PAYMENT,5,,126++# bankofireland-checking.csv.rules++# skip the header line+skip++# name the csv fields, and assign some of them as journal entry fields+fields  date, description, amount-out, amount-in, balance++# We generate balance assertions by assigning to "balance"+# above, but you may sometimes need to remove these because:+#+# - the CSV balance differs from the true balance, +#   by up to 0.0000000000005 in my experience+#+# - it is sometimes calculated based on non-chronological ordering,+#   eg when multiple transactions clear on the same day++# date is in UK/Ireland format+date-format  %d/%m/%Y++# set the currency+currency  EUR++# set the base account for all txns+account1  assets:bank:boi:checking++$ hledger -f bankofireland-checking.csv print+2012/12/07 LODGMENT       529898+    assets:bank:boi:checking         EUR10.0 = EUR131.2+    income:unknown                  EUR-10.0++2012/12/07 PAYMENT+    assets:bank:boi:checking         EUR-5.0 = EUR126.0+    expenses:unknown                  EUR5.0++   The balance assertions don't raise an error above, because we're+reading directly from CSV, but they will be checked if these entries are+imported into a journal file.+++File: hledger_csv.info,  Node: Amazon,  Next: Paypal,  Prev: Bank of Ireland,  Up: EXAMPLES++1.3 Amazon+==========++Here we convert amazon.com order history, and use an if block to+generate a third posting if there's a fee.  (In practice you'd probably+get this data from your bank instead, but it's an example.)++"Date","Type","To/From","Name","Status","Amount","Fees","Transaction ID"+"Jul 29, 2012","Payment","To","Foo.","Completed","$20.00","$0.00","16000000000000DGLNJPI1P9B8DKPVHL"+"Jul 30, 2012","Payment","To","Adapteva, Inc.","Completed","$25.00","$1.00","17LA58JSKRD4HDGLNJPI1P9B8DKPVHL"++# amazon-orders.csv.rules++# skip one header line+skip 1++# name the csv fields, and assign the transaction's date, amount and code.+# Avoided the "status" and "amount" hledger field names to prevent confusion.+fields date, _, toorfrom, name, amzstatus, amzamount, fees, code++# how to parse the date+date-format %b %-d, %Y++# combine two fields to make the description+description %toorfrom %name++# save the status as a tag+comment     status:%amzstatus++# set the base account for all transactions+account1    assets:amazon+# leave amount1 blank so it can balance the other(s).+# I'm assuming amzamount excludes the fees, don't remember++# set a generic account2+account2    expenses:misc+amount2     %amzamount+# and maybe refine it further:+#include categorisation.rules++# add a third posting for fees, but only if they are non-zero.+# Commas in the data makes counting fields hard, so count from the right instead.+# (Regex translation: "a field containing a non-zero dollar amount, +# immediately before the 1 right-most fields")+if ,\$[1-9][.0-9]+(,[^,]*){1}$+ account3    expenses:fees+ amount3     %fees++$ hledger -f amazon-orders.csv print+2012/07/29 (16000000000000DGLNJPI1P9B8DKPVHL) To Foo.  ; status:Completed+    assets:amazon+    expenses:misc          $20.00++2012/07/30 (17LA58JSKRD4HDGLNJPI1P9B8DKPVHL) To Adapteva, Inc.  ; status:Completed+    assets:amazon+    expenses:misc          $25.00+    expenses:fees           $1.00+++File: hledger_csv.info,  Node: Paypal,  Prev: Amazon,  Up: EXAMPLES++1.4 Paypal+==========++Here's a real-world rules file for (customised) Paypal CSV, with some+Paypal-specific rules, and a second rules file included:++"Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Item Title","Item ID","Reference Txn ID","Receipt ID","Balance","Note"+"10/01/2019","03:46:20","PDT","Calm Radio","Subscription Payment","Completed","USD","-6.99","0.00","-6.99","simon@joyful.com","memberships@calmradio.com","60P57143A8206782E","MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month","","I-R8YLY094FJYR","","-6.99",""+"10/01/2019","03:46:20","PDT","","Bank Deposit to PP Account ","Pending","USD","6.99","0.00","6.99","","simon@joyful.com","0TU1544T080463733","","","60P57143A8206782E","","0.00",""+"10/01/2019","08:57:01","PDT","Patreon","PreApproved Payment Bill User Payment","Completed","USD","-7.00","0.00","-7.00","simon@joyful.com","support@patreon.com","2722394R5F586712G","Patreon* Membership","","B-0PG93074E7M86381M","","-7.00",""+"10/01/2019","08:57:01","PDT","","Bank Deposit to PP Account ","Pending","USD","7.00","0.00","7.00","","simon@joyful.com","71854087RG994194F","Patreon* Membership","","2722394R5F586712G","","0.00",""+"10/19/2019","03:02:12","PDT","Wikimedia Foundation, Inc.","Subscription Payment","Completed","USD","-2.00","0.00","-2.00","simon@joyful.com","tle@wikimedia.org","K9U43044RY432050M","Monthly donation to the Wikimedia Foundation","","I-R5C3YUS3285L","","-2.00",""+"10/19/2019","03:02:12","PDT","","Bank Deposit to PP Account ","Pending","USD","2.00","0.00","2.00","","simon@joyful.com","3XJ107139A851061F","","","K9U43044RY432050M","","0.00",""+"10/22/2019","05:07:06","PDT","Noble Benefactor","Subscription Payment","Completed","USD","10.00","-0.59","9.41","noble@bene.fac.tor","simon@joyful.com","6L8L1662YP1334033","Joyful Systems","","I-KC9VBGY2GWDB","","9.41",""++# paypal-custom.csv.rules++# Tips:+# Export from Activity -> Statements -> Custom -> Activity download+# Suggested transaction type: "Balance affecting"+# Paypal's default fields in 2018 were:+# "Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Shipping Address","Address Status","Item Title","Item ID","Shipping and Handling Amount","Insurance Amount","Sales Tax","Option 1 Name","Option 1 Value","Option 2 Name","Option 2 Value","Reference Txn ID","Invoice Number","Custom Number","Quantity","Receipt ID","Balance","Address Line 1","Address Line 2/District/Neighborhood","Town/City","State/Province/Region/County/Territory/Prefecture/Republic","Zip/Postal Code","Country","Contact Phone Number","Subject","Note","Country Code","Balance Impact"+# This rules file assumes the following more detailed fields, configured in "Customize report fields":+# "Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Item Title","Item ID","Reference Txn ID","Receipt ID","Balance","Note"++fields date, time, timezone, description_, type, status_, currency, grossamount, feeamount, netamount, fromemail, toemail, code, itemtitle, itemid, referencetxnid, receiptid, balance, note++skip  1++date-format  %-m/%-d/%Y++# ignore some paypal events+if+In Progress+Temporary Hold+Update to + skip++# add more fields to the description+description %description_ %itemtitle ++# save some other fields as tags+comment  itemid:%itemid, fromemail:%fromemail, toemail:%toemail, time:%time, type:%type, status:%status_++# convert to short currency symbols+# Note: in conditional block regexps, the line of csv being matched is+# a synthetic one: the unquoted field values, with commas between them.+if ,USD,+ currency $+if ,EUR,+ currency E+if ,GBP,+ currency P++# generate postings++# the first posting will be the money leaving/entering my paypal account+# (negative means leaving my account, in all amount fields)+account1 assets:online:paypal+amount1  %netamount++# the second posting will be money sent to/received from other party+# (account2 is set below)+amount2  -%grossamount++# if there's a fee (9th field), add a third posting for the money taken by paypal.+# TODO: This regexp fails when fields contain a comma (generates a third posting with zero amount)+if ^([^,]+,){8}[^0]+ account3 expenses:banking:paypal+ amount3  -%feeamount+ comment3 business:++# choose an account for the second posting++# override the default account names:+# if amount (8th field) is positive, it's income (a debit)+if ^([^,]+,){7}[0-9]+ account2 income:unknown+# if negative, it's an expense (a credit)+if ^([^,]+,){7}-+ account2 expenses:unknown++# apply common rules for setting account2 & other tweaks+include common.rules++# apply some overrides specific to this csv++# Transfers from/to bank. These are usually marked Pending, +# which can be disregarded in this case.+if +Bank Account+Bank Deposit to PP Account+ description %type for %referencetxnid %itemtitle+ account2 assets:bank:wf:pchecking+ account1 assets:online:paypal++# Currency conversions+if Currency Conversion+ account2 equity:currency conversion++# common.rules++if+darcs+noble benefactor+ account2 revenues:foss donations:darcshub+ comment2 business:++if+Calm Radio+ account2 expenses:online:apps++if+electronic frontier foundation+Patreon+wikimedia+Advent of Code+ account2 expenses:dues++if Google+ account2 expenses:online:apps+ description google | music++$ hledger -f paypal-custom.csv  print+2019/10/01 (60P57143A8206782E) Calm Radio MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month  ; itemid:, fromemail:simon@joyful.com, toemail:memberships@calmradio.com, time:03:46:20, type:Subscription Payment, status:Completed+    assets:online:paypal          $-6.99 = $-6.99+    expenses:online:apps           $6.99++2019/10/01 (0TU1544T080463733) Bank Deposit to PP Account for 60P57143A8206782E  ; itemid:, fromemail:, toemail:simon@joyful.com, time:03:46:20, type:Bank Deposit to PP Account, status:Pending+    assets:online:paypal               $6.99 = $0.00+    assets:bank:wf:pchecking          $-6.99++2019/10/01 (2722394R5F586712G) Patreon Patreon* Membership  ; itemid:, fromemail:simon@joyful.com, toemail:support@patreon.com, time:08:57:01, type:PreApproved Payment Bill User Payment, status:Completed+    assets:online:paypal          $-7.00 = $-7.00+    expenses:dues                  $7.00++2019/10/01 (71854087RG994194F) Bank Deposit to PP Account for 2722394R5F586712G Patreon* Membership  ; itemid:, fromemail:, toemail:simon@joyful.com, time:08:57:01, type:Bank Deposit to PP Account, status:Pending+    assets:online:paypal               $7.00 = $0.00+    assets:bank:wf:pchecking          $-7.00++2019/10/19 (K9U43044RY432050M) Wikimedia Foundation, Inc. Monthly donation to the Wikimedia Foundation  ; itemid:, fromemail:simon@joyful.com, toemail:tle@wikimedia.org, time:03:02:12, type:Subscription Payment, status:Completed+    assets:online:paypal             $-2.00 = $-2.00+    expenses:dues                     $2.00+    expenses:banking:paypal      ; business:++2019/10/19 (3XJ107139A851061F) Bank Deposit to PP Account for K9U43044RY432050M  ; itemid:, fromemail:, toemail:simon@joyful.com, time:03:02:12, type:Bank Deposit to PP Account, status:Pending+    assets:online:paypal               $2.00 = $0.00+    assets:bank:wf:pchecking          $-2.00++2019/10/22 (6L8L1662YP1334033) Noble Benefactor Joyful Systems  ; itemid:, fromemail:noble@bene.fac.tor, toemail:simon@joyful.com, time:05:07:06, type:Subscription Payment, status:Completed+    assets:online:paypal                       $9.41 = $9.41+    revenues:foss donations:darcshub         $-10.00  ; business:+    expenses:banking:paypal                    $0.59  ; business:+++File: hledger_csv.info,  Node: CSV RULES,  Next: TIPS,  Prev: EXAMPLES,  Up: Top++2 CSV RULES+***********++The following kinds of rule can appear in the rules file, in any order.+Blank lines and lines beginning with '#' or ';' are ignored.++* Menu:++* skip::+* fields::+* field assignment::+* if::+* end::+* date-format::+* newest-first::+* include::+++File: hledger_csv.info,  Node: skip,  Next: fields,  Up: CSV RULES++2.1 'skip'+==========++skip N++   The word "skip" followed by a number (or no number, meaning 1) tells+hledger to ignore this many non-empty lines preceding the CSV data.+(Empty/blank lines are skipped automatically.)  You'll need this+whenever your CSV data contains header lines.++   It also has a second purpose: it can be used inside if blocks to+ignore certain CSV records (described below).+++File: hledger_csv.info,  Node: fields,  Next: field assignment,  Prev: skip,  Up: CSV RULES++2.2 'fields'+============++fields FIELDNAME1, FIELDNAME2, ...++   A fields list (the word "fields" followed by comma-separated field+names) is the quick way to assign CSV field values to hledger fields.+It does two things:++  1. it names the CSV fields.  This is optional, but can be convenient+     later for interpolating them.++  2. when you use a standard hledger field name, it assigns the CSV+     value to that part of the hledger transaction.++   Here's an example that says "use the 1st, 2nd and 4th fields as the+transaction's date, description and amount; name the last two fields for+later reference; and ignore the others":++fields date, description, , amount, , , somefield, anotherfield++   Field names may not contain whitespace.  Fields you don't care about+can be left unnamed.  Currently there must be least two items (there+must be at least one comma).++   Here are the standard hledger field/pseudo-field names.  For more+about the transaction parts they refer to, see the manual for hledger's+journal format.++* Menu:++* Transaction field names::+* Posting field names::+++File: hledger_csv.info,  Node: Transaction field names,  Next: Posting field names,  Up: fields++2.2.1 Transaction field names+-----------------------------++'date', 'date2', 'status', 'code', 'description', 'comment' can be used+to form the transaction's first line.+++File: hledger_csv.info,  Node: Posting field names,  Prev: Transaction field names,  Up: fields++2.2.2 Posting field names+-------------------------++'accountN', where N is 1 to 9, generates a posting, with that account+name.  Most often there are two postings, so you'll want to set+'account1' and 'account2'.  If a posting's account name is left unset+but its amount is set, a default account name will be chosen (like+expenses:unknown or income:unknown).++   'amountN' sets posting N's amount.  Or, 'amount' with no N sets+posting 1's.  If the CSV has debits and credits in separate fields, use+'amountN-in' and 'amountN-out' instead.  Or 'amount-in' and 'amount-out'+with no N for posting 1.++   For convenience and backwards compatibility, if you set the amount of+posting 1 only, a second posting with the negative amount will be+generated automatically.  (Unless the account name is parenthesised+indicating an unbalanced posting.)++   If the CSV has the currency symbol in a separate field, you can use+'currencyN' to prepend it to posting N's amount.  'currency' with no N+affects ALL postings.++   'balanceN' sets a balance assertion amount (or if the posting amount+is left empty, a balance assignment).++   Finally, 'commentN' sets a comment on the Nth posting.  Comments can+also contain tags, as usual.++   See TIPS below for more about setting amounts and currency.+++File: hledger_csv.info,  Node: field assignment,  Next: if,  Prev: fields,  Up: CSV RULES++2.3 field assignment+====================++HLEDGERFIELDNAME FIELDVALUE++   Instead of or in addition to a fields list, you can use a "field+assignment" rule to set the value of a single hledger field, by writing+its name (any of the standard hledger field names above) followed by a+text value.  The value may contain interpolated CSV fields, referenced+by their 1-based position in the CSV record ('%N'), or by the name they+were given in the fields list ('%CSVFIELDNAME').  Some examples:++# set the amount to the 4th CSV field, with " USD" appended+amount %4 USD++# combine three fields to make a comment, containing note: and date: tags+comment note: %somefield - %anotherfield, date: %1++   Interpolation strips outer whitespace (so a CSV value like '" 1 "'+becomes '1' when interpolated) (#1051).  See TIPS below for more about+referencing other fields.+++File: hledger_csv.info,  Node: if,  Next: end,  Prev: field assignment,  Up: CSV RULES++2.4 'if'+========++if PATTERN+ RULE++if+PATTERN+PATTERN+PATTERN+ RULE+ RULE++   Conditional blocks ("if blocks") are a block of rules that are+applied only to CSV records which match certain patterns.  They are+often used for customising account names based on transaction+descriptions.++   A single pattern can be written on the same line as the "if"; or+multiple patterns can be written on the following lines, non-indented.+Multiple patterns are OR'd (any one of them can match).  Patterns are+case-insensitive regular expressions which try to match anywhere within+the whole CSV record (POSIX extended regular expressions with some+additions, see https://hledger.org/hledger.html#regular-expressions).+Note the CSV record they see is close to, but not identical to, the one+in the CSV file; enclosing double quotes will be removed, and the+separator character is always comma.++   It's not yet easy to match within a specific field.  If the data does+not contain commas, you can hack it with a regular expression like:++# match "foo" in the fourth field+if ^([^,]*,){3}foo++   After the patterns there should be one or more rules to apply, all+indented by at least one space.  Three kinds of rule are allowed in+conditional blocks:++   * field assignments (to set a hledger field)+   * skip (to skip the matched CSV record)+   * end (to skip all remaining CSV records).++   Examples:++# if the CSV record contains "groceries", set account2 to "expenses:groceries"+if groceries+ account2 expenses:groceries++# if the CSV record contains any of these patterns, set account2 and comment as shown+if+monthly service fee+atm transaction fee+banking thru software+ account2 expenses:business:banking+ comment  XXX deductible ? check it+++File: hledger_csv.info,  Node: end,  Next: date-format,  Prev: if,  Up: CSV RULES++2.5 'end'+=========++This rule can be used inside if blocks (only), to make hledger stop+reading this CSV file and move on to the next input file, or to command+execution.  Eg:++# ignore everything following the first empty record+if ,,,,+ end+++File: hledger_csv.info,  Node: date-format,  Next: newest-first,  Prev: end,  Up: CSV RULES++2.6 'date-format'+=================++date-format DATEFMT++   This is a helper for the 'date' (and 'date2') fields.  If your CSV+dates are not formatted like 'YYYY-MM-DD', 'YYYY/MM/DD' or 'YYYY.MM.DD',+you'll need to add a date-format rule describing them with a strptime+date parsing pattern, which must parse the CSV date value completely.+Some examples:++# MM/DD/YY+date-format %m/%d/%y++# D/M/YYYY+# The - makes leading zeros optional.+date-format %-d/%-m/%Y++# YYYY-Mmm-DD+date-format %Y-%h-%d++# M/D/YYYY HH:MM AM some other junk+# Note the time and junk must be fully parsed, though only the date is used.+date-format %-m/%-d/%Y %l:%M %p some other junk++   For the supported strptime syntax, see:+https://hackage.haskell.org/package/time/docs/Data-Time-Format.html#v:formatTime+++File: hledger_csv.info,  Node: newest-first,  Next: include,  Prev: date-format,  Up: CSV RULES++2.7 'newest-first'+==================++hledger always sorts the generated transactions by date.  Transactions+on the same date should appear in the same order as their CSV records,+as hledger can usually auto-detect whether the CSV's normal order is+oldest first or newest first.  But if all of the following are true:++   * the CSV might sometimes contain just one day of data (all records+     having the same date)+   * the CSV records are normally in reverse chronological order (newest+     at the top)+   * and you care about preserving the order of same-day transactions++   then, you should add the 'newest-first' rule as a hint.  Eg:++# tell hledger explicitly that the CSV is normally newest first+newest-first+++File: hledger_csv.info,  Node: include,  Prev: newest-first,  Up: CSV RULES++2.8 'include'+=============++include RULESFILE++   This includes the contents of another CSV rules file at this point.+'RULESFILE' is an absolute file path or a path relative to the current+file's directory.  This can be useful for sharing common rules between+several rules files, eg:++# someaccount.csv.rules++## someaccount-specific rules+fields   date,description,amount+account1 assets:someaccount+account2 expenses:misc++## common rules+include categorisation.rules+++File: hledger_csv.info,  Node: TIPS,  Prev: CSV RULES,  Up: Top++3 TIPS+******++* Menu:++* Valid CSV::+* Other separator characters::+* Reading multiple CSV files::+* Valid transactions::+* Deduplicating importing::+* Setting amounts::+* Setting currency/commodity::+* Referencing other fields::+* How CSV rules are evaluated::+++File: hledger_csv.info,  Node: Valid CSV,  Next: Other separator characters,  Up: TIPS++3.1 Valid CSV+=============++hledger accepts CSV conforming to RFC 4180.  When CSV values are+enclosed in quotes, note:++   * they must be double quotes (not single quotes)+   * spaces outside the quotes are not allowed+++File: hledger_csv.info,  Node: Other separator characters,  Next: Reading multiple CSV files,  Prev: Valid CSV,  Up: TIPS++3.2 Other separator characters+==============================++With the '--separator 'CHAR'' option (experimental), hledger will expect+the separator to be CHAR instead of a comma.  Ie it will read other+"Character Separated Values" formats, such as TSV (Tab Separated+Values).  Note: on the command line, use a real tab character in quotes,+not++$ hledger -f foo.tsv --separator '  ' print+++File: hledger_csv.info,  Node: Reading multiple CSV files,  Next: Valid transactions,  Prev: Other separator characters,  Up: TIPS++3.3 Reading multiple CSV files+==============================++If you use multiple '-f' options to read multiple CSV files at once,+hledger will look for a correspondingly-named rules file for each CSV+file.  But if you use the '--rules-file' option, that rules file will be+used for all the CSV files.+++File: hledger_csv.info,  Node: Valid transactions,  Next: Deduplicating importing,  Prev: Reading multiple CSV files,  Up: TIPS++3.4 Valid transactions+======================++After reading a CSV file, hledger post-processes and validates the+generated journal entries as it would for a journal file - balancing+them, applying balance assignments, and canonicalising amount styles.+Any errors at this stage will be reported in the usual way, displaying+the problem entry.++   There is one exception: balance assertions, if you have generated+them, will not be checked, since normally these will work only when the+CSV data is part of the main journal.  If you do need to check balance+assertions generated from CSV right away, pipe into another hledger:++$ hledger -f file.csv print | hledger -f- print+++File: hledger_csv.info,  Node: Deduplicating importing,  Next: Setting amounts,  Prev: Valid transactions,  Up: TIPS++3.5 Deduplicating, importing+============================++When you download a CSV file periodically, eg to get your latest bank+transactions, the new file may overlap with the old one, containing some+of the same records.++   The import command will (a) detect the new transactions, and (b)+append just those transactions to your main journal.  It is idempotent,+so you don't have to remember how many times you ran it or with which+version of the CSV. (It keeps state in a hidden '.latest.FILE.csv'+file.)  This is the easiest way to import CSV data.  Eg:++# download the latest CSV files, then run this command.+# Note, no -f flags needed here.+$ hledger import *.csv [--dry]++   This method works for most CSV files.  (Where records have a stable+chronological order, and new records appear only at the new end.)++   A number of other tools and workflows, hledger-specific and+otherwise, exist for converting, deduplicating, classifying and managing+CSV data.  See:++   * https://hledger.org -> sidebar -> real world setups+   * https://plaintextaccounting.org -> data import/conversion+++File: hledger_csv.info,  Node: Setting amounts,  Next: Setting currency/commodity,  Prev: Deduplicating importing,  Up: TIPS++3.6 Setting amounts+===================++A posting amount can be set in one of these ways:++   * by assigning (with a fields list or field assigment) to 'amountN'+     (posting N's amount) or 'amount' (posting 1's amount)++   * by assigning to 'amountN-in' and 'amountN-out' (or 'amount-in' and+     'amount-out').  For each CSV record, whichever of these has a+     non-zero value will be used, with appropriate sign.  If both+     contain a non-zero value, this may not work.++   * by assigning to 'balanceN' (or 'balance') instead of the above,+     setting the amount indirectly via a balance assignment.  If you do+     this the default account name may be wrong, so you should set that+     explicitly.++   There is some special handling for an amount's sign:++   * If an amount value is parenthesised, it will be de-parenthesised+     and sign-flipped.+   * If an amount value begins with a double minus sign, those cancel+     out and are removed.+   * If an amount value begins with a plus sign, that will be removed+++File: hledger_csv.info,  Node: Setting currency/commodity,  Next: Referencing other fields,  Prev: Setting amounts,  Up: TIPS++3.7 Setting currency/commodity+==============================++If the currency/commodity symbol is included in the CSV's amount+field(s), you don't have to do anything special.++   If the currency is provided as a separate CSV field, you can either:++   * assign that to 'currency', which adds it to all posting amounts.+     The symbol will prepended to the amount quantity (on the left+     side).  If you write a trailing space after the symbol, there will+     be a space between symbol and amount (an exception to the usual+     whitespace stripping).++   * or assign it to 'currencyN' which adds it to posting N's amount+     only.++   * or for more control, construct the amount from symbol and quantity+     using field assignment, eg:++     fields date,description,currency,quantity+     # add currency symbol on the right:+     amount %quantity %currency+++File: hledger_csv.info,  Node: Referencing other fields,  Next: How CSV rules are evaluated,  Prev: Setting currency/commodity,  Up: TIPS++3.8 Referencing other fields+============================++In field assignments, you can interpolate only CSV fields, not hledger+fields.  In the example below, there's both a CSV field and a hledger+field named amount1, but %amount1 always means the CSV field, not the+hledger field:++# Name the third CSV field "amount1"+fields date,description,amount1++# Set hledger's amount1 to the CSV amount1 field followed by USD+amount1 %amount1 USD++# Set comment to the CSV amount1 (not the amount1 assigned above)+comment %amount1++   Here, since there's no CSV amount1 field, %amount1 will produce a+literal "amount1":++fields date,description,csvamount+amount1 %csvamount USD+# Can't interpolate amount1 here+comment %amount1++   When there are multiple field assignments to the same hledger field,+only the last one takes effect.  Here, comment's value will be be B, or+C if "something" is matched, but never A:++comment A+comment B+if something+ comment C+++File: hledger_csv.info,  Node: How CSV rules are evaluated,  Prev: Referencing other fields,  Up: TIPS++3.9 How CSV rules are evaluated+===============================++Here's how to think of CSV rules being evaluated (if you really need+to).  First,++   * 'include' - all includes are inlined, from top to bottom, depth+     first.  (At each include point the file is inlined and scanned for+     further includes, recursively, before proceeding.)++   Then "global" rules are evaluated, top to bottom.  If a rule is+repeated, the last one wins:++   * 'skip' (at top level)+   * 'date-format'+   * 'newest-first'+   * 'fields' - names the CSV fields, optionally sets up initial+     assignments to hledger fields++   Then for each CSV record in turn:++   * test all 'if' blocks.  If any of them contain a 'end' rule, skip+     all remaining CSV records.  Otherwise if any of them contain a+     'skip' rule, skip that many CSV records.  If there are multiple+     matched 'skip' rules, the first one wins.+   * collect all field assignments at top level and in matched 'if'+     blocks.  When there are multiple assignments for a field, keep only+     the last one.+   * compute a value for each hledger field - either the one that was+     assigned to it (and interpolate the %CSVFIELDNAME references), or a+     default+   * generate a synthetic hledger transaction from these values.++   This is all part of the CSV reader, one of several readers hledger+can use to parse input files.  When all files have been read+successfully, the transactions are passed as input to whichever hledger+command the user specified.+++Tag Table:+Node: Top72+Node: EXAMPLES1829+Ref: #examples1935+Node: Basic2143+Ref: #basic2243+Node: Bank of Ireland2785+Ref: #bank-of-ireland2920+Node: Amazon4383+Ref: #amazon4501+Node: Paypal6434+Ref: #paypal6528+Node: CSV RULES14411+Ref: #csv-rules14520+Node: skip14765+Ref: #skip14858+Node: fields15233+Ref: #fields15355+Node: Transaction field names16422+Ref: #transaction-field-names16582+Node: Posting field names16693+Ref: #posting-field-names16845+Node: field assignment18077+Ref: #field-assignment18213+Node: if19031+Ref: #if19140+Node: end20856+Ref: #end20962+Node: date-format21186+Ref: #date-format21318+Node: newest-first22067+Ref: #newest-first22205+Node: include22888+Ref: #include22996+Node: TIPS23440+Ref: #tips23522+Node: Valid CSV23771+Ref: #valid-csv23890+Node: Other separator characters24082+Ref: #other-separator-characters24270+Node: Reading multiple CSV files24599+Ref: #reading-multiple-csv-files24796+Node: Valid transactions25037+Ref: #valid-transactions25215+Node: Deduplicating importing25843+Ref: #deduplicating-importing26022+Node: Setting amounts27055+Ref: #setting-amounts27224+Node: Setting currency/commodity28210+Ref: #setting-currencycommodity28402+Node: Referencing other fields29205+Ref: #referencing-other-fields29405+Node: How CSV rules are evaluated30302+Ref: #how-csv-rules-are-evaluated30473  End Tag Table