diff --git a/CHANGES.md b/CHANGES.md --- a/CHANGES.md +++ b/CHANGES.md @@ -1,6 +1,29 @@ Internal/api/developer-ish changes in the hledger-lib (and hledger) packages. For user-visible changes, see the hledger package changelog. +# 1.16 2019-12-01 + +- drop support for GHC 7.10, due to MonadFail hassles in JournalReader.hs + +- add support for GHC 8.8, base-compat 0.11 (#1090) + + We are now using the new fail from the MonadFail class, which we + always import qualified as Fail.fail, from base-compat-batteries + Control.Monad.Fail.Compat to work with old GHC versions. If old fail + is needed (shouldn't be) it should be imported qualified as + Prelude.Fail, using imports such as: + + import Prelude hiding (fail) + import qualified Prelude (fail) + import Control.Monad.State.Strict hiding (fail) + import "base-compat-batteries" Prelude.Compat hiding (fail) + import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail + +- hledger and hledger-lib unit tests have been ported to tasty. + +- The doctest suite has been disabled for now since it doesn't run + well with cabal (#1139) + # 1.15.2 2019-09-05 Changes: diff --git a/Hledger/Data/AccountName.hs b/Hledger/Data/AccountName.hs --- a/Hledger/Data/AccountName.hs +++ b/Hledger/Data/AccountName.hs @@ -227,27 +227,23 @@ --isAccountRegex s = take 1 s == "^" && take 5 (reverse s) == ")$|:(" tests_AccountName = tests "AccountName" [ - tests "accountNameTreeFrom" [ - accountNameTreeFrom ["a"] `is` Node "root" [Node "a" []] - ,accountNameTreeFrom ["a","b"] `is` Node "root" [Node "a" [], Node "b" []] - ,accountNameTreeFrom ["a","a:b"] `is` Node "root" [Node "a" [Node "a:b" []]] - ,accountNameTreeFrom ["a:b:c"] `is` Node "root" [Node "a" [Node "a:b" [Node "a:b:c" []]]] - ] - ,tests "expandAccountNames" [ - expandAccountNames ["assets:cash","assets:checking","expenses:vacation"] `is` + test "accountNameTreeFrom" $ do + accountNameTreeFrom ["a"] @?= Node "root" [Node "a" []] + accountNameTreeFrom ["a","b"] @?= Node "root" [Node "a" [], Node "b" []] + accountNameTreeFrom ["a","a:b"] @?= Node "root" [Node "a" [Node "a:b" []]] + accountNameTreeFrom ["a:b:c"] @?= Node "root" [Node "a" [Node "a:b" [Node "a:b:c" []]]] + ,test "expandAccountNames" $ do + expandAccountNames ["assets:cash","assets:checking","expenses:vacation"] @?= ["assets","assets:cash","assets:checking","expenses","expenses:vacation"] - ] - ,tests "isAccountNamePrefixOf" [ - "assets" `isAccountNamePrefixOf` "assets" `is` False - ,"assets" `isAccountNamePrefixOf` "assets:bank" `is` True - ,"assets" `isAccountNamePrefixOf` "assets:bank:checking" `is` True - ,"my assets" `isAccountNamePrefixOf` "assets:bank" `is` False - ] - ,tests "isSubAccountNameOf" [ - "assets" `isSubAccountNameOf` "assets" `is` False - ,"assets:bank" `isSubAccountNameOf` "assets" `is` True - ,"assets:bank:checking" `isSubAccountNameOf` "assets" `is` False - ,"assets:bank" `isSubAccountNameOf` "my assets" `is` False - ] + ,test "isAccountNamePrefixOf" $ do + "assets" `isAccountNamePrefixOf` "assets" @?= False + "assets" `isAccountNamePrefixOf` "assets:bank" @?= True + "assets" `isAccountNamePrefixOf` "assets:bank:checking" @?= True + "my assets" `isAccountNamePrefixOf` "assets:bank" @?= False + ,test "isSubAccountNameOf" $ do + "assets" `isSubAccountNameOf` "assets" @?= False + "assets:bank" `isSubAccountNameOf` "assets" @?= True + "assets:bank:checking" `isSubAccountNameOf` "assets" @?= False + "assets:bank" `isSubAccountNameOf` "my assets" @?= False ] diff --git a/Hledger/Data/Amount.hs b/Hledger/Data/Amount.hs --- a/Hledger/Data/Amount.hs +++ b/Hledger/Data/Amount.hs @@ -51,6 +51,7 @@ usd, eur, gbp, + per, hrs, at, (@@), @@ -66,6 +67,7 @@ -- ** rendering amountstyle, styleAmount, + styleAmountExceptPrecision, showAmount, cshowAmount, showAmountWithZeroCommodity, @@ -180,6 +182,7 @@ usd n = amount{acommodity="$", aquantity=roundTo 2 n, astyle=amountstyle{asprecision=2}} eur n = amount{acommodity="€", aquantity=roundTo 2 n, astyle=amountstyle{asprecision=2}} gbp n = amount{acommodity="£", aquantity=roundTo 2 n, astyle=amountstyle{asprecision=2}} +per n = amount{acommodity="%", aquantity=n, astyle=amountstyle{asprecision=1, ascommodityside=R, ascommodityspaced=True}} amt `at` priceamt = amt{aprice=Just $ UnitPrice priceamt} amt @@ priceamt = amt{aprice=Just $ TotalPrice priceamt} @@ -274,17 +277,17 @@ showAmountWithPrecision :: Int -> Amount -> String showAmountWithPrecision p = showAmount . setAmountPrecision p --- | Set an amount's display precision. -setAmountPrecision :: Int -> Amount -> Amount -setAmountPrecision p a@Amount{astyle=s} = a{astyle=s{asprecision=p}} - -- | Set an amount's display precision, flipped. withPrecision :: Amount -> Int -> Amount withPrecision = flip setAmountPrecision --- | Increase an amount's display precision, if necessary, enough so --- that it will be shown exactly, with all significant decimal places --- (excluding trailing zeros). +-- | Set an amount's display precision. +setAmountPrecision :: Int -> Amount -> Amount +setAmountPrecision p a@Amount{astyle=s} = a{astyle=s{asprecision=p}} + +-- | Increase an amount's display precision, if needed, to enough +-- decimal places to show it exactly (showing all significant decimal +-- digits, excluding trailing zeros). setFullPrecision :: Amount -> Amount setFullPrecision a = setAmountPrecision p a where @@ -292,16 +295,18 @@ displayprecision = asprecision $ astyle a normalprecision = fromIntegral $ decimalPlaces $ normalizeDecimal $ aquantity a --- | Set an amount's display precision to just enough so that it will --- be shown exactly, with all significant decimal places. +-- | Set an amount's display precision to just enough decimal places +-- to show it exactly (possibly less than the number specified by +-- the amount's display style). setNaturalPrecision :: Amount -> Amount setNaturalPrecision a = setAmountPrecision normalprecision a where normalprecision = fromIntegral $ decimalPlaces $ normalizeDecimal $ aquantity a --- | Set an amount's display precision to just enough so that all --- significant decimal digits will be shown, but not more than the --- given maximum number of decimal digits. +-- | Set an amount's display precision to just enough decimal places +-- to show it exactly (possibly less than the number specified by the +-- amount's display style), but not more than the given maximum number +-- of decimal digits. setNaturalPrecisionUpTo :: Int -> Amount -> Amount setNaturalPrecisionUpTo n a = setAmountPrecision (min n normalprecision) a where @@ -368,6 +373,13 @@ Just s -> a{astyle=s} Nothing -> a +-- | Like styleAmount, but keep the number of decimal places unchanged. +styleAmountExceptPrecision :: M.Map CommoditySymbol AmountStyle -> Amount -> Amount +styleAmountExceptPrecision styles a@Amount{astyle=AmountStyle{asprecision=origp}} = + case M.lookup (acommodity a) styles of + Just s -> a{astyle=s{asprecision=origp}} + Nothing -> a + -- | Get the string representation of an amount, based on its -- commodity's display settings. String representations equivalent to -- zero are converted to just \"0\". The special "missing" amount is @@ -412,9 +424,9 @@ | p == maxprecision = chopdotzero $ show q | otherwise = show $ roundTo (fromIntegral p) q --- | Replace a number string's decimal point with the specified character, --- and add the specified digit group separators. The last digit group will --- be repeated as needed. +-- | Replace a number string's decimal mark with the specified +-- character, and add the specified digit group marks. The last digit +-- group will be repeated as needed. punctuatenumber :: Char -> Maybe DigitGroupStyle -> String -> String punctuatenumber dec mgrps s = sign ++ reverse (applyDigitGroupStyle mgrps (reverse int)) ++ frac'' where @@ -725,99 +737,88 @@ tests_Amount = tests "Amount" [ tests "Amount" [ - tests "costOfAmount" [ - costOfAmount (eur 1) `is` eur 1 - ,costOfAmount (eur 2){aprice=Just $ UnitPrice $ usd 2} `is` usd 4 - ,costOfAmount (eur 1){aprice=Just $ TotalPrice $ usd 2} `is` usd 2 - ,costOfAmount (eur (-1)){aprice=Just $ TotalPrice $ usd 2} `is` usd (-2) - ] + test "costOfAmount" $ do + costOfAmount (eur 1) @?= eur 1 + costOfAmount (eur 2){aprice=Just $ UnitPrice $ usd 2} @?= usd 4 + costOfAmount (eur 1){aprice=Just $ TotalPrice $ usd 2} @?= usd 2 + costOfAmount (eur (-1)){aprice=Just $ TotalPrice $ usd 2} @?= usd (-2) - ,tests "isZeroAmount" [ - expect $ isZeroAmount amount - ,expect $ isZeroAmount $ usd 0 - ] + ,test "isZeroAmount" $ do + assertBool "" $ isZeroAmount amount + assertBool "" $ isZeroAmount $ usd 0 - ,tests "negating amounts" [ - negate (usd 1) `is` (usd 1){aquantity= -1} - ,let b = (usd 1){aprice=Just $ UnitPrice $ eur 2} in negate b `is` b{aquantity= -1} - ] + ,test "negating amounts" $ do + negate (usd 1) @?= (usd 1){aquantity= -1} + let b = (usd 1){aprice=Just $ UnitPrice $ eur 2} in negate b @?= b{aquantity= -1} - ,tests "adding amounts without prices" [ - (usd 1.23 + usd (-1.23)) `is` usd 0 - ,(usd 1.23 + usd (-1.23)) `is` usd 0 - ,(usd (-1.23) + usd (-1.23)) `is` usd (-2.46) - ,sum [usd 1.23,usd (-1.23),usd (-1.23),-(usd (-1.23))] `is` usd 0 - -- highest precision is preserved - ,asprecision (astyle $ sum [usd 1 `withPrecision` 1, usd 1 `withPrecision` 3]) `is` 3 - ,asprecision (astyle $ sum [usd 1 `withPrecision` 3, usd 1 `withPrecision` 1]) `is` 3 - -- adding different commodities assumes conversion rate 1 - ,expect $ isZeroAmount (usd 1.23 - eur 1.23) - ] + ,test "adding amounts without prices" $ do + (usd 1.23 + usd (-1.23)) @?= usd 0 + (usd 1.23 + usd (-1.23)) @?= usd 0 + (usd (-1.23) + usd (-1.23)) @?= usd (-2.46) + sum [usd 1.23,usd (-1.23),usd (-1.23),-(usd (-1.23))] @?= usd 0 + -- highest precision is preserved + asprecision (astyle $ sum [usd 1 `withPrecision` 1, usd 1 `withPrecision` 3]) @?= 3 + asprecision (astyle $ sum [usd 1 `withPrecision` 3, usd 1 `withPrecision` 1]) @?= 3 + -- adding different commodities assumes conversion rate 1 + assertBool "" $ isZeroAmount (usd 1.23 - eur 1.23) - ,tests "showAmount" [ - showAmount (usd 0 + gbp 0) `is` "0" - ] + ,test "showAmount" $ do + showAmount (usd 0 + gbp 0) @?= "0" ] ,tests "MixedAmount" [ - tests "adding mixed amounts to zero, the commodity and amount style are preserved" [ + test "adding mixed amounts to zero, the commodity and amount style are preserved" $ sum (map (Mixed . (:[])) [usd 1.25 ,usd (-1) `withPrecision` 3 ,usd (-0.25) ]) - `is` Mixed [usd 0 `withPrecision` 3] - ] + @?= Mixed [usd 0 `withPrecision` 3] - ,tests "adding mixed amounts with total prices" [ + ,test "adding mixed amounts with total prices" $ do sum (map (Mixed . (:[])) [usd 1 @@ eur 1 ,usd (-2) @@ eur 1 ]) - `is` Mixed [usd 1 @@ eur 1 + @?= Mixed [usd 1 @@ eur 1 ,usd (-2) @@ eur 1 ] - ] - ,tests "showMixedAmount" [ - showMixedAmount (Mixed [usd 1]) `is` "$1.00" - ,showMixedAmount (Mixed [usd 1 `at` eur 2]) `is` "$1.00 @ €2.00" - ,showMixedAmount (Mixed [usd 0]) `is` "0" - ,showMixedAmount (Mixed []) `is` "0" - ,showMixedAmount missingmixedamt `is` "" - ] + ,test "showMixedAmount" $ do + showMixedAmount (Mixed [usd 1]) @?= "$1.00" + showMixedAmount (Mixed [usd 1 `at` eur 2]) @?= "$1.00 @ €2.00" + showMixedAmount (Mixed [usd 0]) @?= "0" + showMixedAmount (Mixed []) @?= "0" + showMixedAmount missingmixedamt @?= "" - ,tests "showMixedAmountWithoutPrice" $ - let a = usd 1 `at` eur 2 in - [ - showMixedAmountWithoutPrice (Mixed [a]) `is` "$1.00" - ,showMixedAmountWithoutPrice (Mixed [a, -a]) `is` "0" - ] + ,test "showMixedAmountWithoutPrice" $ do + let a = usd 1 `at` eur 2 + showMixedAmountWithoutPrice (Mixed [a]) @?= "$1.00" + showMixedAmountWithoutPrice (Mixed [a, -a]) @?= "0" ,tests "normaliseMixedAmount" [ test "a missing amount overrides any other amounts" $ - normaliseMixedAmount (Mixed [usd 1, missingamt]) `is` missingmixedamt + normaliseMixedAmount (Mixed [usd 1, missingamt]) @?= missingmixedamt ,test "unpriced same-commodity amounts are combined" $ - normaliseMixedAmount (Mixed [usd 0, usd 2]) `is` Mixed [usd 2] + normaliseMixedAmount (Mixed [usd 0, usd 2]) @?= Mixed [usd 2] ,test "amounts with same unit price are combined" $ - normaliseMixedAmount (Mixed [usd 1 `at` eur 1, usd 1 `at` eur 1]) `is` Mixed [usd 2 `at` eur 1] + normaliseMixedAmount (Mixed [usd 1 `at` eur 1, usd 1 `at` eur 1]) @?= Mixed [usd 2 `at` eur 1] ,test "amounts with different unit prices are not combined" $ - normaliseMixedAmount (Mixed [usd 1 `at` eur 1, usd 1 `at` eur 2]) `is` Mixed [usd 1 `at` eur 1, usd 1 `at` eur 2] + normaliseMixedAmount (Mixed [usd 1 `at` eur 1, usd 1 `at` eur 2]) @?= Mixed [usd 1 `at` eur 1, usd 1 `at` eur 2] ,test "amounts with total prices are not combined" $ - normaliseMixedAmount (Mixed [usd 1 @@ eur 1, usd 1 @@ eur 1]) `is` Mixed [usd 1 @@ eur 1, usd 1 @@ eur 1] + normaliseMixedAmount (Mixed [usd 1 @@ eur 1, usd 1 @@ eur 1]) @?= Mixed [usd 1 @@ eur 1, usd 1 @@ eur 1] ] - ,tests "normaliseMixedAmountSquashPricesForDisplay" [ - normaliseMixedAmountSquashPricesForDisplay (Mixed []) `is` Mixed [nullamt] - ,expect $ isZeroMixedAmount $ normaliseMixedAmountSquashPricesForDisplay + ,test "normaliseMixedAmountSquashPricesForDisplay" $ do + normaliseMixedAmountSquashPricesForDisplay (Mixed []) @?= Mixed [nullamt] + assertBool "" $ isZeroMixedAmount $ normaliseMixedAmountSquashPricesForDisplay (Mixed [usd 10 ,usd 10 @@ eur 7 ,usd (-10) ,usd (-10) @@ eur 7 ]) - ] ] diff --git a/Hledger/Data/Dates.hs b/Hledger/Data/Dates.hs --- a/Hledger/Data/Dates.hs +++ b/Hledger/Data/Dates.hs @@ -60,7 +60,6 @@ spansSpan, spanIntersect, spansIntersect, - spanIntervalIntersect, spanDefaultsFrom, spanUnion, spansUnion, @@ -76,10 +75,9 @@ ) where -import Prelude () -import "base-compat-batteries" Prelude.Compat +import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail (MonadFail, fail) import Control.Applicative.Permutations -import Control.Monad +import Control.Monad (unless) import "base-compat-batteries" Data.List.Compat import Data.Default import Data.Maybe @@ -262,27 +260,6 @@ b = latest b1 b2 e = earliest e1 e2 --- | Calculate the intersection of two DateSpans, adjusting the start date so --- the interval is preserved. --- --- >>> let intervalIntersect = spanIntervalIntersect (Days 3) --- >>> mkdatespan "2018-01-01" "2018-01-03" `intervalIntersect` mkdatespan "2018-01-01" "2018-01-05" --- DateSpan 2018/01/01-2018/01/02 --- >>> mkdatespan "2018-01-01" "2018-01-05" `intervalIntersect` mkdatespan "2018-01-02" "2018-01-05" --- DateSpan 2018/01/04 --- >>> mkdatespan "2018-01-01" "2018-01-05" `intervalIntersect` mkdatespan "2018-01-03" "2018-01-05" --- DateSpan 2018/01/04 --- >>> mkdatespan "2018-01-01" "2018-01-05" `intervalIntersect` mkdatespan "2018-01-04" "2018-01-05" --- DateSpan 2018/01/04 --- >>> mkdatespan "2018-01-01" "2018-01-05" `intervalIntersect` mkdatespan "2017-12-01" "2018-01-05" --- DateSpan 2018/01/01-2018/01/04 -spanIntervalIntersect :: Interval -> DateSpan -> DateSpan -> DateSpan -spanIntervalIntersect (Days n) (DateSpan (Just b1) e1) sp2@(DateSpan (Just b2) _) = - DateSpan (Just b) e1 `spanIntersect` sp2 - where - b = if b1 < b2 then addDays (diffDays b1 b2 `mod` toInteger n) b2 else b1 -spanIntervalIntersect _ sp1 sp2 = sp1 `spanIntersect` sp2 - -- | Fill any unspecified dates in the first span with the dates from -- the second one. Sort of a one-way spanIntersect. spanDefaultsFrom (DateSpan a1 b1) (DateSpan a2 b2) = DateSpan a b @@ -771,10 +748,10 @@ validMonth s = maybe False (\n -> n>=1 && n<=12) $ readMay s validDay s = maybe False (\n -> n>=1 && n<=31) $ readMay s -failIfInvalidYear, failIfInvalidMonth, failIfInvalidDay :: (Monad m) => String -> m () -failIfInvalidYear s = unless (validYear s) $ fail $ "bad year number: " ++ s -failIfInvalidMonth s = unless (validMonth s) $ fail $ "bad month number: " ++ s -failIfInvalidDay s = unless (validDay s) $ fail $ "bad day number: " ++ s +failIfInvalidYear, failIfInvalidMonth, failIfInvalidDay :: (Fail.MonadFail m) => String -> m () +failIfInvalidYear s = unless (validYear s) $ Fail.fail $ "bad year number: " ++ s +failIfInvalidMonth s = unless (validMonth s) $ Fail.fail $ "bad month number: " ++ s +failIfInvalidDay s = unless (validDay s) $ Fail.fail $ "bad day number: " ++ s yyyymmdd :: TextParser m SmartDate yyyymmdd = do @@ -864,8 +841,8 @@ wday <- T.toLower <$> (choice . map string' $ weekdays ++ weekdayabbrevs) case catMaybes $ [wday `elemIndex` weekdays, wday `elemIndex` weekdayabbrevs] of (i:_) -> return (i+1) - [] -> fail $ "weekday: should not happen: attempted to find " <> - show wday <> " in " <> show (weekdays ++ weekdayabbrevs) + [] -> Fail.fail $ "weekday: should not happen: attempted to find " <> + show wday <> " in " <> show (weekdays ++ weekdayabbrevs) today,yesterday,tomorrow :: TextParser m SmartDate today = string' "today" >> return ("","","today") diff --git a/Hledger/Data/Journal.hs b/Hledger/Data/Journal.hs --- a/Hledger/Data/Journal.hs +++ b/Hledger/Data/Journal.hs @@ -24,7 +24,7 @@ journalApplyCommodityStyles, commodityStylesFromAmounts, journalCommodityStyles, - journalConvertAmountsToCost, + journalToCost, journalReverse, journalSetLastReadTime, journalPivot, @@ -905,18 +905,23 @@ -- | Choose and apply a consistent display format to the posting -- amounts in each commodity. Each commodity's format is specified by -- a commodity format directive, or otherwise inferred from posting --- amounts as in hledger < 0.28. -journalApplyCommodityStyles :: Journal -> Journal -journalApplyCommodityStyles j@Journal{jtxns=ts, jpricedirectives=pds} = j'' - where - j' = journalInferCommodityStyles j - styles = journalCommodityStyles j' - j'' = j'{jtxns=map fixtransaction ts, jpricedirectives=map fixpricedirective pds} - fixtransaction t@Transaction{tpostings=ps} = t{tpostings=map fixposting ps} - fixposting p = p{pamount=styleMixedAmount styles $ pamount p - ,pbalanceassertion=fixbalanceassertion <$> pbalanceassertion p} - fixbalanceassertion ba = ba{baamount=styleAmount styles $ baamount ba} - fixpricedirective pd@PriceDirective{pdamount=a} = pd{pdamount=styleAmount styles a} +-- amounts as in hledger < 0.28. Can return an error message +-- eg if inconsistent number formats are found. +journalApplyCommodityStyles :: Journal -> Either String Journal +journalApplyCommodityStyles j@Journal{jtxns=ts, jpricedirectives=pds} = + case journalInferCommodityStyles j of + Left e -> Left e + Right j' -> Right j'' + where + styles = journalCommodityStyles j' + j'' = j'{jtxns=map fixtransaction ts + ,jpricedirectives=map fixpricedirective pds + } + fixtransaction t@Transaction{tpostings=ps} = t{tpostings=map fixposting ps} + fixposting p = p{pamount=styleMixedAmount styles $ pamount p + ,pbalanceassertion=fixbalanceassertion <$> pbalanceassertion p} + fixbalanceassertion ba = ba{baamount=styleAmount styles $ baamount ba} + fixpricedirective pd@PriceDirective{pdamount=a} = pd{pdamount=styleAmountExceptPrecision styles a} -- | Get all the amount styles defined in this journal, either declared by -- a commodity directive or inferred from amounts, as a map from symbol to style. @@ -931,35 +936,60 @@ -- | Collect and save inferred amount styles for each commodity based on -- the posting amounts in that commodity (excluding price amounts), ie: -- "the format of the first amount, adjusted to the highest precision of all amounts". -journalInferCommodityStyles :: Journal -> Journal +-- Can return an error message eg if inconsistent number formats are found. +journalInferCommodityStyles :: Journal -> Either String Journal journalInferCommodityStyles j = - j{jinferredcommodities = + case commodityStylesFromAmounts $ - dbg8 "journalInferCommmodityStyles using amounts" $ journalAmounts j} + dbg8 "journalInferCommodityStyles using amounts" $ + journalAmounts j + of + Left e -> Left e + Right cs -> Right j{jinferredcommodities = cs} --- | Given a list of amounts in parse order, build a map from their commodity names --- to standard commodity display formats. -commodityStylesFromAmounts :: [Amount] -> M.Map CommoditySymbol AmountStyle -commodityStylesFromAmounts amts = M.fromList commstyles +-- | Given a list of parsed amounts, in parse order, build a map from +-- their commodity names to standard commodity display formats. Can +-- return an error message eg if inconsistent number formats are +-- found. +-- +-- Though, these amounts may have come from multiple files, so we +-- shouldn't assume they use consistent number formats. +-- Currently we don't enforce that even within a single file, +-- and this function never reports an error. +-- +commodityStylesFromAmounts :: [Amount] -> Either String (M.Map CommoditySymbol AmountStyle) +commodityStylesFromAmounts amts = + Right $ M.fromList commstyles where commamts = groupSort [(acommodity as, as) | as <- amts] commstyles = [(c, canonicalStyleFrom $ map astyle as) | (c,as) <- commamts] --- | Given an ordered list of amount styles, choose a canonical style. --- That is: the style of the first, and the maximum precision of all. +-- TODO: should probably detect and report inconsistencies here +-- | Given a list of amount styles (assumed to be from parsed amounts +-- in a single commodity), in parse order, choose a canonical style. +-- Traditionally it's "the style of the first, with the maximum precision of all". +-- canonicalStyleFrom :: [AmountStyle] -> AmountStyle canonicalStyleFrom [] = amountstyle -canonicalStyleFrom ss@(first:_) = first {asprecision = prec, asdecimalpoint = mdec, asdigitgroups = mgrps} +canonicalStyleFrom ss@(s:_) = + s{asprecision=prec, asdecimalpoint=Just decmark, asdigitgroups=mgrps} where - mgrps = headMay $ mapMaybe asdigitgroups ss -- precision is maximum of all precisions prec = maximumStrict $ map asprecision ss - mdec = Just $ headDef '.' $ mapMaybe asdecimalpoint ss - -- precision is that of first amount with a decimal point - -- (mdec, prec) = - -- case filter (isJust . asdecimalpoint) ss of - -- (s:_) -> (asdecimalpoint s, asprecision s) - -- [] -> (Just '.', 0) + -- identify the digit group mark (& group sizes) + mgrps = headMay $ mapMaybe asdigitgroups ss + -- if a digit group mark was identified above, we can rely on that; + -- make sure the decimal mark is different. If not, default to period. + defdecmark = + case mgrps of + Just (DigitGroups '.' _) -> ',' + _ -> '.' + -- identify the decimal mark: the first one used, or the above default, + -- but never the same character as the digit group mark. + -- urgh.. refactor.. + decmark = case mgrps of + Just _ -> defdecmark + _ -> headDef defdecmark $ mapMaybe asdecimalpoint ss -- -- | Apply this journal's historical price records to unpriced amounts where possible. -- journalApplyPriceDirectives :: Journal -> Journal @@ -983,14 +1013,9 @@ -- | Convert all this journal's amounts to cost using the transaction prices, if any. -- The journal's commodity styles are applied to the resulting amounts. -journalConvertAmountsToCost :: Journal -> Journal -journalConvertAmountsToCost j@Journal{jtxns=ts} = j{jtxns=map fixtransaction ts} +journalToCost :: Journal -> Journal +journalToCost j@Journal{jtxns=ts} = j{jtxns=map (transactionToCost styles) ts} where - -- similar to journalApplyCommodityStyles - fixtransaction t@Transaction{tpostings=ps} = t{tpostings=map fixposting ps} - fixposting p@Posting{pamount=a} = p{pamount=fixmixedamount a} - fixmixedamount (Mixed as) = Mixed $ map fixamount as - fixamount = styleAmount styles . costOfAmount styles = journalCommodityStyles j -- -- | Get this journal's unique, display-preference-canonicalised commodities, by symbol. @@ -1281,7 +1306,7 @@ } ] } - `is` (DateSpan (Just $ fromGregorian 2014 1 10) (Just $ fromGregorian 2014 10 11)) + @?= (DateSpan (Just $ fromGregorian 2014 1 10) (Just $ fromGregorian 2014 10 11)) ,tests "standard account type queries" $ let @@ -1290,11 +1315,11 @@ journalAccountNamesMatching q = filter (q `matchesAccount`) . journalAccountNames namesfrom qfunc = journalAccountNamesMatching (qfunc j) j in [ - test "assets" $ expectEq (namesfrom journalAssetAccountQuery) ["assets","assets:bank","assets:bank:checking","assets:bank:saving","assets:cash"] - ,test "liabilities" $ expectEq (namesfrom journalLiabilityAccountQuery) ["liabilities","liabilities:debts"] - ,test "equity" $ expectEq (namesfrom journalEquityAccountQuery) [] - ,test "income" $ expectEq (namesfrom journalRevenueAccountQuery) ["income","income:gifts","income:salary"] - ,test "expenses" $ expectEq (namesfrom journalExpenseAccountQuery) ["expenses","expenses:food","expenses:supplies"] + test "assets" $ assertEqual "" (namesfrom journalAssetAccountQuery) ["assets","assets:bank","assets:bank:checking","assets:bank:saving","assets:cash"] + ,test "liabilities" $ assertEqual "" (namesfrom journalLiabilityAccountQuery) ["liabilities","liabilities:debts"] + ,test "equity" $ assertEqual "" (namesfrom journalEquityAccountQuery) [] + ,test "income" $ assertEqual "" (namesfrom journalRevenueAccountQuery) ["income","income:gifts","income:salary"] + ,test "expenses" $ assertEqual "" (namesfrom journalExpenseAccountQuery) ["expenses","expenses:food","expenses:supplies"] ] ,tests "journalBalanceTransactions" [ @@ -1306,12 +1331,12 @@ nulljournal{ jtxns = [ transaction "2019/01/01" [ vpost' "a" missingamt (balassert (num 1)) ] ]} - expectRight ej + assertRight ej let Right j = ej - (jtxns j & head & tpostings & head & pamount) `is` Mixed [num 1] + (jtxns j & head & tpostings & head & pamount) @?= Mixed [num 1] ,test "same-day-1" $ do - expectRight $ journalBalanceTransactions True $ + assertRight $ journalBalanceTransactions True $ --2019/01/01 -- (a) = 1 --2019/01/01 @@ -1322,7 +1347,7 @@ ]} ,test "same-day-2" $ do - expectRight $ journalBalanceTransactions True $ + assertRight $ journalBalanceTransactions True $ --2019/01/01 -- (a) 2 = 2 --2019/01/01 @@ -1340,7 +1365,7 @@ ]} ,test "out-of-order" $ do - expectRight $ journalBalanceTransactions True $ + assertRight $ journalBalanceTransactions True $ --2019/1/2 -- (a) 1 = 2 --2019/1/1 @@ -1351,5 +1376,40 @@ ]} ] + + ,tests "commodityStylesFromAmounts" $ [ + + -- Journal similar to the one on #1091: + -- 2019/09/24 + -- (a) 1,000.00 + -- + -- 2019/09/26 + -- (a) 1000,000 + -- + test "1091a" $ do + commodityStylesFromAmounts [ + nullamt{aquantity=1000, astyle=AmountStyle L False 3 (Just ',') Nothing} + ,nullamt{aquantity=1000, astyle=AmountStyle L False 2 (Just '.') (Just (DigitGroups ',' [3]))} + ] + @?= + -- The commodity style should have period as decimal mark + -- and comma as digit group mark. + Right (M.fromList [ + ("", AmountStyle L False 3 (Just '.') (Just (DigitGroups ',' [3]))) + ]) + -- same journal, entries in reverse order + ,test "1091b" $ do + commodityStylesFromAmounts [ + nullamt{aquantity=1000, astyle=AmountStyle L False 2 (Just '.') (Just (DigitGroups ',' [3]))} + ,nullamt{aquantity=1000, astyle=AmountStyle L False 3 (Just ',') Nothing} + ] + @?= + -- The commodity style should have period as decimal mark + -- and comma as digit group mark. + Right (M.fromList [ + ("", AmountStyle L False 3 (Just '.') (Just (DigitGroups ',' [3]))) + ]) + + ] ] diff --git a/Hledger/Data/Ledger.hs b/Hledger/Data/Ledger.hs --- a/Hledger/Data/Ledger.hs +++ b/Hledger/Data/Ledger.hs @@ -109,12 +109,9 @@ -- tests tests_Ledger = - tests - "Ledger" - [ tests - "ledgerFromJournal" - [ length (ledgerPostings $ ledgerFromJournal Any nulljournal) `is` 0 - , length (ledgerPostings $ ledgerFromJournal Any samplejournal) `is` 13 - , length (ledgerPostings $ ledgerFromJournal (Depth 2) samplejournal) `is` 7 - ] - ] + tests "Ledger" [ + test "ledgerFromJournal" $ do + length (ledgerPostings $ ledgerFromJournal Any nulljournal) @?= 0 + length (ledgerPostings $ ledgerFromJournal Any samplejournal) @?= 13 + length (ledgerPostings $ ledgerFromJournal (Depth 2) samplejournal) @?= 7 + ] diff --git a/Hledger/Data/PeriodicTransaction.hs b/Hledger/Data/PeriodicTransaction.hs --- a/Hledger/Data/PeriodicTransaction.hs +++ b/Hledger/Data/PeriodicTransaction.hs @@ -24,7 +24,7 @@ import Hledger.Data.Posting (post, commentAddTagNextLine) import Hledger.Data.Transaction import Hledger.Utils.UTF8IOCompat (error') --- import Hledger.Utils.Debug +import Hledger.Utils.Debug -- $setup -- >>> :set -XOverloadedStrings @@ -45,6 +45,17 @@ nullperiodictransaction{ ptperiodexpr=t , ptspan=s, ptinterval=i, ptpostings=["a" `post` usd 1] } nulldatespan +_ptgenspan str span = do + let + t = T.pack str + (i,s) = parsePeriodExpr' nulldate t + case checkPeriodicTransactionStartDate i s t of + Just e -> error' e + Nothing -> + mapM_ (putStr . showTransaction) $ + runPeriodicTransaction + nullperiodictransaction{ ptperiodexpr=t , ptspan=s, ptinterval=i, ptpostings=["a" `post` usd 1] } + span --deriving instance Show PeriodicTransaction -- for better pretty-printing: @@ -76,107 +87,107 @@ -- -- >>> _ptgen "monthly from 2017/1 to 2017/4" -- 2017/01/01 --- ; generated-transaction:~ monthly from 2017/1 to 2017/4 +-- ; generated-transaction: ~ monthly from 2017/1 to 2017/4 -- a $1.00 -- -- 2017/02/01 --- ; generated-transaction:~ monthly from 2017/1 to 2017/4 +-- ; generated-transaction: ~ monthly from 2017/1 to 2017/4 -- a $1.00 -- -- 2017/03/01 --- ; generated-transaction:~ monthly from 2017/1 to 2017/4 +-- ; generated-transaction: ~ monthly from 2017/1 to 2017/4 -- a $1.00 -- -- -- >>> _ptgen "monthly from 2017/1 to 2017/5" -- 2017/01/01 --- ; generated-transaction:~ monthly from 2017/1 to 2017/5 +-- ; generated-transaction: ~ monthly from 2017/1 to 2017/5 -- a $1.00 -- -- 2017/02/01 --- ; generated-transaction:~ monthly from 2017/1 to 2017/5 +-- ; generated-transaction: ~ monthly from 2017/1 to 2017/5 -- a $1.00 -- -- 2017/03/01 --- ; generated-transaction:~ monthly from 2017/1 to 2017/5 +-- ; generated-transaction: ~ monthly from 2017/1 to 2017/5 -- a $1.00 -- -- 2017/04/01 --- ; generated-transaction:~ monthly from 2017/1 to 2017/5 +-- ; generated-transaction: ~ monthly from 2017/1 to 2017/5 -- a $1.00 -- -- -- >>> _ptgen "every 2nd day of month from 2017/02 to 2017/04" -- 2017/01/02 --- ; generated-transaction:~ every 2nd day of month from 2017/02 to 2017/04 +-- ; generated-transaction: ~ every 2nd day of month from 2017/02 to 2017/04 -- a $1.00 -- -- 2017/02/02 --- ; generated-transaction:~ every 2nd day of month from 2017/02 to 2017/04 +-- ; generated-transaction: ~ every 2nd day of month from 2017/02 to 2017/04 -- a $1.00 -- -- 2017/03/02 --- ; generated-transaction:~ every 2nd day of month from 2017/02 to 2017/04 +-- ; generated-transaction: ~ every 2nd day of month from 2017/02 to 2017/04 -- a $1.00 -- -- -- >>> _ptgen "every 30th day of month from 2017/1 to 2017/5" -- 2016/12/30 --- ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5 +-- ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 -- a $1.00 -- -- 2017/01/30 --- ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5 +-- ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 -- a $1.00 -- -- 2017/02/28 --- ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5 +-- ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 -- a $1.00 -- -- 2017/03/30 --- ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5 +-- ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 -- a $1.00 -- -- 2017/04/30 --- ; generated-transaction:~ every 30th day of month from 2017/1 to 2017/5 +-- ; generated-transaction: ~ every 30th day of month from 2017/1 to 2017/5 -- a $1.00 -- -- -- >>> _ptgen "every 2nd Thursday of month from 2017/1 to 2017/4" -- 2016/12/08 --- ; generated-transaction:~ every 2nd Thursday of month from 2017/1 to 2017/4 +-- ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4 -- a $1.00 -- -- 2017/01/12 --- ; generated-transaction:~ every 2nd Thursday of month from 2017/1 to 2017/4 +-- ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4 -- a $1.00 -- -- 2017/02/09 --- ; generated-transaction:~ every 2nd Thursday of month from 2017/1 to 2017/4 +-- ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4 -- a $1.00 -- -- 2017/03/09 --- ; generated-transaction:~ every 2nd Thursday of month from 2017/1 to 2017/4 +-- ; generated-transaction: ~ every 2nd Thursday of month from 2017/1 to 2017/4 -- a $1.00 -- -- -- >>> _ptgen "every nov 29th from 2017 to 2019" -- 2016/11/29 --- ; generated-transaction:~ every nov 29th from 2017 to 2019 +-- ; generated-transaction: ~ every nov 29th from 2017 to 2019 -- a $1.00 -- -- 2017/11/29 --- ; generated-transaction:~ every nov 29th from 2017 to 2019 +-- ; generated-transaction: ~ every nov 29th from 2017 to 2019 -- a $1.00 -- -- 2018/11/29 --- ; generated-transaction:~ every nov 29th from 2017 to 2019 +-- ; generated-transaction: ~ every nov 29th from 2017 to 2019 -- a $1.00 -- -- -- >>> _ptgen "2017/1" -- 2017/01/01 --- ; generated-transaction:~ 2017/1 +-- ; generated-transaction: ~ 2017/1 -- a $1.00 -- -- @@ -199,11 +210,32 @@ -- >>> let reportperiod="daily from 2018/01/03" in let (i,s) = parsePeriodExpr' nulldate reportperiod in runPeriodicTransaction (nullperiodictransaction{ptperiodexpr=reportperiod, ptspan=s, ptinterval=i, ptpostings=["a" `post` usd 1]}) (DateSpan (Just $ parsedate "2018-01-01") (Just $ parsedate "2018-01-03")) -- [] -- +-- >>> _ptgenspan "every 3 months from 2019-05" (mkdatespan "2020-01-01" "2020-02-01") +-- +-- >>> _ptgenspan "every 3 months from 2019-05" (mkdatespan "2020-02-01" "2020-03-01") +-- 2020/02/01 +-- ; generated-transaction: ~ every 3 months from 2019-05 +-- a $1.00 +-- +-- >>> _ptgenspan "every 3 days from 2018" (mkdatespan "2018-01-01" "2018-01-05") +-- 2018/01/01 +-- ; generated-transaction: ~ every 3 days from 2018 +-- a $1.00 +-- +-- 2018/01/04 +-- ; generated-transaction: ~ every 3 days from 2018 +-- a $1.00 +-- +-- >>> _ptgenspan "every 3 days from 2018" (mkdatespan "2018-01-02" "2018-01-05") +-- 2018/01/04 +-- ; generated-transaction: ~ every 3 days from 2018 +-- a $1.00 +-- + runPeriodicTransaction :: PeriodicTransaction -> DateSpan -> [Transaction] runPeriodicTransaction PeriodicTransaction{..} requestedspan = - [ t{tdate=d} | (DateSpan (Just d) _) <- ptinterval `splitSpan` spantofill ] + [ t{tdate=d} | (DateSpan (Just d) _) <- alltxnspans, spanContainsDate requestedspan d ] where - spantofill = spanIntervalIntersect ptinterval ptspan requestedspan t = nulltransaction{ tstatus = ptstatus ,tcode = ptcode @@ -216,7 +248,11 @@ ,tpostings = ptpostings } period = "~ " <> ptperiodexpr - + -- All spans described by this periodic transaction, where spanStart is event date. + -- If transaction does not have start/end date, we set them to start/end of requested span, + -- to avoid generating (infinitely) many events. + alltxnspans = dbg3 "alltxnspans" $ ptinterval `splitSpan` (spanDefaultsFrom ptspan requestedspan) + -- | Check that this date span begins at a boundary of this interval, -- or return an explanatory error message including the provided period expression -- (from which the span and interval are derived). diff --git a/Hledger/Data/Posting.hs b/Hledger/Data/Posting.hs --- a/Hledger/Data/Posting.hs +++ b/Hledger/Data/Posting.hs @@ -372,58 +372,54 @@ | otherwise = c1 <> ", " <> c2 -- | Add a tag to a comment, comma-separated from any prior content. +-- A space is inserted following the colon, before the value. commentAddTag :: Text -> Tag -> Text commentAddTag c (t,v) | T.null c' = tag | otherwise = c' `commentJoin` tag where c' = textchomp c - tag = t <> ":" <> v + tag = t <> ": " <> v -- | Add a tag on its own line to a comment, preserving any prior content. +-- A space is inserted following the colon, before the value. commentAddTagNextLine :: Text -> Tag -> Text commentAddTagNextLine cmt (t,v) = - cmt <> if "\n" `T.isSuffixOf` cmt then "" else "\n" <> t <> ":" <> v + cmt <> if "\n" `T.isSuffixOf` cmt then "" else "\n" <> t <> ": " <> v -- tests tests_Posting = tests "Posting" [ - tests "accountNamePostingType" [ - accountNamePostingType "a" `is` RegularPosting - ,accountNamePostingType "(a)" `is` VirtualPosting - ,accountNamePostingType "[a]" `is` BalancedVirtualPosting - ] + test "accountNamePostingType" $ do + accountNamePostingType "a" @?= RegularPosting + accountNamePostingType "(a)" @?= VirtualPosting + accountNamePostingType "[a]" @?= BalancedVirtualPosting - ,tests "accountNameWithoutPostingType" [ - accountNameWithoutPostingType "(a)" `is` "a" - ] + ,test "accountNameWithoutPostingType" $ do + accountNameWithoutPostingType "(a)" @?= "a" - ,tests "accountNameWithPostingType" [ - accountNameWithPostingType VirtualPosting "[a]" `is` "(a)" - ] + ,test "accountNameWithPostingType" $ do + accountNameWithPostingType VirtualPosting "[a]" @?= "(a)" - ,tests "joinAccountNames" [ - "a" `joinAccountNames` "b:c" `is` "a:b:c" - ,"a" `joinAccountNames` "(b:c)" `is` "(a:b:c)" - ,"[a]" `joinAccountNames` "(b:c)" `is` "[a:b:c]" - ,"" `joinAccountNames` "a" `is` "a" - ] + ,test "joinAccountNames" $ do + "a" `joinAccountNames` "b:c" @?= "a:b:c" + "a" `joinAccountNames` "(b:c)" @?= "(a:b:c)" + "[a]" `joinAccountNames` "(b:c)" @?= "[a:b:c]" + "" `joinAccountNames` "a" @?= "a" - ,tests "concatAccountNames" [ - concatAccountNames [] `is` "" - ,concatAccountNames ["a","(b)","[c:d]"] `is` "(a:b:c:d)" - ] + ,test "concatAccountNames" $ do + concatAccountNames [] @?= "" + concatAccountNames ["a","(b)","[c:d]"] @?= "(a:b:c:d)" - ,tests "commentAddTag" [ - commentAddTag "" ("a","") `is` "a:" - ,commentAddTag "[1/2]" ("a","") `is` "[1/2], a:" - ] + ,test "commentAddTag" $ do + commentAddTag "" ("a","") @?= "a: " + commentAddTag "[1/2]" ("a","") @?= "[1/2], a: " - ,tests "commentAddTagNextLine" [ - commentAddTagNextLine "" ("a","") `is` "\na:" - ,commentAddTagNextLine "[1/2]" ("a","") `is` "[1/2]\na:" - ] + ,test "commentAddTagNextLine" $ do + commentAddTagNextLine "" ("a","") @?= "\na: " + commentAddTagNextLine "[1/2]" ("a","") @?= "[1/2]\na: " + ] diff --git a/Hledger/Data/RawOptions.hs b/Hledger/Data/RawOptions.hs --- a/Hledger/Data/RawOptions.hs +++ b/Hledger/Data/RawOptions.hs @@ -1,3 +1,5 @@ +{-# LANGUAGE DeriveDataTypeable #-} + {-| hledger's cmdargs modes parse command-line arguments to an @@ -13,6 +15,8 @@ setboolopt, inRawOpts, boolopt, + choiceopt, + collectopts, stringopt, maybestringopt, listofstringopt, @@ -23,39 +27,69 @@ where import Data.Maybe -import qualified Data.Text as T +import Data.Data +import Data.Default import Safe import Hledger.Utils -- | The result of running cmdargs: an association list of option names to string values. -type RawOpts = [(String,String)] +newtype RawOpts = RawOpts { unRawOpts :: [(String,String)] } + deriving (Show, Data, Typeable) +instance Default RawOpts where def = RawOpts [] + +overRawOpts f = RawOpts . f . unRawOpts + setopt :: String -> String -> RawOpts -> RawOpts -setopt name val = (++ [(name, quoteIfNeeded $ val)]) +setopt name val = overRawOpts (++ [(name, val)]) setboolopt :: String -> RawOpts -> RawOpts -setboolopt name = (++ [(name,"")]) +setboolopt name = overRawOpts (++ [(name,"")]) -- | Is the named option present ? inRawOpts :: String -> RawOpts -> Bool -inRawOpts name = isJust . lookup name +inRawOpts name = isJust . lookup name . unRawOpts boolopt :: String -> RawOpts -> Bool boolopt = inRawOpts +-- | From a list of RawOpts, get the last one (ie the right-most on the command line) +-- for which the given predicate returns a Just value. +-- Useful for exclusive choice flags like --daily|--weekly|--quarterly... +-- +-- >>> choiceopt Just (RawOpts [("a",""), ("b",""), ("c","")]) +-- Just "c" +-- >>> choiceopt (const Nothing) (RawOpts [("a","")]) +-- Nothing +-- >>> choiceopt readMay (RawOpts [("LT",""),("EQ",""),("Neither","")]) :: Maybe Ordering +-- Just EQ +choiceopt :: (String -> Maybe a) -- ^ "parser" that returns 'Just' value for valid choice + -> RawOpts -- ^ actual options where to look for flag + -> Maybe a -- ^ exclusive choice among those returned as 'Just' from "parser" +choiceopt f = lastMay . collectopts (f . fst) + +-- | Collects processed and filtered list of options preserving their order +-- +-- >>> collectopts (const Nothing) (RawOpts [("x","")]) +-- [] +-- >>> collectopts Just (RawOpts [("a",""),("b","")]) +-- [("a",""),("b","")] +collectopts :: ((String, String) -> Maybe a) -> RawOpts -> [a] +collectopts f = mapMaybe f . unRawOpts + maybestringopt :: String -> RawOpts -> Maybe String -maybestringopt name = fmap (T.unpack . stripquotes . T.pack) . lookup name . reverse +maybestringopt name = lookup name . reverse . unRawOpts stringopt :: String -> RawOpts -> String stringopt name = fromMaybe "" . maybestringopt name maybecharopt :: String -> RawOpts -> Maybe Char -maybecharopt name rawopts = lookup name rawopts >>= headMay +maybecharopt name (RawOpts rawopts) = lookup name rawopts >>= headMay listofstringopt :: String -> RawOpts -> [String] -listofstringopt name rawopts = [v | (k,v) <- rawopts, k==name] +listofstringopt name (RawOpts rawopts) = [v | (k,v) <- rawopts, k==name] maybeintopt :: String -> RawOpts -> Maybe Int maybeintopt name rawopts = diff --git a/Hledger/Data/StringFormat.hs b/Hledger/Data/StringFormat.hs --- a/Hledger/Data/StringFormat.hs +++ b/Hledger/Data/StringFormat.hs @@ -18,7 +18,7 @@ import Numeric import Data.Char (isPrint) import Data.Maybe -import qualified Data.Text as T +-- import qualified Data.Text as T import Text.Megaparsec import Text.Megaparsec.Char @@ -137,7 +137,7 @@ ---------------------------------------------------------------------- -formatStringTester fs value expected = actual `is` expected +formatStringTester fs value expected = actual @?= expected where actual = case fs of FormatLiteral l -> formatString False Nothing Nothing l @@ -145,20 +145,18 @@ tests_StringFormat = tests "StringFormat" [ - tests "formatStringHelper" [ + test "formatStringHelper" $ do formatStringTester (FormatLiteral " ") "" " " - , formatStringTester (FormatField False Nothing Nothing DescriptionField) "description" "description" - , formatStringTester (FormatField False (Just 20) Nothing DescriptionField) "description" " description" - , formatStringTester (FormatField False Nothing (Just 20) DescriptionField) "description" "description" - , formatStringTester (FormatField True Nothing (Just 20) DescriptionField) "description" "description" - , formatStringTester (FormatField True (Just 20) Nothing DescriptionField) "description" "description " - , formatStringTester (FormatField True (Just 20) (Just 20) DescriptionField) "description" "description " - , formatStringTester (FormatField True Nothing (Just 3) DescriptionField) "description" "des" - ] + formatStringTester (FormatField False Nothing Nothing DescriptionField) "description" "description" + formatStringTester (FormatField False (Just 20) Nothing DescriptionField) "description" " description" + formatStringTester (FormatField False Nothing (Just 20) DescriptionField) "description" "description" + formatStringTester (FormatField True Nothing (Just 20) DescriptionField) "description" "description" + formatStringTester (FormatField True (Just 20) Nothing DescriptionField) "description" "description " + formatStringTester (FormatField True (Just 20) (Just 20) DescriptionField) "description" "description " + formatStringTester (FormatField True Nothing (Just 3) DescriptionField) "description" "des" - ,tests "parseStringFormat" $ - let s `gives` expected = test (T.pack s) $ parseStringFormat s `is` Right expected - in [ + ,let s `gives` expected = test s $ parseStringFormat s @?= Right expected + in tests "parseStringFormat" [ "" `gives` (defaultStringFormatStyle []) , "D" `gives` (defaultStringFormatStyle [FormatLiteral "D"]) , "%(date)" `gives` (defaultStringFormatStyle [FormatField False Nothing Nothing DescriptionField]) @@ -176,6 +174,6 @@ ,FormatLiteral " " ,FormatField False Nothing (Just 10) TotalField ]) - , test "newline not parsed" $ expectLeft $ parseStringFormat "\n" + , test "newline not parsed" $ assertLeft $ parseStringFormat "\n" ] ] diff --git a/Hledger/Data/Timeclock.hs b/Hledger/Data/Timeclock.hs --- a/Hledger/Data/Timeclock.hs +++ b/Hledger/Data/Timeclock.hs @@ -115,32 +115,31 @@ -- tests tests_Timeclock = tests "Timeclock" [ - do - today <- io getCurrentDay - now' <- io getCurrentTime - tz <- io getCurrentTimeZone - let now = utcToLocalTime tz now' - nowstr = showtime now - yesterday = prevday today - clockin = TimeclockEntry nullsourcepos In - mktime d = LocalTime d . fromMaybe midnight . + testCaseSteps "timeclockEntriesToTransactions tests" $ \step -> do + step "gathering data" + today <- getCurrentDay + now' <- getCurrentTime + tz <- getCurrentTimeZone + let now = utcToLocalTime tz now' + nowstr = showtime now + yesterday = prevday today + clockin = TimeclockEntry nullsourcepos In + mktime d = LocalTime d . fromMaybe midnight . #if MIN_VERSION_time(1,5,0) - parseTimeM True defaultTimeLocale "%H:%M:%S" + parseTimeM True defaultTimeLocale "%H:%M:%S" #else - parseTime defaultTimeLocale "%H:%M:%S" + parseTime defaultTimeLocale "%H:%M:%S" #endif - showtime = formatTime defaultTimeLocale "%H:%M" - txndescs = map (T.unpack . tdescription) . timeclockEntriesToTransactions now - future = utcToLocalTime tz $ addUTCTime 100 now' - futurestr = showtime future - tests "timeclockEntriesToTransactions" [ - test "started yesterday, split session at midnight" $ - txndescs [clockin (mktime yesterday "23:00:00") "" ""] `is` ["23:00-23:59","00:00-"++nowstr] - ,test "split multi-day sessions at each midnight" $ - txndescs [clockin (mktime (addDays (-2) today) "23:00:00") "" ""] `is `["23:00-23:59","00:00-23:59","00:00-"++nowstr] - ,test "auto-clock-out if needed" $ - txndescs [clockin (mktime today "00:00:00") "" ""] `is` ["00:00-"++nowstr] - ,test "use the clockin time for auto-clockout if it's in the future" $ - txndescs [clockin future "" ""] `is` [printf "%s-%s" futurestr futurestr] - ] + showtime = formatTime defaultTimeLocale "%H:%M" + txndescs = map (T.unpack . tdescription) . timeclockEntriesToTransactions now + future = utcToLocalTime tz $ addUTCTime 100 now' + futurestr = showtime future + step "started yesterday, split session at midnight" + txndescs [clockin (mktime yesterday "23:00:00") "" ""] @?= ["23:00-23:59","00:00-"++nowstr] + step "split multi-day sessions at each midnight" + txndescs [clockin (mktime (addDays (-2) today) "23:00:00") "" ""] @?= ["23:00-23:59","00:00-23:59","00:00-"++nowstr] + step "auto-clock-out if needed" + txndescs [clockin (mktime today "00:00:00") "" ""] @?= ["00:00-"++nowstr] + step "use the clockin time for auto-clockout if it's in the future" + txndescs [clockin future "" ""] @?= [printf "%s-%s" futurestr futurestr] ] diff --git a/Hledger/Data/Transaction.hs b/Hledger/Data/Transaction.hs --- a/Hledger/Data/Transaction.hs +++ b/Hledger/Data/Transaction.hs @@ -30,6 +30,9 @@ isTransactionBalanced, balanceTransaction, balanceTransactionHelper, + transactionTransformPostings, + transactionApplyValuation, + transactionToCost, -- nonzerobalanceerror, -- * date operations transactionDate2, @@ -41,6 +44,7 @@ -- payeeAndNoteFromDescription, -- * rendering showTransaction, + showTransactionOneLineAmounts, showTransactionUnelided, showTransactionUnelidedOneLineAmounts, -- showPostingLine, @@ -60,13 +64,14 @@ import qualified Data.Text as T import Data.Time.Calendar import Text.Printf -import qualified Data.Map as Map +import qualified Data.Map as M import Hledger.Utils import Hledger.Data.Types import Hledger.Data.Dates import Hledger.Data.Posting import Hledger.Data.Amount +import Hledger.Data.Valuation sourceFilePath :: GenericSourcePos -> FilePath sourceFilePath = \case @@ -142,40 +147,29 @@ To facilitate this, postings with explicit multi-commodity amounts are displayed as multiple similar postings, one per commodity. (Normally does not happen with this function). - -If there are multiple postings, all with explicit amounts, -and the transaction appears obviously balanced -(postings sum to 0, without needing to infer conversion prices), -the last posting's amount will not be shown. -} --- XXX why that logic ? --- XXX where is/should this be still used ? --- XXX rename these, after amount expressions/mixed posting amounts lands --- eg showTransactionSimpleAmountsElidingLast, showTransactionSimpleAmounts, showTransaction showTransaction :: Transaction -> String -showTransaction = showTransactionHelper True False +showTransaction = showTransactionHelper False --- | Like showTransaction, but does not change amounts' explicitness. --- Explicit amounts are shown and implicit amounts are not. --- The output will be parseable journal syntax. --- To facilitate this, postings with explicit multi-commodity amounts --- are displayed as multiple similar postings, one per commodity. --- Most often, this is the one you want to use. +-- | Deprecated alias for 'showTransaction' showTransactionUnelided :: Transaction -> String -showTransactionUnelided = showTransactionHelper False False +showTransactionUnelided = showTransaction -- TODO: drop it --- | Like showTransactionUnelided, but explicit multi-commodity amounts +-- | Like showTransaction, but explicit multi-commodity amounts -- are shown on one line, comma-separated. In this case the output will -- not be parseable journal syntax. -showTransactionUnelidedOneLineAmounts :: Transaction -> String -showTransactionUnelidedOneLineAmounts = showTransactionHelper False True +showTransactionOneLineAmounts :: Transaction -> String +showTransactionOneLineAmounts = showTransactionHelper True +-- | Deprecated alias for 'showTransactionOneLineAmounts' +showTransactionUnelidedOneLineAmounts = showTransactionOneLineAmounts -- TODO: drop it + -- | Helper for showTransaction*. -showTransactionHelper :: Bool -> Bool -> Transaction -> String -showTransactionHelper elide onelineamounts t = +showTransactionHelper :: Bool -> Transaction -> String +showTransactionHelper onelineamounts t = unlines $ [descriptionline] ++ newlinecomments - ++ (postingsAsLines elide onelineamounts t (tpostings t)) + ++ (postingsAsLines onelineamounts (tpostings t)) ++ [""] where descriptionline = rstrip $ concat [date, status, code, desc, samelinecomment] @@ -207,10 +201,6 @@ -- -- Explicit amounts are shown, any implicit amounts are not. -- --- Setting elide to true forces the last posting's amount to be implicit, if: --- there are other postings, all with explicit amounts, and the transaction --- appears balanced. --- -- Postings with multicommodity explicit amounts are handled as follows: -- if onelineamounts is true, these amounts are shown on one line, -- comma-separated, and the output will not be valid journal syntax. @@ -223,11 +213,8 @@ -- Posting amounts will be aligned with each other, starting about 4 columns -- beyond the widest account name (see postingAsLines for details). -- -postingsAsLines :: Bool -> Bool -> Transaction -> [Posting] -> [String] -postingsAsLines elide onelineamounts t ps - | elide && length ps > 1 && all hasAmount ps && isTransactionBalanced Nothing t -- imprecise balanced check - = concatMap (postingAsLines False onelineamounts ps) (init ps) ++ postingAsLines True onelineamounts ps (last ps) - | otherwise = concatMap (postingAsLines False onelineamounts ps) ps +postingsAsLines :: Bool -> [Posting] -> [String] +postingsAsLines onelineamounts ps = concatMap (postingAsLines False onelineamounts ps) ps -- | Render one posting, on one or more lines, suitable for `print` output. -- There will be an indented account name, plus one or more of status flag, @@ -356,7 +343,7 @@ -- and summing the real postings, and summing the balanced virtual postings; -- and applying the given display styles if any (maybe affecting decimal places); -- do both totals appear to be zero when rendered ? -isTransactionBalanced :: Maybe (Map.Map CommoditySymbol AmountStyle) -> Transaction -> Bool +isTransactionBalanced :: Maybe (M.Map CommoditySymbol AmountStyle) -> Transaction -> Bool isTransactionBalanced styles t = -- isReallyZeroMixedAmountCost rsum && isReallyZeroMixedAmountCost bvsum isZeroMixedAmount rsum' && isZeroMixedAmount bvsum' @@ -380,7 +367,7 @@ -- if provided, so that the result agrees with the numbers users can see. -- balanceTransaction :: - Maybe (Map.Map CommoditySymbol AmountStyle) -- ^ commodity display styles + Maybe (M.Map CommoditySymbol AmountStyle) -- ^ commodity display styles -> Transaction -> Either String Transaction balanceTransaction mstyles = fmap fst . balanceTransactionHelper mstyles @@ -389,12 +376,12 @@ -- use one of those instead. It also returns a list of accounts -- and amounts that were inferred. balanceTransactionHelper :: - Maybe (Map.Map CommoditySymbol AmountStyle) -- ^ commodity display styles + Maybe (M.Map CommoditySymbol AmountStyle) -- ^ commodity display styles -> Transaction -> Either String (Transaction, [(AccountName, MixedAmount)]) balanceTransactionHelper mstyles t = do (t', inferredamtsandaccts) <- - inferBalancingAmount (fromMaybe Map.empty mstyles) $ inferBalancingPrices t + inferBalancingAmount (fromMaybe M.empty mstyles) $ inferBalancingPrices t if isTransactionBalanced mstyles t' then Right (txnTieKnot t', inferredamtsandaccts) else Left $ annotateErrorWithTransaction t' $ nonzerobalanceerror t' @@ -413,7 +400,7 @@ sep = if not (null rmsg) && not (null bvmsg) then "; " else "" :: String annotateErrorWithTransaction :: Transaction -> String -> String -annotateErrorWithTransaction t s = intercalate "\n" [showGenericSourcePos $ tsourcepos t, s, showTransactionUnelided t] +annotateErrorWithTransaction t s = intercalate "\n" [showGenericSourcePos $ tsourcepos t, s, showTransaction t] -- | Infer up to one missing amount for this transactions's real postings, and -- likewise for its balanced virtual postings, if needed; or return an error @@ -424,7 +411,7 @@ -- one of them is amountless. If the amounts had price(s) the inferred amount -- have the same price(s), and will be converted to the price commodity. inferBalancingAmount :: - Map.Map CommoditySymbol AmountStyle -- ^ commodity display styles + M.Map CommoditySymbol AmountStyle -- ^ commodity display styles -> Transaction -> Either String (Transaction, [(AccountName, MixedAmount)]) inferBalancingAmount styles t@Transaction{tpostings=ps} @@ -553,48 +540,31 @@ postingSetTransaction :: Transaction -> Posting -> Posting postingSetTransaction t p = p{ptransaction=Just t} +-- | Apply a transform function to this transaction's amounts. +transactionTransformPostings :: (Posting -> Posting) -> Transaction -> Transaction +transactionTransformPostings f t@Transaction{tpostings=ps} = t{tpostings=map f ps} + +-- | Apply a specified valuation to this transaction's amounts, using +-- the provided price oracle, commodity styles, reference dates, and +-- whether this is for a multiperiod report or not. See +-- amountApplyValuation. +transactionApplyValuation :: PriceOracle -> M.Map CommoditySymbol AmountStyle -> Day -> Maybe Day -> Day -> Bool -> Transaction -> ValuationType -> Transaction +transactionApplyValuation priceoracle styles periodlast mreportlast today ismultiperiod t v = + transactionTransformPostings (\p -> postingApplyValuation priceoracle styles periodlast mreportlast today ismultiperiod p v) t + +-- | Convert this transaction's amounts to cost, and apply the appropriate amount styles. +transactionToCost :: M.Map CommoditySymbol AmountStyle -> Transaction -> Transaction +transactionToCost styles t@Transaction{tpostings=ps} = t{tpostings=map (postingToCost styles) ps} + -- tests tests_Transaction = - tests - "Transaction" - [ tests - "showTransactionUnelided" - [ showTransactionUnelided nulltransaction `is` "0000/01/01\n\n" - , showTransactionUnelided - nulltransaction - { tdate = parsedate "2012/05/14" - , tdate2 = Just $ parsedate "2012/05/15" - , tstatus = Unmarked - , tcode = "code" - , tdescription = "desc" - , tcomment = "tcomment1\ntcomment2\n" - , ttags = [("ttag1", "val1")] - , tpostings = - [ nullposting - { pstatus = Cleared - , paccount = "a" - , pamount = Mixed [usd 1, hrs 2] - , pcomment = "\npcomment2\n" - , ptype = RegularPosting - , ptags = [("ptag1", "val1"), ("ptag2", "val2")] - } - ] - } `is` - unlines - [ "2012/05/14=2012/05/15 (code) desc ; tcomment1" - , " ; tcomment2" - , " * a $1.00" - , " ; pcomment2" - , " * a 2.00h" - , " ; pcomment2" - , "" - ] - ] - , tests - "postingAsLines" - [ postingAsLines False False [posting] posting `is` [""] - , let p = + tests "Transaction" [ + + tests "postingAsLines" [ + test "null posting" $ postingAsLines False False [posting] posting @?= [""] + , test "non-null posting" $ + let p = posting { pstatus = Cleared , paccount = "a" @@ -603,7 +573,7 @@ , ptype = RegularPosting , ptags = [("ptag1", "val1"), ("ptag2", "val2")] } - in postingAsLines False False [p] p `is` + in postingAsLines False False [p] p @?= [ " * a $1.00 ; pcomment1" , " ; pcomment2" , " ; tag3: val3 " @@ -612,106 +582,90 @@ , " ; tag3: val3 " ] ] - -- postingsAsLines - -- one implicit amount - , let timp = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` missingamt]} - -- explicit amounts, balanced - texp = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` usd (-1)]} - -- explicit amount, only one posting - texp1 = nulltransaction {tpostings = ["(a)" `post` usd 1]} - -- explicit amounts, two commodities, explicit balancing price - texp2 = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` (hrs (-1) `at` usd 1)]} - -- explicit amounts, two commodities, implicit balancing price - texp2b = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` hrs (-1)]} - -- one missing amount, not the last one - t3 = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` missingamt, "c" `post` usd (-1)]} - -- unbalanced amounts when precision is limited (#931) - t4 = nulltransaction {tpostings = ["a" `post` usd (-0.01), "b" `post` usd (0.005), "c" `post` usd (0.005)]} - in tests - "postingsAsLines" - [ test "null-transaction" $ - let t = nulltransaction - in postingsAsLines True False t (tpostings t) `is` [] - , test "implicit-amount-elide-false" $ - let t = timp - in postingsAsLines False False t (tpostings t) `is` - [ " a $1.00" - , " b" -- implicit amount remains implicit - ] - , test "implicit-amount-elide-true" $ - let t = timp - in postingsAsLines True False t (tpostings t) `is` + + , let + -- one implicit amount + timp = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` missingamt]} + -- explicit amounts, balanced + texp = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` usd (-1)]} + -- explicit amount, only one posting + texp1 = nulltransaction {tpostings = ["(a)" `post` usd 1]} + -- explicit amounts, two commodities, explicit balancing price + texp2 = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` (hrs (-1) `at` usd 1)]} + -- explicit amounts, two commodities, implicit balancing price + texp2b = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` hrs (-1)]} + -- one missing amount, not the last one + t3 = nulltransaction {tpostings = ["a" `post` usd 1, "b" `post` missingamt, "c" `post` usd (-1)]} + -- unbalanced amounts when precision is limited (#931) + -- t4 = nulltransaction {tpostings = ["a" `post` usd (-0.01), "b" `post` usd (0.005), "c" `post` usd (0.005)]} + in tests "postingsAsLines" [ + test "null-transaction" $ postingsAsLines False (tpostings nulltransaction) @?= [] + , test "implicit-amount" $ postingsAsLines False (tpostings timp) @?= [ " a $1.00" , " b" -- implicit amount remains implicit ] - , test "explicit-amounts-elide-false" $ - let t = texp - in postingsAsLines False False t (tpostings t) `is` - [ " a $1.00" - , " b $-1.00" -- both amounts remain explicit - ] - , test "explicit-amounts-elide-true" $ - let t = texp - in postingsAsLines True False t (tpostings t) `is` + , test "explicit-amounts" $ postingsAsLines False (tpostings texp) @?= [ " a $1.00" - , " b" -- explicit amount is made implicit + , " b $-1.00" ] - , test "one-explicit-amount-elide-true" $ - let t = texp1 - in postingsAsLines True False t (tpostings t) `is` - [ " (a) $1.00" -- explicit amount remains explicit since only one posting + , test "one-explicit-amount" $ postingsAsLines False (tpostings texp1) @?= + [ " (a) $1.00" ] - , test "explicit-amounts-two-commodities-elide-true" $ - let t = texp2 - in postingsAsLines True False t (tpostings t) `is` + , test "explicit-amounts-two-commodities" $ postingsAsLines False (tpostings texp2) @?= [ " a $1.00" - , " b" -- explicit amount is made implicit since txn is explicitly balanced + , " b -1.00h @ $1.00" ] - , test "explicit-amounts-not-explicitly-balanced-elide-true" $ - let t = texp2b - in postingsAsLines True False t (tpostings t) `is` + , test "explicit-amounts-not-explicitly-balanced" $ postingsAsLines False (tpostings texp2b) @?= [ " a $1.00" - , " b -1.00h" -- explicit amount remains explicit since a conversion price would have be inferred to balance + , " b -1.00h" ] - , test "implicit-amount-not-last" $ - let t = t3 - in postingsAsLines True False t (tpostings t) `is` + , test "implicit-amount-not-last" $ postingsAsLines False (tpostings t3) @?= [" a $1.00", " b", " c $-1.00"] - , _test "ensure-visibly-balanced" $ - let t = t4 - in postingsAsLines False False t (tpostings t) `is` - [" a $-0.01", " b $0.005", " c $0.005"] + -- , test "ensure-visibly-balanced" $ + -- in postingsAsLines False (tpostings t4) @?= + -- [" a $-0.01", " b $0.005", " c $0.005"] + ] - , tests - "inferBalancingAmount" - [ (fst <$> inferBalancingAmount Map.empty nulltransaction) `is` Right nulltransaction - , (fst <$> inferBalancingAmount Map.empty nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` missingamt]}) `is` + + , test "inferBalancingAmount" $ do + (fst <$> inferBalancingAmount M.empty nulltransaction) @?= Right nulltransaction + (fst <$> inferBalancingAmount M.empty nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` missingamt]}) @?= Right nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` usd 5]} - , (fst <$> inferBalancingAmount Map.empty nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` (eur 3 @@ usd 4), "c" `post` missingamt]}) `is` + (fst <$> inferBalancingAmount M.empty nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` (eur 3 @@ usd 4), "c" `post` missingamt]}) @?= Right nulltransaction{tpostings = ["a" `post` usd (-5), "b" `post` (eur 3 @@ usd 4), "c" `post` usd 1]} - ] - , tests - "showTransaction" - [ test "show a balanced transaction, eliding last amount" $ - let t = - Transaction - 0 - "" - nullsourcepos - (parsedate "2007/01/28") - Nothing - Unmarked - "" - "coopportunity" - "" - [] - [ posting {paccount = "expenses:food:groceries", pamount = Mixed [usd 47.18], ptransaction = Just t} - , posting {paccount = "assets:checking", pamount = Mixed [usd (-47.18)], ptransaction = Just t} + + , tests "showTransaction" [ + test "null transaction" $ showTransaction nulltransaction @?= "0000/01/01\n\n" + , test "non-null transaction" $ showTransaction + nulltransaction + { tdate = parsedate "2012/05/14" + , tdate2 = Just $ parsedate "2012/05/15" + , tstatus = Unmarked + , tcode = "code" + , tdescription = "desc" + , tcomment = "tcomment1\ntcomment2\n" + , ttags = [("ttag1", "val1")] + , tpostings = + [ nullposting + { pstatus = Cleared + , paccount = "a" + , pamount = Mixed [usd 1, hrs 2] + , pcomment = "\npcomment2\n" + , ptype = RegularPosting + , ptags = [("ptag1", "val1"), ("ptag2", "val2")] + } ] - in showTransaction t `is` - unlines - ["2007/01/28 coopportunity", " expenses:food:groceries $47.18", " assets:checking", ""] - , test "show a balanced transaction, no eliding" $ + } @?= + unlines + [ "2012/05/14=2012/05/15 (code) desc ; tcomment1" + , " ; tcomment2" + , " * a $1.00" + , " ; pcomment2" + , " * a 2.00h" + , " ; pcomment2" + , "" + ] + , test "show a balanced transaction" $ (let t = Transaction 0 @@ -727,14 +681,13 @@ [ posting {paccount = "expenses:food:groceries", pamount = Mixed [usd 47.18], ptransaction = Just t} , posting {paccount = "assets:checking", pamount = Mixed [usd (-47.18)], ptransaction = Just t} ] - in showTransactionUnelided t) `is` + in showTransaction t) @?= (unlines [ "2007/01/28 coopportunity" , " expenses:food:groceries $47.18" , " assets:checking $-47.18" , "" ]) - -- document some cases that arise in debug/testing: , test "show an unbalanced transaction, should not elide" $ (showTransaction (txnTieKnot $ @@ -751,29 +704,13 @@ [] [ posting {paccount = "expenses:food:groceries", pamount = Mixed [usd 47.18]} , posting {paccount = "assets:checking", pamount = Mixed [usd (-47.19)]} - ])) `is` + ])) @?= (unlines [ "2007/01/28 coopportunity" , " expenses:food:groceries $47.18" , " assets:checking $-47.19" , "" ]) - , test "show an unbalanced transaction with one posting, should not elide" $ - (showTransaction - (txnTieKnot $ - Transaction - 0 - "" - nullsourcepos - (parsedate "2007/01/28") - Nothing - Unmarked - "" - "coopportunity" - "" - [] - [posting {paccount = "expenses:food:groceries", pamount = Mixed [usd 47.18]}])) `is` - (unlines ["2007/01/28 coopportunity", " expenses:food:groceries $47.18", ""]) , test "show a transaction with one posting and a missing amount" $ (showTransaction (txnTieKnot $ @@ -788,7 +725,7 @@ "coopportunity" "" [] - [posting {paccount = "expenses:food:groceries", pamount = missingmixedamt}])) `is` + [posting {paccount = "expenses:food:groceries", pamount = missingmixedamt}])) @?= (unlines ["2007/01/28 coopportunity", " expenses:food:groceries", ""]) , test "show a transaction with a priced commodityless amount" $ (showTransaction @@ -806,13 +743,12 @@ [] [ posting {paccount = "a", pamount = Mixed [num 1 `at` (usd 2 `withPrecision` 0)]} , posting {paccount = "b", pamount = missingmixedamt} - ])) `is` + ])) @?= (unlines ["2010/01/01 x", " a 1 @ $2", " b", ""]) ] - , tests - "balanceTransaction" - [ test "detect unbalanced entry, sign error" $ - expectLeft + , tests "balanceTransaction" [ + test "detect unbalanced entry, sign error" $ + assertLeft (balanceTransaction Nothing (Transaction @@ -827,8 +763,8 @@ "" [] [posting {paccount = "a", pamount = Mixed [usd 1]}, posting {paccount = "b", pamount = Mixed [usd 1]}])) - , test "detect unbalanced entry, multiple missing amounts" $ - expectLeft $ + ,test "detect unbalanced entry, multiple missing amounts" $ + assertLeft $ balanceTransaction Nothing (Transaction @@ -845,7 +781,7 @@ [ posting {paccount = "a", pamount = missingmixedamt} , posting {paccount = "b", pamount = missingmixedamt} ]) - , test "one missing amount is inferred" $ + ,test "one missing amount is inferred" $ (pamount . last . tpostings <$> balanceTransaction Nothing @@ -860,9 +796,9 @@ "" "" [] - [posting {paccount = "a", pamount = Mixed [usd 1]}, posting {paccount = "b", pamount = missingmixedamt}])) `is` + [posting {paccount = "a", pamount = Mixed [usd 1]}, posting {paccount = "b", pamount = missingmixedamt}])) @?= Right (Mixed [usd (-1)]) - , test "conversion price is inferred" $ + ,test "conversion price is inferred" $ (pamount . head . tpostings <$> balanceTransaction Nothing @@ -879,10 +815,10 @@ [] [ posting {paccount = "a", pamount = Mixed [usd 1.35]} , posting {paccount = "b", pamount = Mixed [eur (-1)]} - ])) `is` + ])) @?= Right (Mixed [usd 1.35 @@ (eur 1 `withPrecision` maxprecision)]) - , test "balanceTransaction balances based on cost if there are unit prices" $ - expectRight $ + ,test "balanceTransaction balances based on cost if there are unit prices" $ + assertRight $ balanceTransaction Nothing (Transaction @@ -899,8 +835,8 @@ [ posting {paccount = "a", pamount = Mixed [usd 1 `at` eur 2]} , posting {paccount = "a", pamount = Mixed [usd (-2) `at` eur 1]} ]) - , test "balanceTransaction balances based on cost if there are total prices" $ - expectRight $ + ,test "balanceTransaction balances based on cost if there are total prices" $ + assertRight $ balanceTransaction Nothing (Transaction @@ -918,10 +854,9 @@ , posting {paccount = "a", pamount = Mixed [usd (-2) @@ eur 1]} ]) ] - , tests - "isTransactionBalanced" - [ test "detect balanced" $ - expect $ + , tests "isTransactionBalanced" [ + test "detect balanced" $ + assertBool "" $ isTransactionBalanced Nothing $ Transaction 0 @@ -937,8 +872,8 @@ [ posting {paccount = "b", pamount = Mixed [usd 1.00]} , posting {paccount = "c", pamount = Mixed [usd (-1.00)]} ] - , test "detect unbalanced" $ - expect $ + ,test "detect unbalanced" $ + assertBool "" $ not $ isTransactionBalanced Nothing $ Transaction @@ -955,8 +890,8 @@ [ posting {paccount = "b", pamount = Mixed [usd 1.00]} , posting {paccount = "c", pamount = Mixed [usd (-1.01)]} ] - , test "detect unbalanced, one posting" $ - expect $ + ,test "detect unbalanced, one posting" $ + assertBool "" $ not $ isTransactionBalanced Nothing $ Transaction @@ -971,8 +906,8 @@ "" [] [posting {paccount = "b", pamount = Mixed [usd 1.00]}] - , test "one zero posting is considered balanced for now" $ - expect $ + ,test "one zero posting is considered balanced for now" $ + assertBool "" $ isTransactionBalanced Nothing $ Transaction 0 @@ -986,8 +921,8 @@ "" [] [posting {paccount = "b", pamount = Mixed [usd 0]}] - , test "virtual postings don't need to balance" $ - expect $ + ,test "virtual postings don't need to balance" $ + assertBool "" $ isTransactionBalanced Nothing $ Transaction 0 @@ -1004,8 +939,8 @@ , posting {paccount = "c", pamount = Mixed [usd (-1.00)]} , posting {paccount = "d", pamount = Mixed [usd 100], ptype = VirtualPosting} ] - , test "balanced virtual postings need to balance among themselves" $ - expect $ + ,test "balanced virtual postings need to balance among themselves" $ + assertBool "" $ not $ isTransactionBalanced Nothing $ Transaction @@ -1023,8 +958,8 @@ , posting {paccount = "c", pamount = Mixed [usd (-1.00)]} , posting {paccount = "d", pamount = Mixed [usd 100], ptype = BalancedVirtualPosting} ] - , test "balanced virtual postings need to balance among themselves (2)" $ - expect $ + ,test "balanced virtual postings need to balance among themselves (2)" $ + assertBool "" $ isTransactionBalanced Nothing $ Transaction 0 diff --git a/Hledger/Data/TransactionModifier.hs b/Hledger/Data/TransactionModifier.hs --- a/Hledger/Data/TransactionModifier.hs +++ b/Hledger/Data/TransactionModifier.hs @@ -57,7 +57,7 @@ -- >>> putStr $ showTransaction $ transactionModifierToFunction (TransactionModifier "" ["pong" `post` usd 2]) nulltransaction{tpostings=["ping" `post` usd 1]} -- 0000/01/01 -- ping $1.00 --- pong $2.00 ; generated-posting:= +-- pong $2.00 ; generated-posting: = -- -- >>> putStr $ showTransaction $ transactionModifierToFunction (TransactionModifier "miss" ["pong" `post` usd 2]) nulltransaction{tpostings=["ping" `post` usd 1]} -- 0000/01/01 @@ -66,7 +66,7 @@ -- >>> putStr $ showTransaction $ transactionModifierToFunction (TransactionModifier "ping" ["pong" `post` amount{aismultiplier=True, aquantity=3}]) nulltransaction{tpostings=["ping" `post` usd 2]} -- 0000/01/01 -- ping $2.00 --- pong $6.00 ; generated-posting:= ping +-- pong $6.00 ; generated-posting: = ping -- -- transactionModifierToFunction :: TransactionModifier -> (Transaction -> Transaction) diff --git a/Hledger/Data/Valuation.hs b/Hledger/Data/Valuation.hs --- a/Hledger/Data/Valuation.hs +++ b/Hledger/Data/Valuation.hs @@ -48,11 +48,6 @@ import Hledger.Data.Dates (parsedate) -tests_Valuation = tests "Valuation" [ - tests_priceLookup - ] - - ------------------------------------------------------------------------------ -- Types @@ -278,12 +273,11 @@ ,p "2001/01/01" "A" 11 "B" ] pricesatdate = pricesAtDate ps1 - in tests "priceLookup" [ - priceLookup pricesatdate (d "1999/01/01") "A" Nothing `is` Nothing - ,priceLookup pricesatdate (d "2000/01/01") "A" Nothing `is` Just ("B",10) - ,priceLookup pricesatdate (d "2000/01/01") "B" (Just "A") `is` Just ("A",0.1) - ,priceLookup pricesatdate (d "2000/01/01") "A" (Just "E") `is` Just ("E",500) - ] + in test "priceLookup" $ do + priceLookup pricesatdate (d "1999/01/01") "A" Nothing @?= Nothing + priceLookup pricesatdate (d "2000/01/01") "A" Nothing @?= Just ("B",10) + priceLookup pricesatdate (d "2000/01/01") "B" (Just "A") @?= Just ("A",0.1) + priceLookup pricesatdate (d "2000/01/01") "A" (Just "E") @?= Just ("E",500) ------------------------------------------------------------------------------ -- Building the price graph (network of commodity conversions) on a given day. @@ -365,3 +359,7 @@ nodesEdgeLabel g (from,to) = headMay $ sort [l | (_,t,l) <- out g from, t==to] ------------------------------------------------------------------------------ + +tests_Valuation = tests "Valuation" [ + tests_priceLookup + ] diff --git a/Hledger/Query.hs b/Hledger/Query.hs --- a/Hledger/Query.hs +++ b/Hledger/Query.hs @@ -653,130 +653,122 @@ -- tests tests_Query = tests "Query" [ - tests "simplifyQuery" [ - - (simplifyQuery $ Or [Acct "a"]) `is` (Acct "a") - ,(simplifyQuery $ Or [Any,None]) `is` (Any) - ,(simplifyQuery $ And [Any,None]) `is` (None) - ,(simplifyQuery $ And [Any,Any]) `is` (Any) - ,(simplifyQuery $ And [Acct "b",Any]) `is` (Acct "b") - ,(simplifyQuery $ And [Any,And [Date (DateSpan Nothing Nothing)]]) `is` (Any) - ,(simplifyQuery $ And [Date (DateSpan Nothing (Just $ parsedate "2013-01-01")), Date (DateSpan (Just $ parsedate "2012-01-01") Nothing)]) - `is` (Date (DateSpan (Just $ parsedate "2012-01-01") (Just $ parsedate "2013-01-01"))) - ,(simplifyQuery $ And [Or [],Or [Desc "b b"]]) `is` (Desc "b b") - ] + test "simplifyQuery" $ do + (simplifyQuery $ Or [Acct "a"]) @?= (Acct "a") + (simplifyQuery $ Or [Any,None]) @?= (Any) + (simplifyQuery $ And [Any,None]) @?= (None) + (simplifyQuery $ And [Any,Any]) @?= (Any) + (simplifyQuery $ And [Acct "b",Any]) @?= (Acct "b") + (simplifyQuery $ And [Any,And [Date (DateSpan Nothing Nothing)]]) @?= (Any) + (simplifyQuery $ And [Date (DateSpan Nothing (Just $ parsedate "2013-01-01")), Date (DateSpan (Just $ parsedate "2012-01-01") Nothing)]) + @?= (Date (DateSpan (Just $ parsedate "2012-01-01") (Just $ parsedate "2013-01-01"))) + (simplifyQuery $ And [Or [],Or [Desc "b b"]]) @?= (Desc "b b") - ,tests "parseQuery" [ - (parseQuery nulldate "acct:'expenses:autres d\233penses' desc:b") `is` (And [Acct "expenses:autres d\233penses", Desc "b"], []) - ,parseQuery nulldate "inacct:a desc:\"b b\"" `is` (Desc "b b", [QueryOptInAcct "a"]) - ,parseQuery nulldate "inacct:a inacct:b" `is` (Any, [QueryOptInAcct "a", QueryOptInAcct "b"]) - ,parseQuery nulldate "desc:'x x'" `is` (Desc "x x", []) - ,parseQuery nulldate "'a a' 'b" `is` (Or [Acct "a a",Acct "'b"], []) - ,parseQuery nulldate "\"" `is` (Acct "\"", []) - ] + ,test "parseQuery" $ do + (parseQuery nulldate "acct:'expenses:autres d\233penses' desc:b") @?= (And [Acct "expenses:autres d\233penses", Desc "b"], []) + parseQuery nulldate "inacct:a desc:\"b b\"" @?= (Desc "b b", [QueryOptInAcct "a"]) + parseQuery nulldate "inacct:a inacct:b" @?= (Any, [QueryOptInAcct "a", QueryOptInAcct "b"]) + parseQuery nulldate "desc:'x x'" @?= (Desc "x x", []) + parseQuery nulldate "'a a' 'b" @?= (Or [Acct "a a",Acct "'b"], []) + parseQuery nulldate "\"" @?= (Acct "\"", []) - ,tests "words''" [ - (words'' [] "a b") `is` ["a","b"] - , (words'' [] "'a b'") `is` ["a b"] - , (words'' [] "not:a b") `is` ["not:a","b"] - , (words'' [] "not:'a b'") `is` ["not:a b"] - , (words'' [] "'not:a b'") `is` ["not:a b"] - , (words'' ["desc:"] "not:desc:'a b'") `is` ["not:desc:a b"] - , (words'' prefixes "\"acct:expenses:autres d\233penses\"") `is` ["acct:expenses:autres d\233penses"] - , (words'' prefixes "\"") `is` ["\""] - ] + ,test "words''" $ do + (words'' [] "a b") @?= ["a","b"] + (words'' [] "'a b'") @?= ["a b"] + (words'' [] "not:a b") @?= ["not:a","b"] + (words'' [] "not:'a b'") @?= ["not:a b"] + (words'' [] "'not:a b'") @?= ["not:a b"] + (words'' ["desc:"] "not:desc:'a b'") @?= ["not:desc:a b"] + (words'' prefixes "\"acct:expenses:autres d\233penses\"") @?= ["acct:expenses:autres d\233penses"] + (words'' prefixes "\"") @?= ["\""] - ,tests "filterQuery" [ - filterQuery queryIsDepth Any `is` Any - ,filterQuery queryIsDepth (Depth 1) `is` Depth 1 - ,filterQuery (not.queryIsDepth) (And [And [StatusQ Cleared,Depth 1]]) `is` StatusQ Cleared - ,filterQuery queryIsDepth (And [Date nulldatespan, Not (Or [Any, Depth 1])]) `is` Any -- XXX unclear - ] + ,test "filterQuery" $ do + filterQuery queryIsDepth Any @?= Any + filterQuery queryIsDepth (Depth 1) @?= Depth 1 + filterQuery (not.queryIsDepth) (And [And [StatusQ Cleared,Depth 1]]) @?= StatusQ Cleared + filterQuery queryIsDepth (And [Date nulldatespan, Not (Or [Any, Depth 1])]) @?= Any -- XXX unclear - ,tests "parseQueryTerm" [ - parseQueryTerm nulldate "a" `is` (Left $ Acct "a") - ,parseQueryTerm nulldate "acct:expenses:autres d\233penses" `is` (Left $ Acct "expenses:autres d\233penses") - ,parseQueryTerm nulldate "not:desc:a b" `is` (Left $ Not $ Desc "a b") - ,parseQueryTerm nulldate "status:1" `is` (Left $ StatusQ Cleared) - ,parseQueryTerm nulldate "status:*" `is` (Left $ StatusQ Cleared) - ,parseQueryTerm nulldate "status:!" `is` (Left $ StatusQ Pending) - ,parseQueryTerm nulldate "status:0" `is` (Left $ StatusQ Unmarked) - ,parseQueryTerm nulldate "status:" `is` (Left $ StatusQ Unmarked) - ,parseQueryTerm nulldate "payee:x" `is` (Left $ Tag "payee" (Just "x")) - ,parseQueryTerm nulldate "note:x" `is` (Left $ Tag "note" (Just "x")) - ,parseQueryTerm nulldate "real:1" `is` (Left $ Real True) - ,parseQueryTerm nulldate "date:2008" `is` (Left $ Date $ DateSpan (Just $ parsedate "2008/01/01") (Just $ parsedate "2009/01/01")) - ,parseQueryTerm nulldate "date:from 2012/5/17" `is` (Left $ Date $ DateSpan (Just $ parsedate "2012/05/17") Nothing) - ,parseQueryTerm nulldate "date:20180101-201804" `is` (Left $ Date $ DateSpan (Just $ parsedate "2018/01/01") (Just $ parsedate "2018/04/01")) - ,parseQueryTerm nulldate "inacct:a" `is` (Right $ QueryOptInAcct "a") - ,parseQueryTerm nulldate "tag:a" `is` (Left $ Tag "a" Nothing) - ,parseQueryTerm nulldate "tag:a=some value" `is` (Left $ Tag "a" (Just "some value")) - ,parseQueryTerm nulldate "amt:<0" `is` (Left $ Amt Lt 0) - ,parseQueryTerm nulldate "amt:>10000.10" `is` (Left $ Amt AbsGt 10000.1) - ] + ,test "parseQueryTerm" $ do + parseQueryTerm nulldate "a" @?= (Left $ Acct "a") + parseQueryTerm nulldate "acct:expenses:autres d\233penses" @?= (Left $ Acct "expenses:autres d\233penses") + parseQueryTerm nulldate "not:desc:a b" @?= (Left $ Not $ Desc "a b") + parseQueryTerm nulldate "status:1" @?= (Left $ StatusQ Cleared) + parseQueryTerm nulldate "status:*" @?= (Left $ StatusQ Cleared) + parseQueryTerm nulldate "status:!" @?= (Left $ StatusQ Pending) + parseQueryTerm nulldate "status:0" @?= (Left $ StatusQ Unmarked) + parseQueryTerm nulldate "status:" @?= (Left $ StatusQ Unmarked) + parseQueryTerm nulldate "payee:x" @?= (Left $ Tag "payee" (Just "x")) + parseQueryTerm nulldate "note:x" @?= (Left $ Tag "note" (Just "x")) + parseQueryTerm nulldate "real:1" @?= (Left $ Real True) + parseQueryTerm nulldate "date:2008" @?= (Left $ Date $ DateSpan (Just $ parsedate "2008/01/01") (Just $ parsedate "2009/01/01")) + parseQueryTerm nulldate "date:from 2012/5/17" @?= (Left $ Date $ DateSpan (Just $ parsedate "2012/05/17") Nothing) + parseQueryTerm nulldate "date:20180101-201804" @?= (Left $ Date $ DateSpan (Just $ parsedate "2018/01/01") (Just $ parsedate "2018/04/01")) + parseQueryTerm nulldate "inacct:a" @?= (Right $ QueryOptInAcct "a") + parseQueryTerm nulldate "tag:a" @?= (Left $ Tag "a" Nothing) + parseQueryTerm nulldate "tag:a=some value" @?= (Left $ Tag "a" (Just "some value")) + parseQueryTerm nulldate "amt:<0" @?= (Left $ Amt Lt 0) + parseQueryTerm nulldate "amt:>10000.10" @?= (Left $ Amt AbsGt 10000.1) - ,tests "parseAmountQueryTerm" [ - parseAmountQueryTerm "<0" `is` (Lt,0) -- special case for convenience, since AbsLt 0 would be always false - ,parseAmountQueryTerm ">0" `is` (Gt,0) -- special case for convenience and consistency with above - ,parseAmountQueryTerm ">10000.10" `is` (AbsGt,10000.1) - ,parseAmountQueryTerm "=0.23" `is` (AbsEq,0.23) - ,parseAmountQueryTerm "0.23" `is` (AbsEq,0.23) - ,parseAmountQueryTerm "<=+0.23" `is` (LtEq,0.23) - ,parseAmountQueryTerm "-0.23" `is` (Eq,(-0.23)) - ,_test "number beginning with decimal mark" $ parseAmountQueryTerm "=.23" `is` (AbsEq,0.23) -- XXX - ] + ,test "parseAmountQueryTerm" $ do + parseAmountQueryTerm "<0" @?= (Lt,0) -- special case for convenience, since AbsLt 0 would be always false + parseAmountQueryTerm ">0" @?= (Gt,0) -- special case for convenience and consistency with above + parseAmountQueryTerm ">10000.10" @?= (AbsGt,10000.1) + parseAmountQueryTerm "=0.23" @?= (AbsEq,0.23) + parseAmountQueryTerm "0.23" @?= (AbsEq,0.23) + parseAmountQueryTerm "<=+0.23" @?= (LtEq,0.23) + parseAmountQueryTerm "-0.23" @?= (Eq,(-0.23)) + -- ,test "number beginning with decimal mark" $ parseAmountQueryTerm "=.23" @?= (AbsEq,0.23) -- XXX - ,tests "matchesAccount" [ - expect $ (Acct "b:c") `matchesAccount` "a:bb:c:d" - ,expect $ not $ (Acct "^a:b") `matchesAccount` "c:a:b" - ,expect $ Depth 2 `matchesAccount` "a" - ,expect $ Depth 2 `matchesAccount` "a:b" - ,expect $ not $ Depth 2 `matchesAccount` "a:b:c" - ,expect $ Date nulldatespan `matchesAccount` "a" - ,expect $ Date2 nulldatespan `matchesAccount` "a" - ,expect $ not $ (Tag "a" Nothing) `matchesAccount` "a" - ] + ,test "matchesAccount" $ do + assertBool "" $ (Acct "b:c") `matchesAccount` "a:bb:c:d" + assertBool "" $ not $ (Acct "^a:b") `matchesAccount` "c:a:b" + assertBool "" $ Depth 2 `matchesAccount` "a" + assertBool "" $ Depth 2 `matchesAccount` "a:b" + assertBool "" $ not $ Depth 2 `matchesAccount` "a:b:c" + assertBool "" $ Date nulldatespan `matchesAccount` "a" + assertBool "" $ Date2 nulldatespan `matchesAccount` "a" + assertBool "" $ not $ (Tag "a" Nothing) `matchesAccount` "a" ,tests "matchesPosting" [ test "positive match on cleared posting status" $ - expect $ (StatusQ Cleared) `matchesPosting` nullposting{pstatus=Cleared} + assertBool "" $ (StatusQ Cleared) `matchesPosting` nullposting{pstatus=Cleared} ,test "negative match on cleared posting status" $ - expect $ not $ (Not $ StatusQ Cleared) `matchesPosting` nullposting{pstatus=Cleared} + assertBool "" $ not $ (Not $ StatusQ Cleared) `matchesPosting` nullposting{pstatus=Cleared} ,test "positive match on unmarked posting status" $ - expect $ (StatusQ Unmarked) `matchesPosting` nullposting{pstatus=Unmarked} + assertBool "" $ (StatusQ Unmarked) `matchesPosting` nullposting{pstatus=Unmarked} ,test "negative match on unmarked posting status" $ - expect $ not $ (Not $ StatusQ Unmarked) `matchesPosting` nullposting{pstatus=Unmarked} + assertBool "" $ not $ (Not $ StatusQ Unmarked) `matchesPosting` nullposting{pstatus=Unmarked} ,test "positive match on true posting status acquired from transaction" $ - expect $ (StatusQ Cleared) `matchesPosting` nullposting{pstatus=Unmarked,ptransaction=Just nulltransaction{tstatus=Cleared}} - ,test "real:1 on real posting" $ expect $ (Real True) `matchesPosting` nullposting{ptype=RegularPosting} - ,test "real:1 on virtual posting fails" $ expect $ not $ (Real True) `matchesPosting` nullposting{ptype=VirtualPosting} - ,test "real:1 on balanced virtual posting fails" $ expect $ not $ (Real True) `matchesPosting` nullposting{ptype=BalancedVirtualPosting} - ,test "a" $ expect $ (Acct "'b") `matchesPosting` nullposting{paccount="'b"} - ,test "b" $ expect $ not $ (Tag "a" (Just "r$")) `matchesPosting` nullposting - ,test "c" $ expect $ (Tag "foo" Nothing) `matchesPosting` nullposting{ptags=[("foo","")]} - ,test "d" $ expect $ (Tag "foo" Nothing) `matchesPosting` nullposting{ptags=[("foo","baz")]} - ,test "e" $ expect $ (Tag "foo" (Just "a")) `matchesPosting` nullposting{ptags=[("foo","bar")]} - ,test "f" $ expect $ not $ (Tag "foo" (Just "a$")) `matchesPosting` nullposting{ptags=[("foo","bar")]} - ,test "g" $ expect $ not $ (Tag " foo " (Just "a")) `matchesPosting` nullposting{ptags=[("foo","bar")]} - ,test "h" $ expect $ not $ (Tag "foo foo" (Just " ar ba ")) `matchesPosting` nullposting{ptags=[("foo foo","bar bar")]} - -- a tag match on a posting also sees inherited tags - ,test "i" $ expect $ (Tag "txntag" Nothing) `matchesPosting` nullposting{ptransaction=Just nulltransaction{ttags=[("txntag","")]}} - ,test "j" $ expect $ not $ (Sym "$") `matchesPosting` nullposting{pamount=Mixed [usd 1]} -- becomes "^$$", ie testing for null symbol - ,test "k" $ expect $ (Sym "\\$") `matchesPosting` nullposting{pamount=Mixed [usd 1]} -- have to quote $ for regexpr - ,test "l" $ expect $ (Sym "shekels") `matchesPosting` nullposting{pamount=Mixed [nullamt{acommodity="shekels"}]} - ,test "m" $ expect $ not $ (Sym "shek") `matchesPosting` nullposting{pamount=Mixed [nullamt{acommodity="shekels"}]} + assertBool "" $ (StatusQ Cleared) `matchesPosting` nullposting{pstatus=Unmarked,ptransaction=Just nulltransaction{tstatus=Cleared}} + ,test "real:1 on real posting" $ assertBool "" $ (Real True) `matchesPosting` nullposting{ptype=RegularPosting} + ,test "real:1 on virtual posting fails" $ assertBool "" $ not $ (Real True) `matchesPosting` nullposting{ptype=VirtualPosting} + ,test "real:1 on balanced virtual posting fails" $ assertBool "" $ not $ (Real True) `matchesPosting` nullposting{ptype=BalancedVirtualPosting} + ,test "acct:" $ assertBool "" $ (Acct "'b") `matchesPosting` nullposting{paccount="'b"} + ,test "tag:" $ do + assertBool "" $ not $ (Tag "a" (Just "r$")) `matchesPosting` nullposting + assertBool "" $ (Tag "foo" Nothing) `matchesPosting` nullposting{ptags=[("foo","")]} + assertBool "" $ (Tag "foo" Nothing) `matchesPosting` nullposting{ptags=[("foo","baz")]} + assertBool "" $ (Tag "foo" (Just "a")) `matchesPosting` nullposting{ptags=[("foo","bar")]} + assertBool "" $ not $ (Tag "foo" (Just "a$")) `matchesPosting` nullposting{ptags=[("foo","bar")]} + assertBool "" $ not $ (Tag " foo " (Just "a")) `matchesPosting` nullposting{ptags=[("foo","bar")]} + assertBool "" $ not $ (Tag "foo foo" (Just " ar ba ")) `matchesPosting` nullposting{ptags=[("foo foo","bar bar")]} + ,test "a tag match on a posting also sees inherited tags" $ assertBool "" $ (Tag "txntag" Nothing) `matchesPosting` nullposting{ptransaction=Just nulltransaction{ttags=[("txntag","")]}} + ,test "cur:" $ do + assertBool "" $ not $ (Sym "$") `matchesPosting` nullposting{pamount=Mixed [usd 1]} -- becomes "^$$", ie testing for null symbol + assertBool "" $ (Sym "\\$") `matchesPosting` nullposting{pamount=Mixed [usd 1]} -- have to quote $ for regexpr + assertBool "" $ (Sym "shekels") `matchesPosting` nullposting{pamount=Mixed [nullamt{acommodity="shekels"}]} + assertBool "" $ not $ (Sym "shek") `matchesPosting` nullposting{pamount=Mixed [nullamt{acommodity="shekels"}]} ] - ,tests "matchesTransaction" [ - expect $ Any `matchesTransaction` nulltransaction - ,expect $ not $ (Desc "x x") `matchesTransaction` nulltransaction{tdescription="x"} - ,expect $ (Desc "x x") `matchesTransaction` nulltransaction{tdescription="x x"} + ,test "matchesTransaction" $ do + assertBool "" $ Any `matchesTransaction` nulltransaction + assertBool "" $ not $ (Desc "x x") `matchesTransaction` nulltransaction{tdescription="x"} + assertBool "" $ (Desc "x x") `matchesTransaction` nulltransaction{tdescription="x x"} -- see posting for more tag tests - ,expect $ (Tag "foo" (Just "a")) `matchesTransaction` nulltransaction{ttags=[("foo","bar")]} - ,expect $ (Tag "payee" (Just "payee")) `matchesTransaction` nulltransaction{tdescription="payee|note"} - ,expect $ (Tag "note" (Just "note")) `matchesTransaction` nulltransaction{tdescription="payee|note"} + assertBool "" $ (Tag "foo" (Just "a")) `matchesTransaction` nulltransaction{ttags=[("foo","bar")]} + assertBool "" $ (Tag "payee" (Just "payee")) `matchesTransaction` nulltransaction{tdescription="payee|note"} + assertBool "" $ (Tag "note" (Just "note")) `matchesTransaction` nulltransaction{tdescription="payee|note"} -- a tag match on a transaction also matches posting tags - ,expect $ (Tag "postingtag" Nothing) `matchesTransaction` nulltransaction{tpostings=[nullposting{ptags=[("postingtag","")]}]} - ] + assertBool "" $ (Tag "postingtag" Nothing) `matchesTransaction` nulltransaction{tpostings=[nullposting{ptags=[("postingtag","")]}]} ] diff --git a/Hledger/Read.hs b/Hledger/Read.hs --- a/Hledger/Read.hs +++ b/Hledger/Read.hs @@ -307,6 +307,8 @@ tryReaders :: InputOpts -> Maybe FilePath -> [Reader] -> Text -> IO (Either String Journal) tryReaders iopts mpath readers txt = firstSuccessOrFirstError [] readers where + -- TODO: #1087 when parsing csv with -f -, if the csv (rules) parser fails, + -- we would rather see that error, not the one from the journal parser firstSuccessOrFirstError :: [String] -> [Reader] -> IO (Either String Journal) firstSuccessOrFirstError [] [] = return $ Left "no readers found" firstSuccessOrFirstError errs (r:rs) = do diff --git a/Hledger/Read/Common.hs b/Hledger/Read/Common.hs --- a/Hledger/Read/Common.hs +++ b/Hledger/Read/Common.hs @@ -35,6 +35,7 @@ journalSourcePos, parseAndFinaliseJournal, parseAndFinaliseJournal', + finaliseJournal, setYear, getYear, setDefaultCommodityAndStyle, @@ -99,11 +100,9 @@ ) where --- * imports -import Prelude () -import "base-compat-batteries" Prelude.Compat hiding (readFile) -import "base-compat-batteries" Control.Monad.Compat +import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail (fail) import Control.Monad.Except (ExceptT(..), runExceptT, throwError) -import Control.Monad.State.Strict +import Control.Monad.State.Strict hiding (fail) import Data.Bifunctor (bimap, second) import Data.Char import Data.Data @@ -178,11 +177,11 @@ rawOptsToInputOpts :: RawOpts -> InputOpts rawOptsToInputOpts rawopts = InputOpts{ - -- files_ = map (T.unpack . stripquotes . T.pack) $ listofstringopt "file" rawopts + -- files_ = listofstringopt "file" rawopts mformat_ = Nothing ,mrules_file_ = maybestringopt "rules-file" rawopts ,separator_ = fromMaybe ',' (maybecharopt "separator" rawopts) - ,aliases_ = map (T.unpack . stripquotes . T.pack) $ listofstringopt "alias" rawopts + ,aliases_ = listofstringopt "alias" rawopts ,anon_ = boolopt "anon" rawopts ,ignore_assertions_ = boolopt "ignore-assertions" rawopts ,new_ = boolopt "new" rawopts @@ -228,101 +227,94 @@ -- | Given a megaparsec ParsedJournal parser, input options, file --- path and file content: parse and post-process a Journal, or give an error. +-- path and file content: parse and finalise a Journal, or give an error. parseAndFinaliseJournal :: ErroringJournalParser IO ParsedJournal -> InputOpts -> FilePath -> Text -> ExceptT String IO Journal parseAndFinaliseJournal parser iopts f txt = do - t <- liftIO getClockTime y <- liftIO getCurrentYear - let initJournal = nulljournal - { jparsedefaultyear = Just y - , jincludefilestack = [f] } - eep <- liftIO $ runExceptT $ - runParserT (evalStateT parser initJournal) f txt + let initJournal = nulljournal{ jparsedefaultyear = Just y, jincludefilestack = [f] } + eep <- liftIO $ runExceptT $ runParserT (evalStateT parser initJournal) f txt + -- TODO: urgh.. clean this up somehow case eep of - Left finalParseError -> - throwError $ finalErrorBundlePretty $ attachSource f txt finalParseError - + Left finalParseError -> throwError $ finalErrorBundlePretty $ attachSource f txt finalParseError Right ep -> case ep of - Left e -> throwError $ customErrorBundlePretty e - - Right pj -> - -- If we are using automated transactions, we finalize twice: - -- once before and once after. However, if we are running it - -- twice, we don't check assertions the first time (they might - -- be false pending modifiers) and we don't reorder the second - -- time. If we are only running once, we reorder and follow - -- the options for checking assertions. - let fj = if auto_ iopts && (not . null . jtxnmodifiers) pj - - -- transaction modifiers are active - then - -- first pass, doing most of the work - ( - (journalModifyTransactions <$>) $ -- add auto postings after balancing ? #893b fails - journalBalanceTransactions False $ - -- journalModifyTransactions <$> -- add auto postings before balancing ? probably #893a, #928, #938 fail - journalReverse $ - journalAddFile (f, txt) $ - journalApplyCommodityStyles pj) - -- second pass, checking balance assertions - >>= (\j -> - journalBalanceTransactions (not $ ignore_assertions_ iopts) $ - journalSetLastReadTime t $ - j) - - -- transaction modifiers are not active - else journalBalanceTransactions (not $ ignore_assertions_ iopts) $ - journalReverse $ - journalAddFile (f, txt) $ - journalApplyCommodityStyles $ - journalSetLastReadTime t $ - pj - in - case fj of - Right j -> return j - Left e -> throwError e + Left e -> throwError $ customErrorBundlePretty e + Right pj -> finaliseJournal iopts f txt pj --- Like parseAndFinaliseJournal but takes a (non-Erroring) JournalParser. --- Used for timeclock/timedot. XXX let them use parseAndFinaliseJournal instead +-- | Like parseAndFinaliseJournal but takes a (non-Erroring) JournalParser. +-- Used for timeclock/timedot. +-- TODO: get rid of this, use parseAndFinaliseJournal instead parseAndFinaliseJournal' :: JournalParser IO ParsedJournal -> InputOpts -> FilePath -> Text -> ExceptT String IO Journal parseAndFinaliseJournal' parser iopts f txt = do - t <- liftIO getClockTime y <- liftIO getCurrentYear let initJournal = nulljournal { jparsedefaultyear = Just y , jincludefilestack = [f] } ep <- liftIO $ runParserT (evalStateT parser initJournal) f txt + -- see notes above case ep of Left e -> throwError $ customErrorBundlePretty e + Right pj -> finaliseJournal iopts f txt pj - Right pj -> - -- If we are using automated transactions, we finalize twice: - -- once before and once after. However, if we are running it - -- twice, we don't check assertions the first time (they might - -- be false pending modifiers) and we don't reorder the second - -- time. If we are only running once, we reorder and follow the - -- options for checking assertions. - let fj = if auto_ iopts && (not . null . jtxnmodifiers) pj - then journalModifyTransactions <$> - (journalBalanceTransactions False $ - journalReverse $ - journalApplyCommodityStyles pj) >>= - (\j -> journalBalanceTransactions (not $ ignore_assertions_ iopts) $ - journalAddFile (f, txt) $ - journalSetLastReadTime t $ - j) - else journalBalanceTransactions (not $ ignore_assertions_ iopts) $ - journalReverse $ - journalAddFile (f, txt) $ - journalApplyCommodityStyles $ - journalSetLastReadTime t $ - pj +-- | Post-process a Journal that has just been parsed or generated, in this order: +-- +-- - apply canonical amount styles, +-- +-- - save misc info and reverse transactions into their original parse order, +-- +-- - evaluate balance assignments and balance each transaction, +-- +-- - apply transaction modifiers (auto postings) if enabled, +-- +-- - check balance assertions if enabled. +-- +finaliseJournal :: InputOpts -> FilePath -> Text -> Journal -> ExceptT String IO Journal +finaliseJournal iopts f txt pj = do + t <- liftIO getClockTime + -- Infer and apply canonical styles for each commodity (or fail). + -- TODO: since #903's refactoring for hledger 1.12, + -- journalApplyCommodityStyles here is seeing the + -- transactions before they get reversesd to normal order. + case journalApplyCommodityStyles pj of + Left e -> throwError e + Right pj' -> + -- Finalise the parsed journal. + let fj = + if auto_ iopts && (not . null . jtxnmodifiers) pj + then + -- When automatic postings are active, we finalise twice: + -- once before and once after. However, if we are running it + -- twice, we don't check assertions the first time (they might + -- be false pending modifiers) and we don't reorder the second + -- time. If we are only running once, we reorder and follow + -- the options for checking assertions. + -- + -- first pass, doing most of the work + ( + (journalModifyTransactions <$>) $ -- add auto postings after balancing ? #893b fails + journalBalanceTransactions False $ -- balance transactions without checking assertions + -- journalModifyTransactions <$> -- add auto postings before balancing ? probably #893a, #928, #938 fail + journalReverse $ + journalSetLastReadTime t $ + journalAddFile (f, txt) $ + pj') + -- balance transactions a second time, now just checking balance assertions + >>= (\j -> + journalBalanceTransactions (not $ ignore_assertions_ iopts) $ + j) + + else + -- automatic postings are not active + journalBalanceTransactions (not $ ignore_assertions_ iopts) $ + journalReverse $ + journalSetLastReadTime t $ + journalAddFile (f, txt) $ + pj' in case fj of - Right j -> return j Left e -> throwError e + Right j -> return j setYear :: Year -> JournalParser m () setYear y = modify' (\j -> j{jparsedefaultyear=Just y}) @@ -781,7 +773,7 @@ dbg8 "numberp suggestedStyle" suggestedStyle `seq` return () case dbg8 "numberp quantity,precision,mdecimalpoint,mgrps" $ fromRawNumber rawNum mExp of - Left errMsg -> fail errMsg + Left errMsg -> Fail.fail errMsg Right (q, p, d, g) -> pure (sign q, p, d, g) exponentp :: TextParser m Int @@ -860,14 +852,14 @@ -- | Parse and interpret the structure of a number without external hints. -- Numbers are digit strings, possibly separated into digit groups by one -- of two types of separators. (1) Numbers may optionally have a decimal --- point, which may be either a period or comma. (2) Numbers may --- optionally contain digit group separators, which must all be either a +-- mark, which may be either a period or comma. (2) Numbers may +-- optionally contain digit group marks, which must all be either a -- period, a comma, or a space. -- --- It is our task to deduce the identities of the decimal point and digit --- separator characters, based on the allowed syntax. For instance, we --- make use of the fact that a decimal point can occur at most once and --- must succeed all digit group separators. +-- It is our task to deduce the characters used as decimal mark and +-- digit group mark, based on the allowed syntax. For instance, we +-- make use of the fact that a decimal mark can occur at most once and +-- must be to the right of all digit group marks. -- -- >>> parseTest rawnumberp "1,234,567.89" -- Right (WithSeparators ',' ["1","234","567"] (Just ('.',"89"))) @@ -883,7 +875,7 @@ -- Guard against mistyped numbers mExtraDecimalSep <- optional $ lookAhead $ satisfy isDecimalPointChar when (isJust mExtraDecimalSep) $ - fail "invalid number (invalid use of separator)" + Fail.fail "invalid number (invalid use of separator)" mExtraFragment <- optional $ lookAhead $ try $ char ' ' *> getOffset <* digitChar @@ -946,12 +938,30 @@ isDigitSeparatorChar :: Char -> Bool isDigitSeparatorChar c = isDecimalPointChar c || c == ' ' +-- | Some kinds of number literal we might parse. +data RawNumber + = NoSeparators DigitGrp (Maybe (Char, DigitGrp)) + -- ^ A number with no digit group marks (eg 100), + -- or with a leading or trailing comma or period + -- which (apparently) we interpret as a decimal mark (like 100. or .100) + | WithSeparators Char [DigitGrp] (Maybe (Char, DigitGrp)) + -- ^ A number with identifiable digit group marks + -- (eg 1,000,000 or 1,000.50 or 1 000) + deriving (Show, Eq) +-- | Another kind of number literal: this one contains either a digit +-- group separator or a decimal mark, we're not sure which (eg 1,000 or 100.50). +data AmbiguousNumber = AmbiguousNumber DigitGrp Char DigitGrp + deriving (Show, Eq) + +-- | Description of a single digit group in a number literal. +-- "Thousands" is one well known digit grouping, but there are others. data DigitGrp = DigitGrp { - digitGroupLength :: !Int, - digitGroupNumber :: !Integer + digitGroupLength :: !Int, -- ^ The number of digits in this group. + digitGroupNumber :: !Integer -- ^ The natural number formed by this group's digits. } deriving (Eq) +-- | A custom show instance, showing digit groups as the parser saw them. instance Show DigitGrp where show (DigitGrp len num) | len > 0 = "\"" ++ padding ++ numStr ++ "\"" @@ -973,14 +983,6 @@ makeGroup = uncurry DigitGrp . foldl' step (0, 0) . T.unpack step (!l, !a) c = (l+1, a*10 + fromIntegral (digitToInt c)) -data RawNumber - = NoSeparators DigitGrp (Maybe (Char, DigitGrp)) -- 100 or 100. or .100 or 100.50 - | WithSeparators Char [DigitGrp] (Maybe (Char, DigitGrp)) -- 1,000,000 or 1,000.50 - deriving (Show, Eq) - -data AmbiguousNumber = AmbiguousNumber DigitGrp Char DigitGrp -- 1,000 - deriving (Show, Eq) - --- ** comments multilinecommentp :: TextParser m () @@ -1016,7 +1018,7 @@ -- Several journal items may be followed by comments, which begin with -- semicolons and extend to the end of the line. Such comments may span -- multiple lines, but comment lines below the journal item must be --- preceeded by leading whitespace. +-- preceded by leading whitespace. -- -- This parser combinator accepts a parser that consumes all input up -- until the next newline. This parser should extract the "content" from @@ -1073,7 +1075,7 @@ -- The first line of a transaction may be followed by comments, which -- begin with semicolons and extend to the end of the line. Transaction -- comments may span multiple lines, but comment lines below the --- transaction must be preceeded by leading whitespace. +-- transaction must be preceded by leading whitespace. -- -- 2000/1/1 ; a transaction comment starting on the same line ... -- ; extending to the next line @@ -1127,7 +1129,7 @@ -- -- Postings may be followed by comments, which begin with semicolons and -- extend to the end of the line. Posting comments may span multiple --- lines, but comment lines below the posting must be preceeded by +-- lines, but comment lines below the posting must be preceded by -- leading whitespace. -- -- 2000/1/1 @@ -1273,7 +1275,7 @@ $ between (char '[') (char ']') $ takeWhile1P Nothing isBracketedDateChar unless (T.any isDigit s && T.any isDateSepChar s) $ - fail "not a bracketed date" + Fail.fail "not a bracketed date" -- Looks sufficiently like a bracketed date to commit to parsing a date between (char '[') (char ']') $ do @@ -1304,9 +1306,9 @@ tests_Common = tests "Common" [ tests "amountp" [ - test "basic" $ expectParseEq amountp "$47.18" (usd 47.18) - ,test "ends with decimal mark" $ expectParseEq amountp "$1." (usd 1 `withPrecision` 0) - ,test "unit price" $ expectParseEq amountp "$10 @ €0.5" + test "basic" $ assertParseEq amountp "$47.18" (usd 47.18) + ,test "ends with decimal mark" $ assertParseEq amountp "$1." (usd 1 `withPrecision` 0) + ,test "unit price" $ assertParseEq amountp "$10 @ €0.5" -- not precise enough: -- (usd 10 `withPrecision` 0 `at` (eur 0.5 `withPrecision` 1)) -- `withStyle` asdecimalpoint=Just '.' amount{ @@ -1320,7 +1322,7 @@ ,astyle=amountstyle{asprecision=1, asdecimalpoint=Just '.'} } } - ,test "total price" $ expectParseEq amountp "$10 @@ €5" + ,test "total price" $ assertParseEq amountp "$10 @@ €5" amount{ acommodity="$" ,aquantity=10 @@ -1335,32 +1337,31 @@ ] ,let p = lift (numberp Nothing) :: JournalParser IO (Quantity, Int, Maybe Char, Maybe DigitGroupStyle) in - tests "numberp" [ - test "." $ expectParseEq p "0" (0, 0, Nothing, Nothing) - ,test "." $ expectParseEq p "1" (1, 0, Nothing, Nothing) - ,test "." $ expectParseEq p "1.1" (1.1, 1, Just '.', Nothing) - ,test "." $ expectParseEq p "1,000.1" (1000.1, 1, Just '.', Just $ DigitGroups ',' [3]) - ,test "." $ expectParseEq p "1.00.000,1" (100000.1, 1, Just ',', Just $ DigitGroups '.' [3,2]) - ,test "." $ expectParseEq p "1,000,000" (1000000, 0, Nothing, Just $ DigitGroups ',' [3,3]) -- could be simplified to [3] - ,test "." $ expectParseEq p "1." (1, 0, Just '.', Nothing) - ,test "." $ expectParseEq p "1," (1, 0, Just ',', Nothing) - ,test "." $ expectParseEq p ".1" (0.1, 1, Just '.', Nothing) - ,test "." $ expectParseEq p ",1" (0.1, 1, Just ',', Nothing) - ,test "." $ expectParseError p "" "" - ,test "." $ expectParseError p "1,000.000,1" "" - ,test "." $ expectParseError p "1.000,000.1" "" - ,test "." $ expectParseError p "1,000.000.1" "" - ,test "." $ expectParseError p "1,,1" "" - ,test "." $ expectParseError p "1..1" "" - ,test "." $ expectParseError p ".1," "" - ,test "." $ expectParseError p ",1." "" - ] + test "numberp" $ do + assertParseEq p "0" (0, 0, Nothing, Nothing) + assertParseEq p "1" (1, 0, Nothing, Nothing) + assertParseEq p "1.1" (1.1, 1, Just '.', Nothing) + assertParseEq p "1,000.1" (1000.1, 1, Just '.', Just $ DigitGroups ',' [3]) + assertParseEq p "1.00.000,1" (100000.1, 1, Just ',', Just $ DigitGroups '.' [3,2]) + assertParseEq p "1,000,000" (1000000, 0, Nothing, Just $ DigitGroups ',' [3,3]) -- could be simplified to [3] + assertParseEq p "1." (1, 0, Just '.', Nothing) + assertParseEq p "1," (1, 0, Just ',', Nothing) + assertParseEq p ".1" (0.1, 1, Just '.', Nothing) + assertParseEq p ",1" (0.1, 1, Just ',', Nothing) + assertParseError p "" "" + assertParseError p "1,000.000,1" "" + assertParseError p "1.000,000.1" "" + assertParseError p "1,000.000.1" "" + assertParseError p "1,,1" "" + assertParseError p "1..1" "" + assertParseError p ".1," "" + assertParseError p ",1." "" ,tests "spaceandamountormissingp" [ - test "space and amount" $ expectParseEq spaceandamountormissingp " $47.18" (Mixed [usd 47.18]) - ,test "empty string" $ expectParseEq spaceandamountormissingp "" missingmixedamt - ,_test "just space" $ expectParseEq spaceandamountormissingp " " missingmixedamt -- XXX should it ? - -- ,test "just amount" $ expectParseError spaceandamountormissingp "$47.18" "" -- succeeds, consuming nothing + test "space and amount" $ assertParseEq spaceandamountormissingp " $47.18" (Mixed [usd 47.18]) + ,test "empty string" $ assertParseEq spaceandamountormissingp "" missingmixedamt + -- ,test "just space" $ assertParseEq spaceandamountormissingp " " missingmixedamt -- XXX should it ? + -- ,test "just amount" $ assertParseError spaceandamountormissingp "$47.18" "" -- succeeds, consuming nothing ] ] diff --git a/Hledger/Read/CsvReader.hs b/Hledger/Read/CsvReader.hs --- a/Hledger/Read/CsvReader.hs +++ b/Hledger/Read/CsvReader.hs @@ -30,13 +30,15 @@ tests_CsvReader, ) where -import Prelude () -import "base-compat-batteries" Prelude.Compat -import Control.Exception hiding (try) -import Control.Monad -import Control.Monad.Except +import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail (fail) +import Control.Exception (IOException, handle, throw) +import Control.Monad (liftM, unless, when) +import Control.Monad.Except (ExceptT, throwError) +import Control.Monad.IO.Class (liftIO) import Control.Monad.State.Strict (StateT, get, modify', evalStateT) -import Data.Char (toLower, isDigit, isSpace, ord) +import Control.Monad.Trans.Class (lift) +import Data.Char (toLower, isDigit, isSpace, ord) +import Data.Bifunctor (first) import "base-compat-batteries" Data.List.Compat import Data.Maybe import Data.Ord @@ -67,7 +69,7 @@ import Hledger.Data import Hledger.Utils -import Hledger.Read.Common (Reader(..),InputOpts(..),amountp, statusp, genericSourcePos) +import Hledger.Read.Common (Reader(..),InputOpts(..),amountp, statusp, genericSourcePos, finaliseJournal) type CSV = [Record] @@ -84,15 +86,21 @@ } -- | Parse and post-process a "Journal" from CSV data, or give an error. --- XXX currently ignores the string and reads from the file path +-- Does not check balance assertions. +-- XXX currently ignores the provided data, reads it from the file path instead. parse :: InputOpts -> FilePath -> Text -> ExceptT String IO Journal parse iopts f t = do let rulesfile = mrules_file_ iopts let separator = separator_ iopts r <- liftIO $ readJournalFromCsv separator rulesfile f t - case r of Left e -> throwError e - Right j -> return $ journalNumberAndTieTransactions j --- XXX does not use parseAndFinaliseJournal like the other readers + case r of Left e -> throwError e + Right pj -> finaliseJournal iopts{ignore_assertions_=True} f t pj' + where + -- finaliseJournal assumes the journal's items are + -- reversed, as produced by JournalReader's parser. + -- But here they are already properly ordered. So we'd + -- better preemptively reverse them once more. XXX inefficient + pj' = journalReverse pj -- | Read a Journal from the given CSV data (and filename, used for error -- messages), or return an error. Proceed as follows: @@ -108,32 +116,35 @@ readJournalFromCsv :: Char -> Maybe FilePath -> FilePath -> Text -> IO (Either String Journal) readJournalFromCsv _ Nothing "-" _ = return $ Left "please use --rules-file when reading CSV from stdin" readJournalFromCsv separator mrulesfile csvfile csvdata = - handle (\e -> return $ Left $ show (e :: IOException)) $ do + handle (\(e::IOException) -> return $ Left $ show e) $ do + + -- make and throw an IO exception.. which we catch and convert to an Either above ? let throwerr = throw . userError - -- parse rules + -- parse the csv rules let rulesfile = fromMaybe (rulesFileFor csvfile) mrulesfile rulesfileexists <- doesFileExist rulesfile rulestext <- if rulesfileexists then do dbg1IO "using conversion rules file" rulesfile - liftIO $ (readFilePortably rulesfile >>= expandIncludes (takeDirectory rulesfile)) - else return $ defaultRulesText rulesfile - rules <- liftIO (runExceptT $ parseAndValidateCsvRules rulesfile rulestext) >>= either throwerr return + readFilePortably rulesfile >>= expandIncludes (takeDirectory rulesfile) + else + return $ defaultRulesText rulesfile + rules <- either throwerr return $ parseAndValidateCsvRules rulesfile rulestext dbg2IO "rules" rules - -- apply skip directive - let skip = maybe 0 oneorerror $ getDirective "skip" rules - where - oneorerror "" = 1 - oneorerror s = readDef (throwerr $ "could not parse skip value: " ++ show s) s + -- parse the skip directive's value, if any + let skiplines = case getDirective "skip" rules of + Nothing -> 0 + Just "" -> 1 + Just s -> readDef (throwerr $ "could not parse skip value: " ++ show s) s -- parse csv - -- parsec seems to fail if you pass it "-" here XXX try again with megaparsec + -- parsec seems to fail if you pass it "-" here TODO: try again with megaparsec let parsecfilename = if csvfile == "-" then "(stdin)" else csvfile records <- (either throwerr id . - dbg2 "validateCsv" . validateCsv skip . + dbg2 "validateCsv" . validateCsv rules skiplines . dbg2 "parseCsv") `fmap` parseCsv separator parsecfilename csvdata dbg1IO "first 3 csv records" $ take 3 records @@ -156,10 +167,10 @@ (initialPos parsecfilename) records -- Ensure transactions are ordered chronologically. - -- First, reverse them to get same-date transactions ordered chronologically, - -- if the CSV records seem to be most-recent-first, ie if there's an explicit - -- "newest-first" directive, or if there's more than one date and the first date - -- is more recent than the last. + -- First, if the CSV records seem to be most-recent-first (because + -- there's an explicit "newest-first" directive, or there's more + -- than one date and the first date is more recent than the last): + -- reverse them to get same-date transactions ordered chronologically. txns' = (if newestfirst || mseemsnewestfirst == Just True then reverse else id) txns where @@ -209,20 +220,29 @@ unlined = concat . intersperse "\n" -- | Return the cleaned up and validated CSV data (can be empty), or an error. -validateCsv :: Int -> Either String CSV -> Either String [CsvRecord] -validateCsv _ (Left err) = Left err -validateCsv numhdrlines (Right rs) = validate $ drop numhdrlines $ filternulls rs +validateCsv :: CsvRules -> Int -> Either String CSV -> Either String [CsvRecord] +validateCsv _ _ (Left err) = Left err +validateCsv rules numhdrlines (Right rs) = validate $ applyConditionalSkips $ drop numhdrlines $ filternulls rs where filternulls = filter (/=[""]) + skipCount r = + case (getEffectiveAssignment rules r "end", getEffectiveAssignment rules r "skip") of + (Nothing, Nothing) -> Nothing + (Just _, _) -> Just maxBound + (Nothing, Just "") -> Just 1 + (Nothing, Just x) -> Just (read x) + applyConditionalSkips [] = [] + applyConditionalSkips (r:rest) = + case skipCount r of + Nothing -> r:(applyConditionalSkips rest) + Just cnt -> applyConditionalSkips (drop (cnt-1) rest) validate [] = Right [] - validate rs@(first:_) - | isJust lessthan2 = let r = fromJust lessthan2 in Left $ printf "CSV record %s has less than two fields" (show r) - | isJust different = let r = fromJust different in Left $ printf "the first CSV record %s has %d fields but %s has %d" (show first) length1 (show r) (length r) + validate rs@(_first:_) + | isJust lessthan2 = let r = fromJust lessthan2 in + Left $ printf "CSV record %s has less than two fields" (show r) | otherwise = Right rs where - length1 = length first lessthan2 = headMay $ filter ((<2).length) rs - different = headMay $ filter ((/=length1).length) rs -- -- | The highest (0-based) field index referenced in the field -- -- definitions, or -1 if no fields are defined. @@ -256,7 +276,7 @@ ,"" ,"account1 assets:bank:checking" ,"" - ,"fields date, description, amount" + ,"fields date, description, amount1" ,"" ,"#skip 1" ,"#newest-first" @@ -364,7 +384,7 @@ type DateFormat = String type RegexpPattern = String -rules = CsvRules { +defrules = CsvRules { rdirectives=[], rcsvfieldindexes=[], rassignments=[], @@ -399,16 +419,19 @@ instance ShowErrorComponent String where showErrorComponent = id --- | An error-throwing action that parses this file's content +-- Not used by hledger; just for lib users, +-- | An pure-exception-throwing IO action that parses this file's content -- as CSV conversion rules, interpolating any included files first, -- and runs some extra validation checks. parseRulesFile :: FilePath -> ExceptT String IO CsvRules parseRulesFile f = - liftIO (readFilePortably f >>= expandIncludes (takeDirectory f)) >>= parseAndValidateCsvRules f + liftIO (readFilePortably f >>= expandIncludes (takeDirectory f)) + >>= either throwError return . parseAndValidateCsvRules f -- | Inline all files referenced by include directives in this hledger CSV rules text, recursively. -- Included file paths may be relative to the directory of the provided file path. --- This is a cheap hack to avoid rewriting the CSV rules parser. +-- This is done as a pre-parse step to simplify the CSV rules parser. +expandIncludes :: FilePath -> Text -> IO Text expandIncludes dir content = mapM (expandLine dir) (T.lines content) >>= return . T.unlines where expandLine dir line = @@ -419,47 +442,30 @@ dir' = takeDirectory f' _ -> return line --- | An error-throwing action that parses this text as CSV conversion rules --- and runs some extra validation checks. The file path is for error messages. -parseAndValidateCsvRules :: FilePath -> T.Text -> ExceptT String IO CsvRules -parseAndValidateCsvRules rulesfile s = do - let rules = parseCsvRules rulesfile s - case rules of - Left e -> ExceptT $ return $ Left $ customErrorBundlePretty e - Right r -> do - r_ <- liftIO $ runExceptT $ validateRules r - ExceptT $ case r_ of - Left s -> return $ Left $ parseErrorPretty $ makeParseError s - Right r -> return $ Right r - +-- | An error-throwing IO action that parses this text as CSV conversion rules +-- and runs some extra validation checks. The file path is used in error messages. +parseAndValidateCsvRules :: FilePath -> T.Text -> Either String CsvRules +parseAndValidateCsvRules rulesfile s = + case parseCsvRules rulesfile s of + Left err -> Left $ customErrorBundlePretty err + Right rules -> first makeFancyParseError $ validateRules rules where - makeParseError :: String -> ParseError T.Text String - makeParseError s = FancyError 0 (S.singleton $ ErrorFail s) + makeFancyParseError :: String -> String + makeFancyParseError s = + parseErrorPretty (FancyError 0 (S.singleton $ ErrorFail s) :: ParseError Text String) -- | Parse this text as CSV conversion rules. The file path is for error messages. parseCsvRules :: FilePath -> T.Text -> Either (ParseErrorBundle T.Text CustomErr) CsvRules -- parseCsvRules rulesfile s = runParser csvrulesfile nullrules{baseAccount=takeBaseName rulesfile} rulesfile s parseCsvRules rulesfile s = - runParser (evalStateT rulesp rules) rulesfile s + runParser (evalStateT rulesp defrules) rulesfile s -- | Return the validated rules, or an error. -validateRules :: CsvRules -> ExceptT String IO CsvRules +validateRules :: CsvRules -> Either String CsvRules validateRules rules = do - unless (isAssigned "date") $ ExceptT $ return $ Left "Please specify (at top level) the date field. Eg: date %1\n" - unless ((amount && not (amountin || amountout)) || - (not amount && (amountin && amountout)) || - balance) - $ ExceptT $ return $ Left $ unlines [ - "Please specify (as a top level CSV rule) either the amount field," - ,"both the amount-in and amount-out fields, or the balance field. Eg:" - ,"amount %2\n" - ] - ExceptT $ return $ Right rules + unless (isAssigned "date") $ Left "Please specify (at top level) the date field. Eg: date %1\n" + Right rules where - amount = isAssigned "amount" - amountin = isAssigned "amount-in" - amountout = isAssigned "amount-out" - balance = isAssigned "balance" || isAssigned "balance1" || isAssigned "balance2" isAssigned f = isJust $ getEffectiveAssignment rules [] f -- parsers @@ -510,6 +516,7 @@ ,"newest-first" -- ,"base-account" -- ,"base-currency" + , "balance-type" ] directivevalp :: CsvRulesParser String @@ -545,8 +552,9 @@ fieldassignmentp = do lift $ dbgparse 3 "trying fieldassignmentp" f <- journalfieldnamep - assignmentseparatorp - v <- fieldvalp + v <- choiceInState [ assignmentseparatorp >> fieldvalp + , lift eolof >> return "" + ] return (f,v) "field assignment" @@ -558,14 +566,19 @@ -- Transaction fields and pseudo fields for CSV conversion. -- Names must precede any other name they contain, for the parser -- (amount-in before amount; date2 before date). TODO: fix -journalfieldnames = [ - "account1" - ,"account2" - ,"amount-in" +journalfieldnames = + concat [[ "account" ++ i + ,"amount" ++ i ++ "-in" + ,"amount" ++ i ++ "-out" + ,"amount" ++ i + ,"balance" ++ i + ,"comment" ++ i + ,"currency" ++ i + ] | x <- [1..9], let i = show x] + ++ + ["amount-in" ,"amount-out" ,"amount" - ,"balance1" - ,"balance2" ,"balance" ,"code" ,"comment" @@ -574,17 +587,16 @@ ,"date" ,"description" ,"status" + ,"skip" -- skip and end are not really fields, but we list it here to allow conditional rules that skip records + ,"end" ] assignmentseparatorp :: CsvRulesParser () assignmentseparatorp = do lift $ dbgparse 3 "trying assignmentseparatorp" - choice [ - -- try (lift (skipMany spacenonewline) >> oneOf ":="), - try (lift (skipMany spacenonewline) >> char ':'), - spaceChar - ] - _ <- lift (skipMany spacenonewline) + _ <- choiceInState [ lift (skipMany spacenonewline) >> char ':' >> lift (skipMany spacenonewline) + , lift (skipSome spacenonewline) + ] return () fieldvalp :: CsvRulesParser String @@ -597,9 +609,9 @@ lift $ dbgparse 3 "trying conditionalblockp" string "if" >> lift (skipMany spacenonewline) >> optional newline ms <- some recordmatcherp - as <- many (lift (skipSome spacenonewline) >> fieldassignmentp) + as <- many (try $ lift (skipSome spacenonewline) >> fieldassignmentp) when (null as) $ - fail "start of conditional block found, but no assignment rules afterward\n(assignment rules in a conditional block should be indented)\n" + Fail.fail "start of conditional block found, but no assignment rules afterward\n(assignment rules in a conditional block should be indented)\n" return (ms, as) "conditional block" @@ -610,7 +622,7 @@ _ <- optional (matchoperatorp >> lift (skipMany spacenonewline) >> optional newline) ps <- patternsp when (null ps) $ - fail "start of record matcher found, but no patterns afterward\n(patterns should not be indented)\n" + Fail.fail "start of record matcher found, but no patterns afterward\n(patterns should not be indented)\n" return ps "record matcher" @@ -654,8 +666,9 @@ type CsvRecord = [String] --- Convert a CSV record to a transaction using the rules, or raise an --- error if the data can not be parsed. +showRules rules record = + unlines $ catMaybes [ (("the "++fld++" rule is: ")++) <$> getEffectiveAssignment rules record fld | fld <- journalfieldnames] + transactionFromCsvRecord :: SourcePos -> CsvRules -> CsvRecord -> Transaction transactionFromCsvRecord sourcepos rules record = t where @@ -671,11 +684,11 @@ mdateformat = mdirective "date-format" date = render $ fromMaybe "" $ mfieldtemplate "date" date' = fromMaybe (error' $ dateerror "date" date mdateformat) $ mparsedate date - mdate2 = maybe Nothing (Just . render) $ mfieldtemplate "date2" + mdate2 = render <$> mfieldtemplate "date2" mdate2' = maybe Nothing (maybe (error' $ dateerror "date2" (fromMaybe "" mdate2) mdateformat) Just . mparsedate) mdate2 dateerror datefield value mdateformat = unlines ["error: could not parse \""++value++"\" as a date using date format "++maybe "\"YYYY/M/D\", \"YYYY-M-D\" or \"YYYY.M.D\"" show mdateformat - ,"the CSV record is: "++showRecord record + , showRecord record ,"the "++datefield++" rule is: "++(fromMaybe "required, but missing" $ mfieldtemplate datefield) ,"the date-format is: "++fromMaybe "unspecified" mdateformat ,"you may need to " @@ -695,58 +708,108 @@ ["error: could not parse \""++str++"\" as a cleared status (should be *, ! or empty)" ,"the parse error is: "++customErrorBundlePretty err ] - code = maybe "" render $ mfieldtemplate "code" - description = maybe "" render $ mfieldtemplate "description" - comment = maybe "" render $ mfieldtemplate "comment" - precomment = maybe "" render $ mfieldtemplate "precomment" - currency = maybe (fromMaybe "" mdefaultcurrency) render $ mfieldtemplate "currency" - amountstr = (currency++) <$> simplifySign <$> getAmountStr rules record - maybeamount = either amounterror (Mixed . (:[])) <$> runParser (evalStateT (amountp <* eof) mempty) "" <$> T.pack <$> amountstr - amounterror err = error' $ unlines - ["error: could not parse \""++fromJust amountstr++"\" as an amount" - ,showRecord record - ,"the amount rule is: "++(fromMaybe "" $ mfieldtemplate "amount") - ,"the currency rule is: "++(fromMaybe "unspecified" $ mfieldtemplate "currency") - ,"the default-currency is: "++fromMaybe "unspecified" mdefaultcurrency - ,"the parse error is: "++customErrorBundlePretty err - ,"you may need to " - ++"change your amount or currency rules, " - ++"or "++maybe "add a" (const "change your") mskip++" skip rule" - ] - amount1 = case maybeamount of - Just a -> a - Nothing | balance1 /= Nothing || balance2 /= Nothing -> nullmixedamt - Nothing -> error' $ "amount and balance have no value\n"++showRecord record - -- convert balancing amount to cost like hledger print, so eg if - -- amount1 is "10 GBP @@ 15 USD", amount2 will be "-15 USD". - amount2 = costOfMixedAmount (-amount1) + code = singleline $ maybe "" render $ mfieldtemplate "code" + description = singleline $ maybe "" render $ mfieldtemplate "description" + comment = singleline $ maybe "" render $ mfieldtemplate "comment" + precomment = singleline $ maybe "" render $ mfieldtemplate "precomment" + s `or` def = if null s then def else s - defaccount1 = fromMaybe "unknown" $ mdirective "default-account1" - defaccount2 = case isNegativeMixedAmount amount2 of - Just True -> "income:unknown" - _ -> "expenses:unknown" - account1 = T.pack $ maybe "" render (mfieldtemplate "account1") `or` defaccount1 - account2 = T.pack $ maybe "" render (mfieldtemplate "account2") `or` defaccount2 - balance1template = - case (mfieldtemplate "balance", mfieldtemplate "balance1") of - (Nothing, Nothing) -> Nothing - (balance, Nothing) -> balance - (Nothing, balance1) -> balance1 - (Just _, Just _) -> error' "Please use either balance or balance1, but not both" - balance1 = maybe Nothing (parsebalance "1".render) $ balance1template - balance2 = maybe Nothing (parsebalance "2".render) $ mfieldtemplate "balance2" - parsebalance n str + parsebalance currency n str | all isSpace str = Nothing | otherwise = Just $ (either (balanceerror n str) id $ runParser (evalStateT (amountp <* eof) mempty) "" $ T.pack $ (currency++) $ simplifySign str, nullsourcepos) balanceerror n str err = error' $ unlines ["error: could not parse \""++str++"\" as balance"++n++" amount" ,showRecord record - ,"the balance"++n++" rule is: "++(fromMaybe "" $ mfieldtemplate ("balance"++n)) - ,"the currency rule is: "++(fromMaybe "unspecified" $ mfieldtemplate "currency") + ,showRules rules record ,"the default-currency is: "++fromMaybe "unspecified" mdefaultcurrency ,"the parse error is: "++customErrorBundlePretty err ] + parsePosting' number accountFld amountFld amountInFld amountOutFld balanceFld commentFld = + let currency = maybe (fromMaybe "" mdefaultcurrency) render $ + (mfieldtemplate ("currency"++number) `or `mfieldtemplate "currency") + amount = chooseAmount rules record currency amountFld amountInFld amountOutFld + account' = ((T.pack . render) <$> (mfieldtemplate accountFld + `or` mdirective ("default-account" ++ number))) + balance = (parsebalance currency number.render) =<< mfieldtemplate balanceFld + comment = T.pack $ maybe "" render $ mfieldtemplate commentFld + account = + case account' of + -- If account is explicitly "unassigned", suppress posting + -- Otherwise, generate posting with "expenses:unknown" account if we have amount/balance information + Just "" -> Nothing + Just account -> Just account + Nothing -> + -- If we have amount or balance assertion (which implies potential amount change), + -- but no account name, lets generate "expenses:unknown" account name. + case (amount, balance) of + (Just _, _ ) -> Just "expenses:unknown" + (_, Just _) -> Just "expenses:unknown" + (Nothing, Nothing) -> Nothing + in + case account of + Nothing -> Nothing + Just account -> + Just $ (number, posting {paccount=accountNameWithoutPostingType account + , pamount=fromMaybe missingmixedamt amount + , ptransaction=Just t + , pbalanceassertion=toAssertion <$> balance + , pcomment = comment + , ptype = accountNamePostingType account}) + + parsePosting number = + parsePosting' number + ("account"++number) + ("amount"++number) + ("amount"++number++"-in") + ("amount"++number++"-out") + ("balance"++number) + ("comment" ++ number) + + withAlias fld alias = + case (mfieldtemplate fld, mfieldtemplate alias) of + (Just fld, Just alias) -> error' $ unlines + [ "error: both \"" ++ fld ++ "\" and \"" ++ alias ++ "\" have values." + , showRecord record + , showRules rules record + ] + (Nothing, Just _) -> alias + (_, Nothing) -> fld + + posting1 = parsePosting' "1" + ("account1" `withAlias` "account") + ("amount1" `withAlias` "amount") + ("amount1-in" `withAlias` "amount-in") + ("amount1-out" `withAlias` "amount-out") + ("balance1" `withAlias` "balance") + "comment1" -- comment1 does not have legacy alias + + postings' = catMaybes $ posting1:[ parsePosting i | x<-[2..9], let i = show x] + + improveUnknownAccountName p = + if paccount p /="expenses:unknown" + then p + else case isNegativeMixedAmount (pamount p) of + Just True -> p{paccount = "income:unknown"} + Just False -> p{paccount = "expenses:unknown"} + _ -> p + + postings = + case postings' of + -- To be compatible with the behavior of the old code which allowed two postings only, we enforce + -- second posting when rules generated just first of them, and posting is of type that should be balanced. + -- When we have srictly first and second posting, but second posting does not have amount, we fill it in. + [("1",posting1)] -> + case ptype posting1 of + VirtualPosting -> [posting1] + _ -> + [posting1,improveUnknownAccountName (posting{paccount="expenses:unknown", pamount=costOfMixedAmount(-(pamount posting1)), ptransaction=Just t})] + [("1",posting1),("2",posting2)] -> + case (pamount posting1 == missingmixedamt , pamount posting2 == missingmixedamt) of + (False, True) -> [posting1, improveUnknownAccountName (posting2{pamount=costOfMixedAmount(-(pamount posting1))})] + _ -> [posting1, posting2] + _ -> map snd postings' + -- build the transaction t = nulltransaction{ tsourcepos = genericSourcePos sourcepos, @@ -756,40 +819,71 @@ tcode = T.pack code, tdescription = T.pack description, tcomment = T.pack comment, - tprecedingcomment = T.pack precomment, - tpostings = - [posting {paccount=account1, pamount=amount1, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance1} - ,posting {paccount=account2, pamount=amount2, ptransaction=Just t, pbalanceassertion=toAssertion <$> balance2} - ] + tprecedingcomment = T.pack precomment, + tpostings = postings } - toAssertion (a, b) = assertion{ + + defaultAssertion = + case mdirective "balance-type" of + Nothing -> assertion + Just "=" -> assertion + Just "==" -> assertion {batotal=True} + Just "=*" -> assertion {bainclusive=True} + Just "==*" -> assertion{batotal=True, bainclusive=True} + Just x -> error' $ unlines + [ "balance-type \"" ++ x ++"\" is invalid. Use =, ==, =* or ==*." + , showRecord record + , showRules rules record + ] + + toAssertion (a, b) = defaultAssertion{ baamount = a, baposition = b } -getAmountStr :: CsvRules -> CsvRecord -> Maybe String -getAmountStr rules record = +chooseAmount :: CsvRules -> CsvRecord -> String -> String -> String -> String -> Maybe MixedAmount +chooseAmount rules record currency amountFld amountInFld amountOutFld = let - mamount = getEffectiveAssignment rules record "amount" - mamountin = getEffectiveAssignment rules record "amount-in" - mamountout = getEffectiveAssignment rules record "amount-out" - render = fmap (strip . renderTemplate rules record) + mamount = getEffectiveAssignment rules record amountFld + mamountin = getEffectiveAssignment rules record amountInFld + mamountout = getEffectiveAssignment rules record amountOutFld + parse amt = notZero =<< (parseAmount currency <$> notEmpty =<< (strip . renderTemplate rules record) <$> amt) in - case (render mamount, render mamountin, render mamountout) of - (Just "", Nothing, Nothing) -> Nothing + case (parse mamount, parse mamountin, parse mamountout) of + (Nothing, Nothing, Nothing) -> Nothing (Just a, Nothing, Nothing) -> Just a - (Nothing, Just "", Just "") -> error' $ "neither amount-in or amount-out has a value\n" - ++ " record: " ++ showRecord record - (Nothing, Just i, Just "") -> Just i - (Nothing, Just "", Just o) -> Just $ negateStr o - (Nothing, Just i, Just o) -> error' $ "both amount-in and amount-out have a value\n" - ++ " amount-in: " ++ i ++ "\n" - ++ " amount-out: " ++ o ++ "\n" + (Nothing, Just i, Nothing) -> Just i + (Nothing, Nothing, Just o) -> Just $ negate o + (Nothing, Just i, Just o) -> error' $ "both "++amountInFld++" and "++amountOutFld++" have a value\n" + ++ " "++amountInFld++": " ++ show i ++ "\n" + ++ " "++amountOutFld++": " ++ show o ++ "\n" ++ " record: " ++ showRecord record - _ -> error' $ "found values for amount and for amount-in/amount-out\n" - ++ "please use either amount or amount-in/amount-out\n" + _ -> error' $ "found values for "++amountFld++" and for "++amountInFld++"/"++amountOutFld++"\n" + ++ "please use either "++amountFld++" or "++amountInFld++"/"++amountOutFld++"\n" ++ " record: " ++ showRecord record + where + notZero amt = if isZeroMixedAmount amt then Nothing else Just amt + notEmpty str = if str=="" then Nothing else Just str + parseAmount currency amountstr = + either (amounterror amountstr) (Mixed . (:[])) + <$> runParser (evalStateT (amountp <* eof) mempty) "" + <$> T.pack + <$> (currency++) + <$> simplifySign + <$> amountstr + + amounterror amountstr err = error' $ unlines + ["error: could not parse \""++fromJust amountstr++"\" as an amount" + ,showRecord record + ,showRules rules record + ,"the default-currency is: "++fromMaybe "unspecified" (getDirective "default-currency" rules) + ,"the parse error is: "++customErrorBundlePretty err + ,"you may need to " + ++"change your amount or currency rules, " + ++"or add or change your skip rule" + ] + type CsvAmountString = String -- | Canonicalise the sign in a CSV amount string. @@ -853,7 +947,7 @@ -- | Render a field assigment's template, possibly interpolating referenced -- CSV field values. Outer whitespace is removed from interpolated values. renderTemplate :: CsvRules -> CsvRecord -> FieldTemplate -> String -renderTemplate rules record t = regexReplaceBy "%[A-z0-9]+" replace t +renderTemplate rules record t = regexReplaceBy "%[A-z0-9_-]+" replace t where replace ('%':pat) = maybe pat (\i -> strip $ atDef "" record (i-1)) mindex where @@ -891,22 +985,26 @@ tests_CsvReader = tests "CsvReader" [ tests "parseCsvRules" [ - test "empty file" $ - parseCsvRules "unknown" "" `is` Right rules + test"empty file" $ + parseCsvRules "unknown" "" @?= Right defrules ] ,tests "rulesp" [ - test "trailing comments" $ - parseWithState' rules rulesp "skip\n# \n#\n" `is` Right rules{rdirectives = [("skip","")]} - - ,test "trailing blank lines" $ - parseWithState' rules rulesp "skip\n\n \n" `is` (Right rules{rdirectives = [("skip","")]}) + test"trailing comments" $ + parseWithState' defrules rulesp "skip\n# \n#\n" @?= Right defrules{rdirectives = [("skip","")]} - ,test "no final newline" $ - parseWithState' rules rulesp "skip" `is` (Right rules{rdirectives=[("skip","")]}) + ,test"trailing blank lines" $ + parseWithState' defrules rulesp "skip\n\n \n" @?= (Right defrules{rdirectives = [("skip","")]}) - ,test "assignment with empty value" $ - parseWithState' rules rulesp "account1 \nif foo\n account2 foo\n" `is` - (Right rules{rassignments = [("account1","")], rconditionalblocks = [([["foo"]],[("account2","foo")])]}) + ,test"no final newline" $ + parseWithState' defrules rulesp "skip" @?= (Right defrules{rdirectives=[("skip","")]}) - ] + ,test"assignment with empty value" $ + parseWithState' defrules rulesp "account1 \nif foo\n account2 foo\n" @?= + (Right defrules{rassignments = [("account1","")], rconditionalblocks = [([["foo"]],[("account2","foo")])]}) + ] + ,tests "conditionalblockp" [ + test"space after conditional" $ -- #1120 + parseWithState' defrules conditionalblockp "if a\n account2 b\n \n" @?= + (Right ([["a"]],[("account2","b")])) + ] ] diff --git a/Hledger/Read/JournalReader.hs b/Hledger/Read/JournalReader.hs --- a/Hledger/Read/JournalReader.hs +++ b/Hledger/Read/JournalReader.hs @@ -63,13 +63,17 @@ ) where --- * imports -import Prelude () -import "base-compat-batteries" Prelude.Compat hiding (readFile) +import qualified "base-compat-batteries" Control.Monad.Fail.Compat as Fail (fail) import qualified Control.Exception as C -import Control.Monad +import Control.Monad (forM_, when, void) +import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Except (ExceptT(..), runExceptT) -import Control.Monad.State.Strict +import Control.Monad.State.Strict (get,modify',put) +import Control.Monad.Trans.Class (lift) import qualified Data.Map.Strict as M +#if !(MIN_VERSION_base(4,11,0)) +import Data.Monoid ((<>)) +#endif import Data.Text (Text) import Data.String import Data.List @@ -215,7 +219,7 @@ let parentfilestack = jincludefilestack parentj when (filepath `elem` parentfilestack) $ - fail ("Cyclic include: " ++ filepath) + Fail.fail ("Cyclic include: " ++ filepath) childInput <- lift $ readFilePortably filepath `orRethrowIOError` (show parentpos ++ " reading " ++ filepath) @@ -251,7 +255,7 @@ eResult <- liftIO $ (Right <$> io) `C.catch` \(e::C.IOException) -> pure $ Left $ printf "%s:\n%s" msg (show e) case eResult of Right res -> pure res - Left errMsg -> fail errMsg + Left errMsg -> Fail.fail errMsg -- Parse an account directive, adding its info to the journal's -- list of account declarations. @@ -664,46 +668,46 @@ let p = lift accountnamep :: JournalParser IO AccountName in tests "accountnamep" [ - test "basic" $ expectParse p "a:b:c" - ,_test "empty inner component" $ expectParseError p "a::c" "" -- TODO - ,_test "empty leading component" $ expectParseError p ":b:c" "x" - ,_test "empty trailing component" $ expectParseError p "a:b:" "x" + test "basic" $ assertParse p "a:b:c" + -- ,test "empty inner component" $ assertParseError p "a::c" "" -- TODO + -- ,test "empty leading component" $ assertParseError p ":b:c" "x" + -- ,test "empty trailing component" $ assertParseError p "a:b:" "x" ] -- "Parse a date in YYYY/MM/DD format. -- Hyphen (-) and period (.) are also allowed as separators. -- The year may be omitted if a default year has been set. -- Leading zeroes may be omitted." - ,test "datep" $ do - test "YYYY/MM/DD" $ expectParseEq datep "2018/01/01" (fromGregorian 2018 1 1) - test "YYYY-MM-DD" $ expectParse datep "2018-01-01" - test "YYYY.MM.DD" $ expectParse datep "2018.01.01" - test "yearless date with no default year" $ expectParseError datep "1/1" "current year is unknown" - test "yearless date with default year" $ do + ,tests "datep" [ + test "YYYY/MM/DD" $ assertParseEq datep "2018/01/01" (fromGregorian 2018 1 1) + ,test "YYYY-MM-DD" $ assertParse datep "2018-01-01" + ,test "YYYY.MM.DD" $ assertParse datep "2018.01.01" + ,test "yearless date with no default year" $ assertParseError datep "1/1" "current year is unknown" + ,test "yearless date with default year" $ do let s = "1/1" ep <- parseWithState mempty{jparsedefaultyear=Just 2018} datep s - either (fail.("parse error at "++).customErrorBundlePretty) (const ok) ep - test "no leading zero" $ expectParse datep "2018/1/1" - + either (assertFailure . ("parse error at "++) . customErrorBundlePretty) (const $ return ()) ep + ,test "no leading zero" $ assertParse datep "2018/1/1" + ] ,test "datetimep" $ do - let - good = expectParse datetimep - bad = (\t -> expectParseError datetimep t "") - good "2011/1/1 00:00" - good "2011/1/1 23:59:59" - bad "2011/1/1" - bad "2011/1/1 24:00:00" - bad "2011/1/1 00:60:00" - bad "2011/1/1 00:00:60" - bad "2011/1/1 3:5:7" - test "timezone is parsed but ignored" $ do - let t = LocalTime (fromGregorian 2018 1 1) (TimeOfDay 0 0 (fromIntegral 0)) - expectParseEq datetimep "2018/1/1 00:00-0800" t - expectParseEq datetimep "2018/1/1 00:00+1234" t + let + good = assertParse datetimep + bad = (\t -> assertParseError datetimep t "") + good "2011/1/1 00:00" + good "2011/1/1 23:59:59" + bad "2011/1/1" + bad "2011/1/1 24:00:00" + bad "2011/1/1 00:60:00" + bad "2011/1/1 00:00:60" + bad "2011/1/1 3:5:7" + -- timezone is parsed but ignored + let t = LocalTime (fromGregorian 2018 1 1) (TimeOfDay 0 0 (fromIntegral 0)) + assertParseEq datetimep "2018/1/1 00:00-0800" t + assertParseEq datetimep "2018/1/1 00:00+1234" t ,tests "periodictransactionp" [ - test "more period text in comment after one space" $ expectParseEq periodictransactionp + test "more period text in comment after one space" $ assertParseEq periodictransactionp "~ monthly from 2018/6 ;In 2019 we will change this\n" nullperiodictransaction { ptperiodexpr = "monthly from 2018/6" @@ -713,7 +717,7 @@ ,ptcomment = "In 2019 we will change this\n" } - ,test "more period text in description after two spaces" $ expectParseEq periodictransactionp + ,test "more period text in description after two spaces" $ assertParseEq periodictransactionp "~ monthly from 2018/6 In 2019 we will change this\n" nullperiodictransaction { ptperiodexpr = "monthly from 2018/6" @@ -723,7 +727,7 @@ ,ptcomment = "" } - ,test "Next year in description" $ expectParseEq periodictransactionp + ,test "Next year in description" $ assertParseEq periodictransactionp "~ monthly Next year blah blah\n" nullperiodictransaction { ptperiodexpr = "monthly" @@ -733,7 +737,7 @@ ,ptcomment = "" } - ,test "Just date, no description" $ expectParseEq periodictransactionp + ,test "Just date, no description" $ assertParseEq periodictransactionp "~ 2019-01-04\n" nullperiodictransaction { ptperiodexpr = "2019-01-04" @@ -743,13 +747,13 @@ ,ptcomment = "" } - ,test "Just date, no description + empty transaction comment" $ expectParse periodictransactionp + ,test "Just date, no description + empty transaction comment" $ assertParse periodictransactionp "~ 2019-01-04\n ;\n a 1\n b\n" ] ,tests "postingp" [ - test "basic" $ expectParseEq (postingp Nothing) + test "basic" $ assertParseEq (postingp Nothing) " expenses:food:dining $10.00 ; a: a a \n ; b: b b \n" posting{ paccount="expenses:food:dining", @@ -758,7 +762,7 @@ ptags=[("a","a a"), ("b","b b")] } - ,test "posting dates" $ expectParseEq (postingp Nothing) + ,test "posting dates" $ assertParseEq (postingp Nothing) " a 1. ; date:2012/11/28, date2=2012/11/29,b:b\n" nullposting{ paccount="a" @@ -769,7 +773,7 @@ ,pdate2=Nothing -- Just $ fromGregorian 2012 11 29 } - ,test "posting dates bracket syntax" $ expectParseEq (postingp Nothing) + ,test "posting dates bracket syntax" $ assertParseEq (postingp Nothing) " a 1. ; [2012/11/28=2012/11/29]\n" nullposting{ paccount="a" @@ -780,16 +784,16 @@ ,pdate2=Just $ fromGregorian 2012 11 29 } - ,test "quoted commodity symbol with digits" $ expectParse (postingp Nothing) " a 1 \"DE123\"\n" + ,test "quoted commodity symbol with digits" $ assertParse (postingp Nothing) " a 1 \"DE123\"\n" - ,test "balance assertion and fixed lot price" $ expectParse (postingp Nothing) " a 1 \"DE123\" =$1 { =2.2 EUR} \n" + ,test "balance assertion and fixed lot price" $ assertParse (postingp Nothing) " a 1 \"DE123\" =$1 { =2.2 EUR} \n" - ,test "balance assertion over entire contents of account" $ expectParse (postingp Nothing) " a $1 == $1\n" + ,test "balance assertion over entire contents of account" $ assertParse (postingp Nothing) " a $1 == $1\n" ] ,tests "transactionmodifierp" [ - test "basic" $ expectParseEq transactionmodifierp + test "basic" $ assertParseEq transactionmodifierp "= (some value expr)\n some:postings 1.\n" nulltransactionmodifier { tmquerytxt = "(some value expr)" @@ -799,9 +803,9 @@ ,tests "transactionp" [ - test "just a date" $ expectParseEq transactionp "2015/1/1\n" nulltransaction{tdate=fromGregorian 2015 1 1} + test "just a date" $ assertParseEq transactionp "2015/1/1\n" nulltransaction{tdate=fromGregorian 2015 1 1} - ,test "more complex" $ expectParseEq transactionp + ,test "more complex" $ assertParseEq transactionp (T.unlines [ "2012/05/14=2012/05/15 (code) desc ; tcomment1", " ; tcomment2", @@ -836,7 +840,7 @@ } ,test "parses a well-formed transaction" $ - expect $ isRight $ rjp transactionp $ T.unlines + assertBool "" $ isRight $ rjp transactionp $ T.unlines ["2007/01/28 coopportunity" ," expenses:food:groceries $47.18" ," assets:checking $-47.18" @@ -844,18 +848,18 @@ ] ,test "does not parse a following comment as part of the description" $ - expectParseEqOn transactionp "2009/1/1 a ;comment\n b 1\n" tdescription "a" + assertParseEqOn transactionp "2009/1/1 a ;comment\n b 1\n" tdescription "a" - ,test "transactionp parses a following whitespace line" $ - expect $ isRight $ rjp transactionp $ T.unlines + ,test "parses a following whitespace line" $ + assertBool "" $ isRight $ rjp transactionp $ T.unlines ["2012/1/1" ," a 1" ," b" ," " ] - ,test "transactionp parses an empty transaction comment following whitespace line" $ - expect $ isRight $ rjp transactionp $ T.unlines + ,test "parses an empty transaction comment following whitespace line" $ + assertBool "" $ isRight $ rjp transactionp $ T.unlines ["2012/1/1" ," ;" ," a 1" @@ -864,7 +868,7 @@ ] ,test "comments everywhere, two postings parsed" $ - expectParseEqOn transactionp + assertParseEqOn transactionp (T.unlines ["2009/1/1 x ; transaction comment" ," a 1 ; posting 1 comment" @@ -881,42 +885,45 @@ ,tests "directivep" [ test "supports !" $ do - expectParseE directivep "!account a\n" - expectParseE directivep "!D 1.0\n" - ] + assertParseE directivep "!account a\n" + assertParseE directivep "!D 1.0\n" + ] - ,test "accountdirectivep" $ do - test "with-comment" $ expectParse accountdirectivep "account a:b ; a comment\n" - test "does-not-support-!" $ expectParseError accountdirectivep "!account a:b\n" "" - test "account-type-code" $ expectParse accountdirectivep "account a:b A\n" - test "account-type-tag" $ expectParseStateOn accountdirectivep "account a:b ; type:asset\n" - jdeclaredaccounts - [("a:b", AccountDeclarationInfo{adicomment = "type:asset\n" - ,aditags = [("type","asset")] - ,adideclarationorder = 1 - }) + ,tests "accountdirectivep" [ + test "with-comment" $ assertParse accountdirectivep "account a:b ; a comment\n" + ,test "does-not-support-!" $ assertParseError accountdirectivep "!account a:b\n" "" + ,test "account-type-code" $ assertParse accountdirectivep "account a:b A\n" + ,test "account-type-tag" $ assertParseStateOn accountdirectivep "account a:b ; type:asset\n" + jdeclaredaccounts + [("a:b", AccountDeclarationInfo{adicomment = "type:asset\n" + ,aditags = [("type","asset")] + ,adideclarationorder = 1 + }) + ] ] ,test "commodityconversiondirectivep" $ do - expectParse commodityconversiondirectivep "C 1h = $50.00\n" + assertParse commodityconversiondirectivep "C 1h = $50.00\n" ,test "defaultcommoditydirectivep" $ do - expectParse defaultcommoditydirectivep "D $1,000.0\n" - expectParseError defaultcommoditydirectivep "D $1000\n" "please include a decimal separator" + assertParse defaultcommoditydirectivep "D $1,000.0\n" + assertParseError defaultcommoditydirectivep "D $1000\n" "please include a decimal separator" - ,test "defaultyeardirectivep" $ do - test "1000" $ expectParse defaultyeardirectivep "Y 1000" -- XXX no \n like the others - test "999" $ expectParseError defaultyeardirectivep "Y 999" "bad year number" - test "12345" $ expectParse defaultyeardirectivep "Y 12345" + ,tests "defaultyeardirectivep" [ + test "1000" $ assertParse defaultyeardirectivep "Y 1000" -- XXX no \n like the others + ,test "999" $ assertParseError defaultyeardirectivep "Y 999" "bad year number" + ,test "12345" $ assertParse defaultyeardirectivep "Y 12345" + ] ,test "ignoredpricecommoditydirectivep" $ do - expectParse ignoredpricecommoditydirectivep "N $\n" + assertParse ignoredpricecommoditydirectivep "N $\n" - ,test "includedirectivep" $ do - test "include" $ expectParseErrorE includedirectivep "include nosuchfile\n" "No existing files match pattern: nosuchfile" - test "glob" $ expectParseErrorE includedirectivep "include nosuchfile*\n" "No existing files match pattern: nosuchfile*" + ,tests "includedirectivep" [ + test "include" $ assertParseErrorE includedirectivep "include nosuchfile\n" "No existing files match pattern: nosuchfile" + ,test "glob" $ assertParseErrorE includedirectivep "include nosuchfile*\n" "No existing files match pattern: nosuchfile*" + ] - ,test "marketpricedirectivep" $ expectParseEq marketpricedirectivep + ,test "marketpricedirectivep" $ assertParseEq marketpricedirectivep "P 2017/01/30 BTC $922.83\n" PriceDirective{ pddate = fromGregorian 2017 1 30, @@ -925,23 +932,20 @@ } ,test "tagdirectivep" $ do - expectParse tagdirectivep "tag foo \n" + assertParse tagdirectivep "tag foo \n" ,test "endtagdirectivep" $ do - expectParse endtagdirectivep "end tag \n" - expectParse endtagdirectivep "pop \n" - + assertParse endtagdirectivep "end tag \n" + assertParse endtagdirectivep "pop \n" ,tests "journalp" [ - test "empty file" $ expectParseEqE journalp "" nulljournal + test "empty file" $ assertParseEqE journalp "" nulljournal ] -- these are defined here rather than in Common so they can use journalp - ,tests "parseAndFinaliseJournal" [ - test "basic" $ do - ej <- io $ runExceptT $ parseAndFinaliseJournal journalp definputopts "" "2019-1-1\n" - let Right j = ej - expectEqPP [""] $ journalFilePaths j - ] + ,test "parseAndFinaliseJournal" $ do + ej <- runExceptT $ parseAndFinaliseJournal journalp definputopts "" "2019-1-1\n" + let Right j = ej + assertEqual "" [""] $ journalFilePaths j ] diff --git a/Hledger/Reports/AccountTransactionsReport.hs b/Hledger/Reports/AccountTransactionsReport.hs --- a/Hledger/Reports/AccountTransactionsReport.hs +++ b/Hledger/Reports/AccountTransactionsReport.hs @@ -82,26 +82,44 @@ balancelabel = "Historical Total" accountTransactionsReport :: ReportOpts -> Journal -> Query -> Query -> AccountTransactionsReport -accountTransactionsReport opts j reportq thisacctq = (label, items) +accountTransactionsReport ropts j reportq thisacctq = (label, items) where -- a depth limit does not affect the account transactions report -- seems unnecessary for some reason XXX reportq' = -- filterQuery (not . queryIsDepth) reportq - -- get all transactions, with amounts converted to cost basis if -B - ts1 = jtxns $ journalSelectingAmountFromOpts opts j + + -- get all transactions + ts1 = jtxns j + -- apply any cur:SYM filters in reportq' symq = filterQuery queryIsSym reportq' ts2 = (if queryIsNull symq then id else map (filterTransactionAmounts symq)) ts1 + -- keep just the transactions affecting this account (via possibly realness or status-filtered postings) realq = filterQuery queryIsReal reportq' statusq = filterQuery queryIsStatus reportq' ts3 = filter (matchesTransaction thisacctq . filterTransactionPostings (And [realq, statusq])) ts2 + + -- maybe convert these transactions to cost or value + prices = journalPriceOracle j + styles = journalCommodityStyles j + periodlast = + fromMaybe (error' "journalApplyValuation: expected a non-empty journal") $ -- XXX shouldn't happen + reportPeriodOrJournalLastDay ropts j + mreportlast = reportPeriodLastDay ropts + today = fromMaybe (error' "journalApplyValuation: could not pick a valuation date, ReportOpts today_ is unset") $ today_ ropts + multiperiod = interval_ ropts /= NoInterval + tval = case value_ ropts of + Just v -> \t -> transactionApplyValuation prices styles periodlast mreportlast today multiperiod t v + Nothing -> id + ts4 = map tval ts3 + -- sort by the transaction's register date, for accurate starting balance - ts = sortBy (comparing (transactionRegisterDate reportq' thisacctq)) ts3 + ts = sortBy (comparing (transactionRegisterDate reportq' thisacctq)) ts4 (startbal,label) - | balancetype_ opts == HistoricalBalance = (sumPostings priorps, balancelabel) + | balancetype_ ropts == HistoricalBalance = (sumPostings priorps, balancelabel) | otherwise = (nullmixedamt, totallabel) where priorps = dbg1 "priorps" $ @@ -113,7 +131,7 @@ case mstartdate of Just _ -> Date (DateSpan Nothing mstartdate) Nothing -> None -- no start date specified, there are no prior postings - mstartdate = queryStartDate (date2_ opts) reportq' + mstartdate = queryStartDate (date2_ ropts) reportq' datelessreportq = filterQuery (not . queryIsDateOrDate2) reportq' items = reverse $ diff --git a/Hledger/Reports/BalanceReport.hs b/Hledger/Reports/BalanceReport.hs --- a/Hledger/Reports/BalanceReport.hs +++ b/Hledger/Reports/BalanceReport.hs @@ -12,6 +12,8 @@ balanceReport, flatShowsExclusiveBalance, sortAccountItemsLike, + unifyMixedAmount, + perdivide, -- * Tests tests_BalanceReport @@ -66,7 +68,7 @@ balanceReport :: ReportOpts -> Query -> Journal -> BalanceReport balanceReport ropts@ReportOpts{..} q j@Journal{..} = (if invert_ then brNegate else id) $ - (sorteditems, total) + (mappedsorteditems, mappedtotal) where -- dbg1 = const id -- exclude from debug output dbg1 s = let p = "balanceReport" in Hledger.Utils.dbg1 (p++" "++s) -- add prefix in debug output @@ -142,6 +144,14 @@ if flatShowsExclusiveBalance then sum $ map fourth4 items else sum $ map aebalance $ clipAccountsAndAggregate 1 displayaccts + + -- Calculate percentages if needed. + mappedtotal | percent_ = dbg1 "mappedtotal" $ total `perdivide` total + | otherwise = total + mappedsorteditems | percent_ = + dbg1 "mappedsorteditems" $ + map (\(fname, sname, indent, amount) -> (fname, sname, indent, amount `perdivide` total)) sorteditems + | otherwise = sorteditems -- | A sorting helper: sort a list of things (eg report rows) keyed by account name -- to match the provided ordering of those same account names. @@ -185,7 +195,33 @@ where brItemNegate (a, a', d, amt) = (a, a', d, -amt) +-- | Helper to unify a MixedAmount to a single commodity value. +-- Like normaliseMixedAmount, this consolidates amounts of the same commodity +-- and discards zero amounts; but this one insists on simplifying to +-- a single commodity, and will throw a program-terminating error if +-- this is not possible. +unifyMixedAmount :: MixedAmount -> Amount +unifyMixedAmount mixedAmount = foldl combine (num 0) (amounts mixedAmount) + where + combine amount result = + if isReallyZeroAmount amount + then result + else if isReallyZeroAmount result + then amount + else if acommodity amount == acommodity result + then amount + result + else error' "Cannot calculate percentages for accounts with multiple commodities. (Hint: Try --cost, -V or similar flags.)" +-- | Helper to calculate the percentage from two mixed. Keeps the sign of the first argument. +-- Uses unifyMixedAmount to unify each argument and then divides them. +perdivide :: MixedAmount -> MixedAmount -> MixedAmount +perdivide a b = + let a' = unifyMixedAmount a + b' = unifyMixedAmount b + in if isReallyZeroAmount a' || isReallyZeroAmount b' || acommodity a' == acommodity b' + then mixed [per $ if aquantity b' == 0 then 0 else (aquantity a' / abs (aquantity b') * 100)] + else error' "Cannot calculate percentages if accounts have different commodities. (Hint: Try --cost, -V or similar flags.)" + -- tests Right samplejournal2 = @@ -212,21 +248,21 @@ } tests_BalanceReport = tests "BalanceReport" [ - tests "balanceReport" $ - let - (opts,journal) `gives` r = do - let (eitems, etotal) = r - (aitems, atotal) = balanceReport opts (queryFromOpts nulldate opts) journal - showw (acct,acct',indent,amt) = (acct, acct', indent, showMixedAmountDebug amt) - (map showw eitems) `is` (map showw aitems) - (showMixedAmountDebug etotal) `is` (showMixedAmountDebug atotal) - usd0 = usd 0 - in [ - test "balanceReport with no args on null journal" $ + let + (opts,journal) `gives` r = do + let (eitems, etotal) = r + (aitems, atotal) = balanceReport opts (queryFromOpts nulldate opts) journal + showw (acct,acct',indent,amt) = (acct, acct', indent, showMixedAmountDebug amt) + (map showw eitems) @?= (map showw aitems) + (showMixedAmountDebug etotal) @?= (showMixedAmountDebug atotal) + in + tests "balanceReport" [ + + test "no args, null journal" $ (defreportopts, nulljournal) `gives` ([], Mixed [nullamt]) - ,test "balanceReport with no args on sample journal" $ + ,test "no args, sample journal" $ (defreportopts, samplejournal) `gives` ([ ("assets","assets",0, mamountp' "$0.00") @@ -241,45 +277,46 @@ ,("income:gifts","gifts",1, mamountp' "$-1.00") ,("income:salary","salary",1, mamountp' "$-1.00") ], - Mixed [usd0]) + Mixed [usd 0]) - ,test "balanceReport with --depth=N" $ + ,test "with --depth=N" $ (defreportopts{depth_=Just 1}, samplejournal) `gives` ([ ("expenses", "expenses", 0, mamountp' "$2.00") ,("income", "income", 0, mamountp' "$-2.00") ], - Mixed [usd0]) + Mixed [usd 0]) - ,test "balanceReport with depth:N" $ + ,test "with depth:N" $ (defreportopts{query_="depth:1"}, samplejournal) `gives` ([ ("expenses", "expenses", 0, mamountp' "$2.00") ,("income", "income", 0, mamountp' "$-2.00") ], - Mixed [usd0]) + Mixed [usd 0]) - ,tests "balanceReport with a date or secondary date span" [ + ,test "with date:" $ (defreportopts{query_="date:'in 2009'"}, samplejournal2) `gives` ([], Mixed [nullamt]) - ,(defreportopts{query_="date2:'in 2009'"}, samplejournal2) `gives` + + ,test "with date2:" $ + (defreportopts{query_="date2:'in 2009'"}, samplejournal2) `gives` ([ ("assets:bank:checking","assets:bank:checking",0,mamountp' "$1.00") ,("income:salary","income:salary",0,mamountp' "$-1.00") ], - Mixed [usd0]) - ] + Mixed [usd 0]) - ,test "balanceReport with desc:" $ + ,test "with desc:" $ (defreportopts{query_="desc:income"}, samplejournal) `gives` ([ ("assets:bank:checking","assets:bank:checking",0,mamountp' "$1.00") ,("income:salary","income:salary",0, mamountp' "$-1.00") ], - Mixed [usd0]) + Mixed [usd 0]) - ,test "balanceReport with not:desc:" $ + ,test "with not:desc:" $ (defreportopts{query_="not:desc:income"}, samplejournal) `gives` ([ ("assets","assets",0, mamountp' "$-1.00") @@ -290,18 +327,18 @@ ,("expenses:supplies","supplies",1, mamountp' "$1.00") ,("income:gifts","income:gifts",0, mamountp' "$-1.00") ], - Mixed [usd0]) + Mixed [usd 0]) - ,test "balanceReport with period on a populated period" $ + ,test "with period on a populated period" $ (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 1) (fromGregorian 2008 1 2)}, samplejournal) `gives` ( [ ("assets:bank:checking","assets:bank:checking",0, mamountp' "$1.00") ,("income:salary","income:salary",0, mamountp' "$-1.00") ], - Mixed [usd0]) + Mixed [usd 0]) - ,test "balanceReport with period on an unpopulated period" $ + ,test "with period on an unpopulated period" $ (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 2) (fromGregorian 2008 1 3)}, samplejournal) `gives` ([],Mixed [nullamt]) @@ -413,7 +450,7 @@ ," a:b 10h @ $50" ," c:d " ]) >>= either error' return - let j' = journalCanonicaliseAmounts $ journalConvertAmountsToCost j -- enable cost basis adjustment + let j' = journalCanonicaliseAmounts $ journalToCost j -- enable cost basis adjustment balanceReportAsText defreportopts (balanceReport defreportopts Any j') `is` [" $500 a:b" ," $-500 c:d" @@ -421,7 +458,7 @@ ," 0" ] -} - ] + ] ] diff --git a/Hledger/Reports/BudgetReport.hs b/Hledger/Reports/BudgetReport.hs --- a/Hledger/Reports/BudgetReport.hs +++ b/Hledger/Reports/BudgetReport.hs @@ -81,8 +81,8 @@ concatMap tpostings $ concatMap (flip runPeriodicTransaction reportspan) $ jperiodictxns j - actualj = dbg1 "actualj" $ budgetRollUp budgetedaccts showunbudgeted j - budgetj = dbg1 "budgetj" $ budgetJournal assrt ropts reportspan j + actualj = dbg1With (("actualj"++).show.jtxns) $ budgetRollUp budgetedaccts showunbudgeted j + budgetj = dbg1With (("budgetj"++).show.jtxns) $ budgetJournal assrt ropts reportspan j actualreport@(MultiBalanceReport (actualspans, _, _)) = dbg1 "actualreport" $ multiBalanceReport ropts q actualj budgetgoalreport@(MultiBalanceReport (_, budgetgoalitems, budgetgoaltotals)) = dbg1 "budgetgoalreport" $ multiBalanceReport (ropts{empty_=True}) q budgetj budgetgoalreport' @@ -286,11 +286,11 @@ Just (AtDate d _mc) -> ", valued at "++showDate d Nothing -> "") actualwidth = - maximum [ maybe 0 (length . showMixedAmountOneLineWithoutPrice) amt + maximum' [ maybe 0 (length . showMixedAmountOneLineWithoutPrice) amt | (_, _, _, amtandgoals, _, _) <- rows , (amt, _) <- amtandgoals ] budgetwidth = - maximum [ maybe 0 (length . showMixedAmountOneLineWithoutPrice) goal + maximum' [ maybe 0 (length . showMixedAmountOneLineWithoutPrice) goal | (_, _, _, amtandgoals, _, _) <- rows , (_, goal) <- amtandgoals ] -- XXX lay out actual, percentage and/or goal in the single table cell for now, should probably use separate cells diff --git a/Hledger/Reports/EntriesReport.hs b/Hledger/Reports/EntriesReport.hs --- a/Hledger/Reports/EntriesReport.hs +++ b/Hledger/Reports/EntriesReport.hs @@ -49,8 +49,8 @@ tests_EntriesReport = tests "EntriesReport" [ tests "entriesReport" [ - test "not acct" $ (length $ entriesReport defreportopts (Not $ Acct "bank") samplejournal) `is` 1 - ,test "date" $ (length $ entriesReport defreportopts (Date $ mkdatespan "2008/06/01" "2008/07/01") samplejournal) `is` 3 + test "not acct" $ (length $ entriesReport defreportopts (Not $ Acct "bank") samplejournal) @?= 1 + ,test "date" $ (length $ entriesReport defreportopts (Date $ mkdatespan "2008/06/01" "2008/07/01") samplejournal) @?= 3 ] ] diff --git a/Hledger/Reports/MultiBalanceReport.hs b/Hledger/Reports/MultiBalanceReport.hs --- a/Hledger/Reports/MultiBalanceReport.hs +++ b/Hledger/Reports/MultiBalanceReport.hs @@ -106,7 +106,7 @@ multiBalanceReportWith :: ReportOpts -> Query -> Journal -> PriceOracle -> MultiBalanceReport multiBalanceReportWith ropts@ReportOpts{..} q j@Journal{..} priceoracle = (if invert_ then mbrNegate else id) $ - MultiBalanceReport (colspans, sortedrows, totalsrow) + MultiBalanceReport (colspans, mappedsortedrows, mappedtotalsrow) where dbg1 s = let p = "multiBalanceReport" in Hledger.Utils.dbg1 (p++" "++s) -- add prefix in this function's debug output -- dbg1 = const id -- exclude this function from debug output @@ -162,7 +162,7 @@ -- These balances are unvalued except maybe converted to cost. startbals :: [(AccountName, MixedAmount)] = dbg1 "startbals" $ map (\(a,_,_,b) -> (a,b)) startbalanceitems where - (startbalanceitems,_) = dbg1 "starting balance report" $ balanceReport ropts''{value_=Nothing} startbalq j' + (startbalanceitems,_) = dbg1 "starting balance report" $ balanceReport ropts''{value_=Nothing, percent_=False} startbalq j' where ropts' | tree_ ropts = ropts{no_elide_=True} | otherwise = ropts{accountlistmode_=ALFlat} @@ -344,6 +344,25 @@ totalsrow :: MultiBalanceReportTotals = dbg1 "totalsrow" (coltotals, grandtotal, grandaverage) + ---------------------------------------------------------------------- + -- 9. Map the report rows to percentages if needed + -- It is not correct to do this before step 6 due to the total and average columns. + -- This is not done in step 6, since the report totals are calculated in 8. + + -- Perform the divisions to obtain percentages + mappedsortedrows :: [MultiBalanceReportRow] = + if not percent_ then sortedrows + else dbg1 "mappedsortedrows" + [(aname, alname, alevel, zipWith perdivide rowvals coltotals, rowtotal `perdivide` grandtotal, rowavg `perdivide` grandaverage) + | (aname, alname, alevel, rowvals, rowtotal, rowavg) <- sortedrows + ] + mappedtotalsrow :: MultiBalanceReportTotals = + if not percent_ then totalsrow + else dbg1 "mappedtotalsrow" ( + map (\t -> perdivide t t) coltotals, + perdivide grandtotal grandtotal, + perdivide grandaverage grandaverage) + -- | Given a MultiBalanceReport and its normal balance sign, -- if it is known to be normally negative, convert it to normally positive. mbrNormaliseSign :: NormalSign -> MultiBalanceReport -> MultiBalanceReport @@ -397,15 +416,15 @@ -- tests tests_MultiBalanceReport = tests "MultiBalanceReport" [ + let + amt0 = Amount {acommodity="$", aquantity=0, aprice=Nothing, astyle=AmountStyle {ascommodityside = L, ascommodityspaced = False, asprecision = 2, asdecimalpoint = Just '.', asdigitgroups = Nothing}, aismultiplier=False} (opts,journal) `gives` r = do let (eitems, etotal) = r (MultiBalanceReport (_, aitems, atotal)) = multiBalanceReport opts (queryFromOpts nulldate opts) journal showw (acct,acct',indent,lAmt,amt,amt') = (acct, acct', indent, map showMixedAmountDebug lAmt, showMixedAmountDebug amt, showMixedAmountDebug amt') - (map showw aitems) `is` (map showw eitems) - ((\(_, b, _) -> showMixedAmountDebug b) atotal) `is` (showMixedAmountDebug etotal) -- we only check the sum of the totals - usd0 = usd 0 - amount0 = Amount {acommodity="$", aquantity=0, aprice=Nothing, astyle=AmountStyle {ascommodityside = L, ascommodityspaced = False, asprecision = 2, asdecimalpoint = Just '.', asdigitgroups = Nothing}, aismultiplier=False} + (map showw aitems) @?= (map showw eitems) + ((\(_, b, _) -> showMixedAmountDebug b) atotal) @?= (showMixedAmountDebug etotal) -- we only check the sum of the totals in tests "multiBalanceReport" [ test "null journal" $ @@ -415,32 +434,32 @@ (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 1) (fromGregorian 2008 1 2), balancetype_=HistoricalBalance}, samplejournal) `gives` ( [ - ("assets:bank:checking", "checking", 3, [mamountp' "$1.00"] , Mixed [nullamt], Mixed [amount0 {aquantity=1}]) - ,("income:salary" ,"salary" , 2, [mamountp' "$-1.00"], Mixed [nullamt], Mixed [amount0 {aquantity=(-1)}]) + ("assets:bank:checking", "checking", 3, [mamountp' "$1.00"] , Mixed [nullamt], Mixed [amt0 {aquantity=1}]) + ,("income:salary" ,"salary" , 2, [mamountp' "$-1.00"], Mixed [nullamt], Mixed [amt0 {aquantity=(-1)}]) ], Mixed [nullamt]) - ,_test "a valid history on an empty period" $ - (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 2) (fromGregorian 2008 1 3), balancetype_=HistoricalBalance}, samplejournal) `gives` - ( - [ - ("assets:bank:checking","checking",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=1}]) - ,("income:salary","salary",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amount0 {aquantity=(-1)}]) - ], - Mixed [usd0]) + -- ,test "a valid history on an empty period" $ + -- (defreportopts{period_= PeriodBetween (fromGregorian 2008 1 2) (fromGregorian 2008 1 3), balancetype_=HistoricalBalance}, samplejournal) `gives` + -- ( + -- [ + -- ("assets:bank:checking","checking",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=1}]) + -- ,("income:salary","salary",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amt0 {aquantity=(-1)}]) + -- ], + -- Mixed [usd0]) - ,_test "a valid history on an empty period (more complex)" $ - (defreportopts{period_= PeriodBetween (fromGregorian 2009 1 1) (fromGregorian 2009 1 2), balancetype_=HistoricalBalance}, samplejournal) `gives` - ( - [ - ("assets:bank:checking","checking",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=1}]) - ,("assets:bank:saving","saving",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=1}]) - ,("assets:cash","cash",2, [mamountp' "$-2.00"], mamountp' "$-2.00",Mixed [amount0 {aquantity=(-2)}]) - ,("expenses:food","food",2, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=(1)}]) - ,("expenses:supplies","supplies",2, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amount0 {aquantity=(1)}]) - ,("income:gifts","gifts",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amount0 {aquantity=(-1)}]) - ,("income:salary","salary",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amount0 {aquantity=(-1)}]) - ], - Mixed [usd0]) + -- ,test "a valid history on an empty period (more complex)" $ + -- (defreportopts{period_= PeriodBetween (fromGregorian 2009 1 1) (fromGregorian 2009 1 2), balancetype_=HistoricalBalance}, samplejournal) `gives` + -- ( + -- [ + -- ("assets:bank:checking","checking",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=1}]) + -- ,("assets:bank:saving","saving",3, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=1}]) + -- ,("assets:cash","cash",2, [mamountp' "$-2.00"], mamountp' "$-2.00",Mixed [amt0 {aquantity=(-2)}]) + -- ,("expenses:food","food",2, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=(1)}]) + -- ,("expenses:supplies","supplies",2, [mamountp' "$1.00"], mamountp' "$1.00",Mixed [amt0 {aquantity=(1)}]) + -- ,("income:gifts","gifts",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amt0 {aquantity=(-1)}]) + -- ,("income:salary","salary",2, [mamountp' "$-1.00"], mamountp' "$-1.00",Mixed [amt0 {aquantity=(-1)}]) + -- ], + -- Mixed [usd0]) ] ] diff --git a/Hledger/Reports/PostingsReport.hs b/Hledger/Reports/PostingsReport.hs --- a/Hledger/Reports/PostingsReport.hs +++ b/Hledger/Reports/PostingsReport.hs @@ -270,22 +270,20 @@ tests_PostingsReport = tests "PostingsReport" [ - tests "postingsReport" $ - let (query, journal) `gives` n = (length $ snd $ postingsReport defreportopts query journal) `is` n - in [ - -- with the query specified explicitly - (Any, nulljournal) `gives` 0 - ,(Any, samplejournal) `gives` 13 - -- register --depth just clips account names - ,(Depth 2, samplejournal) `gives` 13 - ,(And [Depth 1, StatusQ Cleared, Acct "expenses"], samplejournal) `gives` 2 - ,(And [And [Depth 1, StatusQ Cleared], Acct "expenses"], samplejournal) `gives` 2 - - -- with query and/or command-line options - ,(length $ snd $ postingsReport defreportopts Any samplejournal) `is` 13 - ,(length $ snd $ postingsReport defreportopts{interval_=Months 1} Any samplejournal) `is` 11 - ,(length $ snd $ postingsReport defreportopts{interval_=Months 1, empty_=True} Any samplejournal) `is` 20 - ,(length $ snd $ postingsReport defreportopts (Acct "assets:bank:checking") samplejournal) `is` 5 + test "postingsReport" $ do + let (query, journal) `gives` n = (length $ snd $ postingsReport defreportopts query journal) @?= n + -- with the query specified explicitly + (Any, nulljournal) `gives` 0 + (Any, samplejournal) `gives` 13 + -- register --depth just clips account names + (Depth 2, samplejournal) `gives` 13 + (And [Depth 1, StatusQ Cleared, Acct "expenses"], samplejournal) `gives` 2 + (And [And [Depth 1, StatusQ Cleared], Acct "expenses"], samplejournal) `gives` 2 + -- with query and/or command-line options + (length $ snd $ postingsReport defreportopts Any samplejournal) @?= 13 + (length $ snd $ postingsReport defreportopts{interval_=Months 1} Any samplejournal) @?= 11 + (length $ snd $ postingsReport defreportopts{interval_=Months 1, empty_=True} Any samplejournal) @?= 20 + (length $ snd $ postingsReport defreportopts (Acct "assets:bank:checking") samplejournal) @?= 5 -- (defreportopts, And [Acct "a a", Acct "'b"], samplejournal2) `gives` 0 -- [(Just (parsedate "2008-01-01","income"),assets:bank:checking $1,$1) @@ -432,13 +430,9 @@ ] -} - ] - ,tests "summarisePostingsByInterval" [ - tests "summarisePostingsByInterval" [ - summarisePostingsByInterval (Quarters 1) PrimaryDate 99999 False (DateSpan Nothing Nothing) [] `is` [] - ] - ] + ,test "summarisePostingsByInterval" $ + summarisePostingsByInterval (Quarters 1) PrimaryDate 99999 False (DateSpan Nothing Nothing) [] @?= [] -- ,tests_summarisePostingsInDateSpan = [ -- "summarisePostingsInDateSpan" ~: do diff --git a/Hledger/Reports/ReportOptions.hs b/Hledger/Reports/ReportOptions.hs --- a/Hledger/Reports/ReportOptions.hs +++ b/Hledger/Reports/ReportOptions.hs @@ -4,7 +4,7 @@ -} -{-# LANGUAGE OverloadedStrings, RecordWildCards, DeriveDataTypeable #-} +{-# LANGUAGE OverloadedStrings, RecordWildCards, LambdaCase, DeriveDataTypeable #-} module Hledger.Reports.ReportOptions ( ReportOpts(..), @@ -38,6 +38,7 @@ reportPeriodLastDay, reportPeriodOrJournalLastDay, valuationTypeIsCost, + valuationTypeIsDefaultValue, tests_ReportOptions ) @@ -98,7 +99,8 @@ ,no_elide_ :: Bool ,real_ :: Bool ,format_ :: Maybe FormatStr - ,query_ :: String -- all arguments, as a string + ,query_ :: String -- ^ All query arguments space sepeareted + -- and quoted if needed (see 'quoteIfNeeded') -- ,average_ :: Bool -- register command only @@ -111,6 +113,7 @@ ,no_total_ :: Bool ,pretty_tables_ :: Bool ,sort_amount_ :: Bool + ,percent_ :: Bool ,invert_ :: Bool -- ^ if true, flip all amount signs in reports ,normalbalance_ :: Maybe NormalSign -- ^ This can be set when running balance reports on a set of accounts @@ -156,6 +159,7 @@ def def def + def rawOptsToReportOpts :: RawOpts -> IO ReportOpts rawOptsToReportOpts rawopts = checkReportOpts <$> do @@ -175,7 +179,7 @@ ,no_elide_ = boolopt "no-elide" rawopts' ,real_ = boolopt "real" rawopts' ,format_ = maybestringopt "format" rawopts' -- XXX move to CliOpts or move validation from Cli.CliOptions to here - ,query_ = unwords $ listofstringopt "args" rawopts' -- doesn't handle an arg like "" right + ,query_ = unwords . map quoteIfNeeded $ listofstringopt "args" rawopts' -- doesn't handle an arg like "" right ,average_ = boolopt "average" rawopts' ,related_ = boolopt "related" rawopts' ,balancetype_ = balancetypeopt rawopts' @@ -184,6 +188,7 @@ ,row_total_ = boolopt "row-total" rawopts' ,no_total_ = boolopt "no-total" rawopts' ,sort_amount_ = boolopt "sort-amount" rawopts' + ,percent_ = boolopt "percent" rawopts' ,invert_ = boolopt "invert" rawopts' ,pretty_tables_ = boolopt "pretty-tables" rawopts' ,color_ = color @@ -215,18 +220,20 @@ _ -> Right () accountlistmodeopt :: RawOpts -> AccountListMode -accountlistmodeopt rawopts = - case reverse $ filter (`elem` ["tree","flat"]) $ map fst rawopts of - ("tree":_) -> ALTree - ("flat":_) -> ALFlat - _ -> ALDefault +accountlistmodeopt = + fromMaybe ALDefault . choiceopt parse where + parse = \case + "tree" -> Just ALTree + "flat" -> Just ALFlat + _ -> Nothing balancetypeopt :: RawOpts -> BalanceType -balancetypeopt rawopts = - case reverse $ filter (`elem` ["change","cumulative","historical"]) $ map fst rawopts of - ("historical":_) -> HistoricalBalance - ("cumulative":_) -> CumulativeChange - _ -> PeriodChange +balancetypeopt = + fromMaybe PeriodChange . choiceopt parse where + parse = \case + "historical" -> Just HistoricalBalance + "cumulative" -> Just CumulativeChange + _ -> Nothing -- Get the period specified by any -b/--begin, -e/--end and/or -p/--period -- options appearing in the command line. @@ -252,7 +259,7 @@ -- Get all begin dates specified by -b/--begin or -p/--period options, in order, -- using the given date to interpret relative date expressions. beginDatesFromRawOpts :: Day -> RawOpts -> [Day] -beginDatesFromRawOpts d = catMaybes . map (begindatefromrawopt d) +beginDatesFromRawOpts d = collectopts (begindatefromrawopt d) where begindatefromrawopt d (n,v) | n == "begin" = @@ -270,7 +277,7 @@ -- Get all end dates specified by -e/--end or -p/--period options, in order, -- using the given date to interpret relative date expressions. endDatesFromRawOpts :: Day -> RawOpts -> [Day] -endDatesFromRawOpts d = catMaybes . map (enddatefromrawopt d) +endDatesFromRawOpts d = collectopts (enddatefromrawopt d) where enddatefromrawopt d (n,v) | n == "end" = @@ -289,7 +296,7 @@ -- -D/--daily, -W/--weekly, -M/--monthly etc. options. -- An interval from --period counts only if it is explicitly defined. intervalFromRawOpts :: RawOpts -> Interval -intervalFromRawOpts = lastDef NoInterval . catMaybes . map intervalfromrawopt +intervalFromRawOpts = lastDef NoInterval . collectopts intervalfromrawopt where intervalfromrawopt (n,v) | n == "period" = @@ -316,7 +323,7 @@ -- -P/--pending, -C/--cleared flags. -UPC is equivalent to no flags, -- so this returns a list of 0-2 unique statuses. statusesFromRawOpts :: RawOpts -> [Status] -statusesFromRawOpts = simplifyStatuses . catMaybes . map statusfromrawopt +statusesFromRawOpts = simplifyStatuses . collectopts statusfromrawopt where statusfromrawopt (n,_) | n == "unmarked" = Just Unmarked @@ -342,7 +349,7 @@ -- -B/--cost, -V, -X/--exchange, or --value flags. If there's more -- than one of these, the rightmost flag wins. valuationTypeFromRawOpts :: RawOpts -> Maybe ValuationType -valuationTypeFromRawOpts = lastDef Nothing . filter isJust . map valuationfromrawopt +valuationTypeFromRawOpts = lastMay . collectopts valuationfromrawopt where valuationfromrawopt (n,v) -- option name, value | n == "B" = Just $ AtCost Nothing @@ -371,6 +378,12 @@ Just (AtCost _) -> True _ -> False +valuationTypeIsDefaultValue :: ReportOpts -> Bool +valuationTypeIsDefaultValue ropts = + case value_ ropts of + Just (AtDefault _) -> True + _ -> False + type DisplayExp = String maybedisplayopt :: Day -> RawOpts -> Maybe DisplayExp @@ -408,7 +421,7 @@ journalSelectingAmountFromOpts :: ReportOpts -> Journal -> Journal journalSelectingAmountFromOpts opts = case value_ opts of - Just (AtCost _) -> journalConvertAmountsToCost + Just (AtCost _) -> journalToCost _ -> id -- | Convert report options and arguments to a query. @@ -526,23 +539,19 @@ -- tests tests_ReportOptions = tests "ReportOptions" [ - tests "queryFromOpts" [ - (queryFromOpts nulldate defreportopts) `is` Any - ,(queryFromOpts nulldate defreportopts{query_="a"}) `is` (Acct "a") - ,(queryFromOpts nulldate defreportopts{query_="desc:'a a'"}) `is` (Desc "a a") - ,(queryFromOpts nulldate defreportopts{period_=PeriodFrom (parsedate "2012/01/01"),query_="date:'to 2013'" }) - `is` (Date $ mkdatespan "2012/01/01" "2013/01/01") - ,(queryFromOpts nulldate defreportopts{query_="date2:'in 2012'"}) `is` (Date2 $ mkdatespan "2012/01/01" "2013/01/01") - ,(queryFromOpts nulldate defreportopts{query_="'a a' 'b"}) `is` (Or [Acct "a a", Acct "'b"]) - ] + test "queryFromOpts" $ do + queryFromOpts nulldate defreportopts @?= Any + queryFromOpts nulldate defreportopts{query_="a"} @?= Acct "a" + queryFromOpts nulldate defreportopts{query_="desc:'a a'"} @?= Desc "a a" + queryFromOpts nulldate defreportopts{period_=PeriodFrom (parsedate "2012/01/01"),query_="date:'to 2013'" } + @?= (Date $ mkdatespan "2012/01/01" "2013/01/01") + queryFromOpts nulldate defreportopts{query_="date2:'in 2012'"} @?= (Date2 $ mkdatespan "2012/01/01" "2013/01/01") + queryFromOpts nulldate defreportopts{query_="'a a' 'b"} @?= Or [Acct "a a", Acct "'b"] - ,tests "queryOptsFromOpts" [ - (queryOptsFromOpts nulldate defreportopts) `is` [] - ,(queryOptsFromOpts nulldate defreportopts{query_="a"}) `is` [] - ,(queryOptsFromOpts nulldate defreportopts{period_=PeriodFrom (parsedate "2012/01/01") - ,query_="date:'to 2013'" - }) - `is` [] - ] + ,test "queryOptsFromOpts" $ do + queryOptsFromOpts nulldate defreportopts @?= [] + queryOptsFromOpts nulldate defreportopts{query_="a"} @?= [] + queryOptsFromOpts nulldate defreportopts{period_=PeriodFrom (parsedate "2012/01/01") + ,query_="date:'to 2013'"} @?= [] ] diff --git a/Hledger/Utils.hs b/Hledger/Utils.hs --- a/Hledger/Utils.hs +++ b/Hledger/Utils.hs @@ -29,7 +29,7 @@ -- Debug.Trace.trace, -- module Data.PPrint, -- module Hledger.Utils.UTF8IOCompat - SystemString,fromSystemString,toSystemString,error',userError',usageError, + error',userError',usageError, -- the rest need to be done in each module I think ) where @@ -66,7 +66,7 @@ import Hledger.Utils.Tree -- import Prelude hiding (readFile,writeFile,appendFile,getContents,putStr,putStrLn) -- import Hledger.Utils.UTF8IOCompat (readFile,writeFile,appendFile,getContents,putStr,putStrLn) -import Hledger.Utils.UTF8IOCompat (SystemString,fromSystemString,toSystemString,error',userError',usageError) +import Hledger.Utils.UTF8IOCompat (error',userError',usageError) -- tuples diff --git a/Hledger/Utils/Parse.hs b/Hledger/Utils/Parse.hs --- a/Hledger/Utils/Parse.hs +++ b/Hledger/Utils/Parse.hs @@ -49,14 +49,15 @@ -- | A parser of strict text to some type. type SimpleTextParser = Parsec CustomErr Text -- XXX an "a" argument breaks the CsvRulesParser declaration somehow --- | A parser of text in some monad. +-- | A parser of text that runs in some monad. type TextParser m a = ParsecT CustomErr Text m a --- | A parser of text in some monad, with a journal as state. +-- | A parser of text that runs in some monad, keeping a Journal as state. type JournalParser m a = StateT Journal (ParsecT CustomErr Text m) a --- | A parser of text in some monad, with a journal as state, that can throw a --- "final" parse error that does not backtrack. +-- | A parser of text that runs in some monad, keeping a Journal as +-- state, that can throw an exception to end parsing, preventing +-- further parser backtracking. type ErroringJournalParser m a = StateT Journal (ParsecT CustomErr Text (ExceptT FinalParseError m)) a diff --git a/Hledger/Utils/String.hs b/Hledger/Utils/String.hs --- a/Hledger/Utils/String.hs +++ b/Hledger/Utils/String.hs @@ -21,6 +21,7 @@ lstrip, rstrip, chomp, + singleline, elideLeft, elideRight, formatString, @@ -75,6 +76,10 @@ -- | Remove trailing newlines/carriage returns. chomp :: String -> String chomp = reverse . dropWhile (`elem` "\r\n") . reverse + +-- | Remove consequtive line breaks, replacing them with single space +singleline :: String -> String +singleline = unwords . filter (/="") . (map strip) . lines stripbrackets :: String -> String stripbrackets = dropWhile (`elem` "([") . reverse . dropWhile (`elem` "])") . reverse :: String -> String diff --git a/Hledger/Utils/Test.hs b/Hledger/Utils/Test.hs --- a/Hledger/Utils/Test.hs +++ b/Hledger/Utils/Test.hs @@ -4,229 +4,192 @@ {-# LANGUAGE ScopedTypeVariables #-} module Hledger.Utils.Test ( - HasCallStack - ,module EasyTest - ,runEasytests + module Test.Tasty + ,module Test.Tasty.HUnit + -- ,module QC + -- ,module SC ,tests - ,_tests ,test - ,_test - ,it - ,_it - ,is - ,expectEqPP - ,expectParse - ,expectParseE - ,expectParseError - ,expectParseErrorE - ,expectParseEq - ,expectParseEqE - ,expectParseEqOn - ,expectParseEqOnE - ,expectParseStateOn + ,assertLeft + ,assertRight + ,assertParse + ,assertParseEq + ,assertParseEqOn + ,assertParseError + ,assertParseE + ,assertParseEqE + ,assertParseErrorE + ,assertParseStateOn ) where -import Control.Exception import Control.Monad.Except (ExceptT, runExceptT) import Control.Monad.State.Strict (StateT, evalStateT, execStateT) #if !(MIN_VERSION_base(4,11,0)) import Data.Monoid ((<>)) #endif -import Data.CallStack -import Data.List +-- import Data.CallStack +import Data.List (isInfixOf) import qualified Data.Text as T -import Safe -import System.Exit +import Test.Tasty hiding (defaultMain) +import Test.Tasty.HUnit +-- import Test.Tasty.QuickCheck as QC +-- import Test.Tasty.SmallCheck as SC import Text.Megaparsec import Text.Megaparsec.Custom - -import EasyTest hiding (char, char', tests) -- reexported -import qualified EasyTest as E -- used here + ( CustomErr, + FinalParseError, + attachSource, + customErrorBundlePretty, + finalErrorBundlePretty, + ) import Hledger.Utils.Debug (pshow) -import Hledger.Utils.UTF8IOCompat (error') +-- import Hledger.Utils.UTF8IOCompat (error') --- * easytest helpers +-- * tasty helpers --- | Name the given test(s). A readability synonym for easytest's "scope". -test :: T.Text -> E.Test a -> E.Test a -test = E.scope +-- TODO: pretty-print values in failure messages --- | Skip the given test(s), with the same type signature as "test". --- If called in a monadic sequence of tests, also skips following tests. -_test :: T.Text -> E.Test a -> E.Test a -_test _name = (E.skip >>) --- | Name the given test(s). A synonym for "test". -it :: T.Text -> E.Test a -> E.Test a -it = test - --- | Skip the given test(s), and any following tests in a monadic sequence. --- A synonym for "_test". -_it :: T.Text -> E.Test a -> E.Test a -_it = _test - --- | Name and group a list of tests. Combines easytest's "scope" and "tests". -tests :: T.Text -> [E.Test ()] -> E.Test () -tests name = E.scope name . E.tests - --- | Skip the given list of tests, and any following tests in a monadic sequence, --- with the same type signature as "group". -_tests :: T.Text -> [E.Test ()] -> E.Test () -_tests _name = (E.skip >>) . E.tests +-- | Name and group a list of tests. Shorter alias for Test.Tasty.HUnit.testGroup. +tests :: String -> [TestTree] -> TestTree +tests = testGroup --- | Run some easytest tests, catching easytest's ExitCode exception, --- returning True if there was a problem. --- With arguments, runs only the scope (or single test) named by the first argument --- (exact, case sensitive). --- If there is a second argument, it should be an integer and will be used --- as the seed for randomness. -runEasytests :: [String] -> E.Test () -> IO Bool -runEasytests args tests = (do - case args of - [] -> E.run tests - [a] -> E.runOnly (T.pack a) tests - a:b:_ -> do - case readMay b :: Maybe Int of - Nothing -> error' "the second argument should be an integer (a seed for easytest)" - Just seed -> E.rerunOnly seed (T.pack a) tests - return False - ) - `catch` (\(_::ExitCode) -> return True) +-- | Name an assertion or sequence of assertions. Shorter alias for Test.Tasty.HUnit.testCase. +test :: String -> Assertion -> TestTree +test = testCase --- | Like easytest's expectEq (asserts the second (actual) value equals the first (expected) value) --- but pretty-prints the values in the failure output. -expectEqPP :: (Eq a, Show a, HasCallStack) => a -> a -> E.Test () -expectEqPP expected actual = if expected == actual then E.ok else E.crash $ - "\nexpected:\n" <> T.pack (pshow expected) <> "\nbut got:\n" <> T.pack (pshow actual) <> "\n" +-- | Assert any Left value. +assertLeft :: (HasCallStack, Eq b, Show b) => Either a b -> Assertion +assertLeft (Left _) = return () +assertLeft (Right b) = assertFailure $ "expected Left, got (Right " ++ show b ++ ")" --- | Shorter and flipped version of expectEqPP. The expected value goes last. -is :: (Eq a, Show a, HasCallStack) => a -> a -> Test () -is = flip expectEqPP +-- | Assert any Right value. +assertRight :: (HasCallStack, Eq a, Show a) => Either a b -> Assertion +assertRight (Right _) = return () +assertRight (Left a) = assertFailure $ "expected Right, got (Left " ++ show a ++ ")" --- | Test that this stateful parser runnable in IO successfully parses +-- | Assert that this stateful parser runnable in IO successfully parses -- all of the given input text, showing the parse error if it fails. -- Suitable for hledger's JournalParser parsers. -expectParse :: (Monoid st, Eq a, Show a, HasCallStack) => - StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> E.Test () -expectParse parser input = do - ep <- E.io (runParserT (evalStateT (parser <* eof) mempty) "" input) - either (fail.(++"\n").("\nparse error at "++).customErrorBundlePretty) - (const ok) +assertParse :: (HasCallStack, Eq a, Show a, Monoid st) => + StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> Assertion +assertParse parser input = do + ep <- runParserT (evalStateT (parser <* eof) mempty) "" input + either (assertFailure.(++"\n").("\nparse error at "++).customErrorBundlePretty) + (const $ return ()) ep --- Suitable for hledger's ErroringJournalParser parsers. -expectParseE - :: (Monoid st, Eq a, Show a, HasCallStack) - => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a - -> T.Text - -> E.Test () -expectParseE parser input = do - let filepath = "" - eep <- E.io $ runExceptT $ - runParserT (evalStateT (parser <* eof) mempty) filepath input - case eep of - Left finalErr -> - let prettyErr = finalErrorBundlePretty $ attachSource filepath input finalErr - in fail $ "parse error at " <> prettyErr - Right ep -> - either (fail.(++"\n").("\nparse error at "++).customErrorBundlePretty) - (const ok) - ep +-- | Assert a parser produces an expected value. +assertParseEq :: (HasCallStack, Eq a, Show a, Monoid st) => + StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> a -> Assertion +assertParseEq parser input expected = assertParseEqOn parser input id expected --- | Test that this stateful parser runnable in IO fails to parse +-- | Like assertParseEq, but transform the parse result with the given function +-- before comparing it. +assertParseEqOn :: (HasCallStack, Eq b, Show b, Monoid st) => + StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> (a -> b) -> b -> Assertion +assertParseEqOn parser input f expected = do + ep <- runParserT (evalStateT (parser <* eof) mempty) "" input + either (assertFailure . (++"\n") . ("\nparse error at "++) . customErrorBundlePretty) + (assertEqual "" expected . f) + ep + +-- | Assert that this stateful parser runnable in IO fails to parse -- the given input text, with a parse error containing the given string. -expectParseError :: (Monoid st, Eq a, Show a, HasCallStack) => - StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> String -> E.Test () -expectParseError parser input errstr = do - ep <- E.io (runParserT (evalStateT parser mempty) "" input) +assertParseError :: (HasCallStack, Eq a, Show a, Monoid st) => + StateT st (ParsecT CustomErr T.Text IO) a -> String -> String -> Assertion +assertParseError parser input errstr = do + ep <- runParserT (evalStateT parser mempty) "" (T.pack input) case ep of - Right v -> fail $ "\nparse succeeded unexpectedly, producing:\n" ++ pshow v ++ "\n" + Right v -> assertFailure $ "\nparse succeeded unexpectedly, producing:\n" ++ pshow v ++ "\n" Left e -> do let e' = customErrorBundlePretty e if errstr `isInfixOf` e' - then ok - else fail $ "\nparse error is not as expected:\n" ++ e' ++ "\n" + then return () + else assertFailure $ "\nparse error is not as expected:\n" ++ e' ++ "\n" -expectParseErrorE - :: (Monoid st, Eq a, Show a, HasCallStack) +-- | Run a stateful parser in IO like assertParse, then assert that the +-- final state (the wrapped state, not megaparsec's internal state), +-- transformed by the given function, matches the given expected value. +assertParseStateOn :: (HasCallStack, Eq b, Show b, Monoid st) => + StateT st (ParsecT CustomErr T.Text IO) a + -> T.Text + -> (st -> b) + -> b + -> Assertion +assertParseStateOn parser input f expected = do + es <- runParserT (execStateT (parser <* eof) mempty) "" input + case es of + Left err -> assertFailure $ (++"\n") $ ("\nparse error at "++) $ customErrorBundlePretty err + Right s -> assertEqual "" expected $ f s + +-- | These "E" variants of the above are suitable for hledger's ErroringJournalParser parsers. +assertParseE + :: (HasCallStack, Eq a, Show a, Monoid st) => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a -> T.Text - -> String - -> E.Test () -expectParseErrorE parser input errstr = do + -> Assertion +assertParseE parser input = do let filepath = "" - eep <- E.io $ runExceptT $ runParserT (evalStateT parser mempty) filepath input + eep <- runExceptT $ + runParserT (evalStateT (parser <* eof) mempty) filepath input case eep of - Left finalErr -> do + Left finalErr -> let prettyErr = finalErrorBundlePretty $ attachSource filepath input finalErr - if errstr `isInfixOf` prettyErr - then ok - else fail $ "\nparse error is not as expected:\n" ++ prettyErr ++ "\n" - Right ep -> case ep of - Right v -> fail $ "\nparse succeeded unexpectedly, producing:\n" ++ pshow v ++ "\n" - Left e -> do - let e' = customErrorBundlePretty e - if errstr `isInfixOf` e' - then ok - else fail $ "\nparse error is not as expected:\n" ++ e' ++ "\n" - --- | Like expectParse, but also test the parse result is an expected value, --- pretty-printing both if it fails. -expectParseEq :: (Monoid st, Eq a, Show a, HasCallStack) => - StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> a -> E.Test () -expectParseEq parser input expected = expectParseEqOn parser input id expected + in assertFailure $ "parse error at " <> prettyErr + Right ep -> + either (assertFailure.(++"\n").("\nparse error at "++).customErrorBundlePretty) + (const $ return ()) + ep -expectParseEqE +assertParseEqE :: (Monoid st, Eq a, Show a, HasCallStack) => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a -> T.Text -> a - -> E.Test () -expectParseEqE parser input expected = expectParseEqOnE parser input id expected - --- | Like expectParseEq, but transform the parse result with the given function --- before comparing it. -expectParseEqOn :: (Monoid st, Eq b, Show b, HasCallStack) => - StateT st (ParsecT CustomErr T.Text IO) a -> T.Text -> (a -> b) -> b -> E.Test () -expectParseEqOn parser input f expected = do - ep <- E.io $ runParserT (evalStateT (parser <* eof) mempty) "" input - either (fail . (++"\n") . ("\nparse error at "++) . customErrorBundlePretty) - (expectEqPP expected . f) - ep + -> Assertion +assertParseEqE parser input expected = assertParseEqOnE parser input id expected -expectParseEqOnE - :: (Monoid st, Eq b, Show b, HasCallStack) +assertParseEqOnE + :: (HasCallStack, Eq b, Show b, Monoid st) => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a -> T.Text -> (a -> b) -> b - -> E.Test () -expectParseEqOnE parser input f expected = do + -> Assertion +assertParseEqOnE parser input f expected = do let filepath = "" - eep <- E.io $ runExceptT $ - runParserT (evalStateT (parser <* eof) mempty) filepath input + eep <- runExceptT $ runParserT (evalStateT (parser <* eof) mempty) filepath input case eep of Left finalErr -> let prettyErr = finalErrorBundlePretty $ attachSource filepath input finalErr - in fail $ "parse error at " <> prettyErr + in assertFailure $ "parse error at " <> prettyErr Right ep -> - either (fail . (++"\n") . ("\nparse error at "++) . customErrorBundlePretty) - (expectEqPP expected . f) + either (assertFailure . (++"\n") . ("\nparse error at "++) . customErrorBundlePretty) + (assertEqual "" expected . f) ep --- | Run a stateful parser in IO like expectParse, then compare the --- final state (the wrapped state, not megaparsec's internal state), --- transformed by the given function, with the given expected value. -expectParseStateOn :: (HasCallStack, Monoid st, Eq b, Show b) => - StateT st (ParsecT CustomErr T.Text IO) a +assertParseErrorE + :: (Monoid st, Eq a, Show a, HasCallStack) + => StateT st (ParsecT CustomErr T.Text (ExceptT FinalParseError IO)) a -> T.Text - -> (st -> b) - -> b - -> E.Test () -expectParseStateOn parser input f expected = do - es <- E.io $ runParserT (execStateT (parser <* eof) mempty) "" input - case es of - Left err -> fail $ (++"\n") $ ("\nparse error at "++) $ customErrorBundlePretty err - Right s -> expectEqPP expected $ f s + -> String + -> Assertion +assertParseErrorE parser input errstr = do + let filepath = "" + eep <- runExceptT $ runParserT (evalStateT parser mempty) filepath input + case eep of + Left finalErr -> do + let prettyErr = finalErrorBundlePretty $ attachSource filepath input finalErr + if errstr `isInfixOf` prettyErr + then return () + else assertFailure $ "\nparse error is not as expected:\n" ++ prettyErr ++ "\n" + Right ep -> case ep of + Right v -> assertFailure $ "\nparse succeeded unexpectedly, producing:\n" ++ pshow v ++ "\n" + Left e -> do + let e' = customErrorBundlePretty e + if errstr `isInfixOf` e' + then return () + else assertFailure $ "\nparse error is not as expected:\n" ++ e' ++ "\n" diff --git a/Hledger/Utils/Text.hs b/Hledger/Utils/Text.hs --- a/Hledger/Utils/Text.hs +++ b/Hledger/Utils/Text.hs @@ -421,13 +421,12 @@ tests_Text = tests "Text" [ - tests "quoteIfSpaced" [ - quoteIfSpaced "a'a" `is` "a'a" - ,quoteIfSpaced "a\"a" `is` "a\"a" - ,quoteIfSpaced "a a" `is` "\"a a\"" - ,quoteIfSpaced "mimi's cafe" `is` "\"mimi's cafe\"" - ,quoteIfSpaced "\"alex\" cafe" `is` "\"\\\"alex\\\" cafe\"" - ,quoteIfSpaced "le'shan's cafe" `is` "\"le'shan's cafe\"" - ,quoteIfSpaced "\"be'any's\" cafe" `is` "\"\\\"be'any's\\\" cafe\"" - ] + test "quoteIfSpaced" $ do + quoteIfSpaced "a'a" @?= "a'a" + quoteIfSpaced "a\"a" @?= "a\"a" + quoteIfSpaced "a a" @?= "\"a a\"" + quoteIfSpaced "mimi's cafe" @?= "\"mimi's cafe\"" + quoteIfSpaced "\"alex\" cafe" @?= "\"\\\"alex\\\" cafe\"" + quoteIfSpaced "le'shan's cafe" @?= "\"le'shan's cafe\"" + quoteIfSpaced "\"be'any's\" cafe" @?= "\"\\\"be'any's\\\" cafe\"" ] diff --git a/Hledger/Utils/UTF8IOCompat.hs b/Hledger/Utils/UTF8IOCompat.hs --- a/Hledger/Utils/UTF8IOCompat.hs +++ b/Hledger/Utils/UTF8IOCompat.hs @@ -15,6 +15,9 @@ do the right thing, so this file is a no-op and on its way to being removed. Not carefully tested. +2019/10/20 update: all packages have base>=4.9 which corresponds to GHC v8.0.1 +and higher. Tear this file apart! + -} -- TODO obsolete ? @@ -29,9 +32,6 @@ hPutStr, hPutStrLn, -- - SystemString, - fromSystemString, - toSystemString, error', userError', usageError, @@ -85,37 +85,19 @@ -- bs_hPutStr = B8.hPut -- bs_hPutStrLn h bs = B8.hPut h bs >> B8.hPut h (B.singleton 0x0a) - --- | A string received from or being passed to the operating system, such --- as a file path, command-line argument, or environment variable name or --- value. With GHC versions before 7.2 on some platforms (posix) these are --- typically encoded. When converting, we assume the encoding is UTF-8 (cf --- ). -type SystemString = String - --- | Convert a system string to an ordinary string, decoding from UTF-8 if --- it appears to be UTF8-encoded and GHC version is less than 7.2. -fromSystemString :: SystemString -> String -fromSystemString = id - --- | Convert a unicode string to a system string, encoding with UTF-8 if --- we are on a posix platform with GHC < 7.2. -toSystemString :: String -> SystemString -toSystemString = id - -- | A SystemString-aware version of error. error' :: String -> a error' = #if __GLASGOW_HASKELL__ < 800 -- (easier than if base < 4.9) - error . toSystemString + error #else - errorWithoutStackTrace . toSystemString + errorWithoutStackTrace #endif -- | A SystemString-aware version of userError. userError' :: String -> IOError -userError' = userError . toSystemString +userError' = userError -- | A SystemString-aware version of error that adds a usage hint. usageError :: String -> a diff --git a/hledger-lib.cabal b/hledger-lib.cabal --- a/hledger-lib.cabal +++ b/hledger-lib.cabal @@ -4,17 +4,17 @@ -- -- see: https://github.com/sol/hpack -- --- hash: 567ed725b211714a0f6db5e17a68d670789c7e603020b42d6b8f18e7af5ceb63 +-- hash: 3228f8dbb178d427e76291b5e60b3dee1eb4d2e5f9ab803ce8e3fe85e79f25ad name: hledger-lib -version: 1.15.2 +version: 1.16 synopsis: Core data types, parsers and functionality for the hledger accounting tools description: This is a reusable library containing hledger's core functionality. . hledger is a cross-platform program for tracking money, time, or any other commodity, using double-entry accounting and a simple, editable file format. It is inspired by and largely compatible - with ledger(1). hledger provides command-line, curses and web + with ledger(1). hledger provides command-line, terminal and web interfaces, and aims to be a reliable, practical tool for daily use. category: Finance @@ -25,11 +25,13 @@ maintainer: Simon Michael license: GPL-3 license-file: LICENSE -tested-with: GHC==7.10.3, GHC==8.0.2, GHC==8.2.2, GHC==8.4.3, GHC==8.6.5 +tested-with: GHC==8.0.2, GHC==8.2.2, GHC==8.4.3, GHC==8.6.5, GHC==8.8.1 build-type: Simple extra-source-files: CHANGES.md README + test/unittest.hs + test/doctests.hs hledger_csv.5 hledger_csv.txt hledger_csv.info @@ -107,8 +109,8 @@ , Glob >=0.9 , ansi-terminal >=0.6.2.3 , array - , base >=4.8 && <4.13 - , base-compat-batteries >=0.10.1 && <0.11 + , base >=4.9 && <4.14 + , base-compat-batteries >=0.10.1 && <0.12 , blaze-markup >=0.5.1 , bytestring , call-stack @@ -119,7 +121,6 @@ , data-default >=0.5 , deepseq , directory - , easytest >=0.2.1 && <0.3 , extra >=1.6.3 , fgl >=5.5.4.0 , file-embed >=0.0.10 @@ -136,6 +137,8 @@ , safe >=0.2 , split >=0.1 , tabular >=0.2 + , tasty >=1.2.3 + , tasty-hunit >=0.10.0.2 , template-haskell , text >=1.2 , time >=1.5 @@ -143,64 +146,11 @@ , transformers >=0.2 , uglymemo , utf8-string >=0.3.5 - if (!impl(ghc >= 8.0)) - build-depends: - semigroups ==0.18.* default-language: Haskell2010 -test-suite doctests +test-suite doctest type: exitcode-stdio-1.0 main-is: doctests.hs - other-modules: - Hledger - Hledger.Data - Hledger.Data.Account - Hledger.Data.AccountName - Hledger.Data.Amount - Hledger.Data.Commodity - Hledger.Data.Dates - Hledger.Data.Journal - Hledger.Data.Ledger - Hledger.Data.Period - Hledger.Data.PeriodicTransaction - Hledger.Data.Posting - Hledger.Data.RawOptions - Hledger.Data.StringFormat - Hledger.Data.Timeclock - Hledger.Data.Transaction - Hledger.Data.TransactionModifier - Hledger.Data.Types - Hledger.Data.Valuation - Hledger.Query - Hledger.Read - Hledger.Read.Common - Hledger.Read.CsvReader - Hledger.Read.JournalReader - Hledger.Read.TimeclockReader - Hledger.Read.TimedotReader - Hledger.Reports - Hledger.Reports.AccountTransactionsReport - Hledger.Reports.BalanceReport - Hledger.Reports.BudgetReport - Hledger.Reports.EntriesReport - Hledger.Reports.MultiBalanceReport - Hledger.Reports.PostingsReport - Hledger.Reports.ReportOptions - Hledger.Reports.ReportTypes - Hledger.Reports.TransactionsReport - Hledger.Utils - Hledger.Utils.Color - Hledger.Utils.Debug - Hledger.Utils.Parse - Hledger.Utils.Regex - Hledger.Utils.String - Hledger.Utils.Test - Hledger.Utils.Text - Hledger.Utils.Tree - Hledger.Utils.UTF8IOCompat - Text.Megaparsec.Custom - Text.Tabular.AsciiWide - Paths_hledger_lib hs-source-dirs: ./. test @@ -210,8 +160,8 @@ , Glob >=0.7 , ansi-terminal >=0.6.2.3 , array - , base >=4.8 && <4.13 - , base-compat-batteries >=0.10.1 && <0.11 + , base >=4.9 && <4.14 + , base-compat-batteries >=0.10.1 && <0.12 , blaze-markup >=0.5.1 , bytestring , call-stack @@ -223,7 +173,6 @@ , deepseq , directory , doctest >=0.16 - , easytest >=0.2.1 && <0.3 , extra >=1.6.3 , fgl >=5.5.4.0 , file-embed >=0.0.10 @@ -240,6 +189,8 @@ , safe >=0.2 , split >=0.1 , tabular >=0.2 + , tasty >=1.2.3 + , tasty-hunit >=0.10.0.2 , template-haskell , text >=1.2 , time >=1.5 @@ -247,66 +198,14 @@ , transformers >=0.2 , uglymemo , utf8-string >=0.3.5 - if (!impl(ghc >= 8.0)) - build-depends: - semigroups ==0.18.* + buildable: False if (impl(ghc < 8.2)) buildable: False default-language: Haskell2010 -test-suite easytests +test-suite unittest type: exitcode-stdio-1.0 - main-is: easytests.hs - other-modules: - Hledger - Hledger.Data - Hledger.Data.Account - Hledger.Data.AccountName - Hledger.Data.Amount - Hledger.Data.Commodity - Hledger.Data.Dates - Hledger.Data.Journal - Hledger.Data.Ledger - Hledger.Data.Period - Hledger.Data.PeriodicTransaction - Hledger.Data.Posting - Hledger.Data.RawOptions - Hledger.Data.StringFormat - Hledger.Data.Timeclock - Hledger.Data.Transaction - Hledger.Data.TransactionModifier - Hledger.Data.Types - Hledger.Data.Valuation - Hledger.Query - Hledger.Read - Hledger.Read.Common - Hledger.Read.CsvReader - Hledger.Read.JournalReader - Hledger.Read.TimeclockReader - Hledger.Read.TimedotReader - Hledger.Reports - Hledger.Reports.AccountTransactionsReport - Hledger.Reports.BalanceReport - Hledger.Reports.BudgetReport - Hledger.Reports.EntriesReport - Hledger.Reports.MultiBalanceReport - Hledger.Reports.PostingsReport - Hledger.Reports.ReportOptions - Hledger.Reports.ReportTypes - Hledger.Reports.TransactionsReport - Hledger.Utils - Hledger.Utils.Color - Hledger.Utils.Debug - Hledger.Utils.Parse - Hledger.Utils.Regex - Hledger.Utils.String - Hledger.Utils.Test - Hledger.Utils.Text - Hledger.Utils.Tree - Hledger.Utils.UTF8IOCompat - Text.Megaparsec.Custom - Text.Tabular.AsciiWide - Paths_hledger_lib + main-is: unittest.hs hs-source-dirs: ./. test @@ -316,8 +215,8 @@ , Glob >=0.9 , ansi-terminal >=0.6.2.3 , array - , base >=4.8 && <4.13 - , base-compat-batteries >=0.10.1 && <0.11 + , base >=4.9 && <4.14 + , base-compat-batteries >=0.10.1 && <0.12 , blaze-markup >=0.5.1 , bytestring , call-stack @@ -328,7 +227,6 @@ , data-default >=0.5 , deepseq , directory - , easytest >=0.2.1 && <0.3 , extra >=1.6.3 , fgl >=5.5.4.0 , file-embed >=0.0.10 @@ -346,6 +244,8 @@ , safe >=0.2 , split >=0.1 , tabular >=0.2 + , tasty >=1.2.3 + , tasty-hunit >=0.10.0.2 , template-haskell , text >=1.2 , time >=1.5 @@ -353,7 +253,5 @@ , transformers >=0.2 , uglymemo , utf8-string >=0.3.5 - if (!impl(ghc >= 8.0)) - build-depends: - semigroups ==0.18.* + buildable: True default-language: Haskell2010 diff --git a/hledger_csv.5 b/hledger_csv.5 --- a/hledger_csv.5 +++ b/hledger_csv.5 @@ -1,364 +1,979 @@ - -.TH "hledger_csv" "5" "September 2019" "hledger 1.15.2" "hledger User Manuals" - - - -.SH NAME -.PP -CSV - how hledger reads CSV data, and the CSV rules file format -.SH DESCRIPTION -.PP -hledger can read CSV (comma-separated value) files as if they were -journal files, automatically converting each CSV record into a -transaction. -(To learn about \f[I]writing\f[R] CSV, see CSV output.) -.PP -Converting CSV to transactions requires some special conversion rules. -These do several things: -.IP \[bu] 2 -they describe the layout and format of the CSV data -.IP \[bu] 2 -they can customize the generated journal entries using a simple -templating language -.IP \[bu] 2 -they can add refinements based on patterns in the CSV data, eg -categorizing transactions with more detailed account names. -.PP -When reading a CSV file named \f[C]FILE.csv\f[R], hledger looks for a -conversion rules file named \f[C]FILE.csv.rules\f[R] in the same -directory. -You can override this with the \f[C]--rules-file\f[R] option. -If the rules file does not exist, hledger will auto-create one with some -example rules, which you\[aq]ll need to adjust. -.PP -At minimum, the rules file must identify the date and amount fields. -It\[aq]s often necessary to specify the date format, and the number of -header lines to skip, also. -Eg: -.IP -.nf -\f[C] -fields date, _, _, amount -date-format %d/%m/%Y -skip 1 -\f[R] -.fi -.PP -A more complete example: -.IP -.nf -\f[C] -# hledger CSV rules for amazon.com order history - -# sample: -# \[dq]Date\[dq],\[dq]Type\[dq],\[dq]To/From\[dq],\[dq]Name\[dq],\[dq]Status\[dq],\[dq]Amount\[dq],\[dq]Fees\[dq],\[dq]Transaction ID\[dq] -# \[dq]Jul 29, 2012\[dq],\[dq]Payment\[dq],\[dq]To\[dq],\[dq]Adapteva, Inc.\[dq],\[dq]Completed\[dq],\[dq]$25.00\[dq],\[dq]$0.00\[dq],\[dq]17LA58JSK6PRD4HDGLNJQPI1PB9N8DKPVHL\[dq] - -# skip one header line -skip 1 - -# name the csv fields (and assign the transaction\[aq]s date, amount and code) -fields date, _, toorfrom, name, amzstatus, amount, fees, code - -# how to parse the date -date-format %b %-d, %Y - -# combine two fields to make the description -description %toorfrom %name - -# save these fields as tags -comment status:%amzstatus, fees:%fees - -# set the base account for all transactions -account1 assets:amazon - -# flip the sign on the amount -amount -%amount -\f[R] -.fi -.PP -For more examples, see Convert CSV files. -.SH CSV RULES -.PP -The following seven kinds of rule can appear in the rules file, in any -order. -Blank lines and lines beginning with \f[C]#\f[R] or \f[C];\f[R] are -ignored. -.SS skip -.PP -\f[C]skip\f[R]\f[I]\f[CI]N\f[I]\f[R] -.PP -Skip this number of CSV records at the beginning. -You\[aq]ll need this whenever your CSV data contains header lines. -Eg: -.IP -.nf -\f[C] -# ignore the first CSV line -skip 1 -\f[R] -.fi -.SS date-format -.PP -\f[C]date-format\f[R]\f[I]\f[CI]DATEFMT\f[I]\f[R] -.PP -When your CSV date fields are not formatted like \f[C]YYYY/MM/DD\f[R] -(or \f[C]YYYY-MM-DD\f[R] or \f[C]YYYY.MM.DD\f[R]), you\[aq]ll need to -specify the format. -DATEFMT is a strptime-like date parsing pattern, which must parse the -date field values completely. -Examples: -.IP -.nf -\f[C] -# for dates like \[dq]11/06/2013\[dq]: -date-format %m/%d/%Y -\f[R] -.fi -.IP -.nf -\f[C] -# for dates like \[dq]6/11/2013\[dq] (note the - to make leading zeros optional): -date-format %-d/%-m/%Y -\f[R] -.fi -.IP -.nf -\f[C] -# for dates like \[dq]2013-Nov-06\[dq]: -date-format %Y-%h-%d -\f[R] -.fi -.IP -.nf -\f[C] -# for dates like \[dq]11/6/2013 11:32 PM\[dq]: -date-format %-m/%-d/%Y %l:%M %p -\f[R] -.fi -.SS field list -.PP -\f[C]fields\f[R]\f[I]\f[CI]FIELDNAME1\f[I]\f[R], -\f[I]\f[CI]FIELDNAME2\f[I]\f[R]... -.PP -This (a) names the CSV fields, in order (names may not contain -whitespace; uninteresting names may be left blank), and (b) assigns them -to journal entry fields if you use any of these standard field names: -\f[C]date\f[R], \f[C]date2\f[R], \f[C]status\f[R], \f[C]code\f[R], -\f[C]description\f[R], \f[C]comment\f[R], \f[C]account1\f[R], -\f[C]account2\f[R], \f[C]amount\f[R], \f[C]amount-in\f[R], -\f[C]amount-out\f[R], \f[C]currency\f[R], \f[C]balance\f[R], -\f[C]balance1\f[R], \f[C]balance2\f[R]. -Eg: -.IP -.nf -\f[C] -# use the 1st, 2nd and 4th CSV fields as the entry\[aq]s date, description and amount, -# and give the 7th and 8th fields meaningful names for later reference: -# -# CSV field: -# 1 2 3 4 5 6 7 8 -# entry field: -fields date, description, , amount, , , somefield, anotherfield -\f[R] -.fi -.SS field assignment -.PP -\f[I]\f[CI]ENTRYFIELDNAME\f[I]\f[R] \f[I]\f[CI]FIELDVALUE\f[I]\f[R] -.PP -This sets a journal entry field (one of the standard names above) to the -given text value, which can include CSV field values interpolated by -name (\f[C]%CSVFIELDNAME\f[R]) or 1-based position (\f[C]%N\f[R]). -Eg: -.IP -.nf -\f[C] -# set the amount to the 4th CSV field with \[dq]USD \[dq] prepended -amount USD %4 -\f[R] -.fi -.IP -.nf -\f[C] -# combine three fields to make a comment (containing two tags) -comment note: %somefield - %anotherfield, date: %1 -\f[R] -.fi -.PP -Field assignments can be used instead of or in addition to a field list. -.PP -Note, interpolation strips any outer whitespace, so a CSV value like -\f[C]\[dq] 1 \[dq]\f[R] becomes \f[C]1\f[R] when interpolated (#1051). -.SS conditional block -.PP -\f[C]if\f[R] \f[I]\f[CI]PATTERN\f[I]\f[R] -.PD 0 -.P -.PD -\ \ \ \ \f[I]\f[CI]FIELDASSIGNMENTS\f[I]\f[R]... -.PP -\f[C]if\f[R] -.PD 0 -.P -.PD -\f[I]\f[CI]PATTERN\f[I]\f[R] -.PD 0 -.P -.PD -\f[I]\f[CI]PATTERN\f[I]\f[R]... -.PD 0 -.P -.PD -\ \ \ \ \f[I]\f[CI]FIELDASSIGNMENTS\f[I]\f[R]... -.PP -This applies one or more field assignments, only to those CSV records -matched by one of the PATTERNs. -The patterns are case-insensitive regular expressions which match -anywhere within the whole CSV record (it\[aq]s not yet possible to match -within a specific field). -When there are multiple patterns they can be written on separate lines, -unindented. -The field assignments are on separate lines indented by at least one -space. -Examples: -.IP -.nf -\f[C] -# if the CSV record contains \[dq]groceries\[dq], set account2 to \[dq]expenses:groceries\[dq] -if groceries - account2 expenses:groceries -\f[R] -.fi -.IP -.nf -\f[C] -# if the CSV record contains any of these patterns, set account2 and comment as shown -if -monthly service fee -atm transaction fee -banking thru software - account2 expenses:business:banking - comment XXX deductible ? check it -\f[R] -.fi -.SS include -.PP -\f[C]include\f[R]\f[I]\f[CI]RULESFILE\f[I]\f[R] -.PP -Include another rules file at this point. -\f[C]RULESFILE\f[R] is either an absolute file path or a path relative -to the current file\[aq]s directory. -Eg: -.IP -.nf -\f[C] -# rules reused with several CSV files -include common.rules -\f[R] -.fi -.SS newest-first -.PP -\f[C]newest-first\f[R] -.PP -Consider adding this rule if all of the following are true: you might be -processing just one day of data, your CSV records are in reverse -chronological order (newest first), and you care about preserving the -order of same-day transactions. -It usually isn\[aq]t needed, because hledger autodetects the CSV order, -but when all CSV records have the same date it will assume they are -oldest first. -.SH CSV TIPS -.SS CSV ordering -.PP -The generated journal entries will be sorted by date. -The order of same-day entries will be preserved (except in the special -case where you might need \f[C]newest-first\f[R], see above). -.SS CSV accounts -.PP -Each journal entry will have two postings, to \f[C]account1\f[R] and -\f[C]account2\f[R] respectively. -It\[aq]s not yet possible to generate entries with more than two -postings. -It\[aq]s conventional and recommended to use \f[C]account1\f[R] for the -account whose CSV we are reading. -.SS CSV amounts -.PP -A transaction amount must be set, in one of these ways: -.IP \[bu] 2 -with an \f[C]amount\f[R] field assignment, which sets the first -posting\[aq]s amount -.IP \[bu] 2 -(When the CSV has debit and credit amounts in separate fields:) -.PD 0 -.P -.PD -with field assignments for the \f[C]amount-in\f[R] and -\f[C]amount-out\f[R] pseudo fields (both of them). -Whichever one has a value will be used, with appropriate sign. -If both contain a value, it might not work so well. -.IP \[bu] 2 -or implicitly by means of a balance assignment (see below). -.PP -There is some special handling for sign in amounts: -.IP \[bu] 2 -If an amount value is parenthesised, it will be de-parenthesised and -sign-flipped. -.IP \[bu] 2 -If an amount value begins with a double minus sign, those will cancel -out and be removed. -.PP -If the currency/commodity symbol is provided as a separate CSV field, -assign it to the \f[C]currency\f[R] pseudo field; the symbol will be -prepended to the amount (TODO: when there is an amount). -Or, you can use an \f[C]amount\f[R] field assignment for more control, -eg: -.IP -.nf -\f[C] -fields date,description,currency,amount -amount %amount %currency -\f[R] -.fi -.SS CSV balance assertions/assignments -.PP -If the CSV includes a running balance, you can assign that to one of the -pseudo fields \f[C]balance\f[R] (or \f[C]balance1\f[R]) or -\f[C]balance2\f[R]. -This will generate a balance assertion (or if the amount is left empty, -a balance assignment), on the first or second posting, whenever the -running balance field is non-empty. -(TODO: #1000) -.SS Reading multiple CSV files -.PP -You can read multiple CSV files at once using multiple \f[C]-f\f[R] -arguments on the command line, and hledger will look for a -correspondingly-named rules file for each. -Note if you use the \f[C]--rules-file\f[R] option, this one rules file -will be used for all the CSV files being read. -.SS Valid CSV -.PP -hledger follows RFC 4180, with the addition of a customisable separator -character. -.PP -Some things to note: -.PP -When quoting fields, -.IP \[bu] 2 -you must use double quotes, not single quotes -.IP \[bu] 2 -spaces outside the quotes are not allowed. - - -.SH "REPORTING BUGS" -Report bugs at http://bugs.hledger.org -(or on the #hledger IRC channel or hledger mail list) - -.SH AUTHORS -Simon Michael and contributors - -.SH COPYRIGHT - -Copyright (C) 2007-2016 Simon Michael. +.\"t + +.TH "hledger_csv" "5" "December 2019" "hledger 1.16" "hledger User Manuals" + + + +.SH NAME +.PP +CSV - how hledger reads CSV data, and the CSV rules file format +.SH DESCRIPTION +.PP +hledger can read CSV (comma-separated value, or character-separated +value) files as if they were journal files, automatically converting +each CSV record into a transaction. +(To learn about \f[I]writing\f[R] CSV, see CSV output.) +.PP +We describe each CSV file\[aq]s format with a corresponding \f[I]rules +file\f[R]. +By default this is named like the CSV file with a \f[C].rules\f[R] +extension added. +Eg when reading \f[C]FILE.csv\f[R], hledger also looks for +\f[C]FILE.csv.rules\f[R] in the same directory. +You can specify a different rules file with the \f[C]--rules-file\f[R] +option. +If a rules file is not found, hledger will create a sample rules file, +which you\[aq]ll need to adjust. +.PP +This file contains rules describing the CSV data (header line, fields +layout, date format etc.), and how to construct hledger journal entries +(transactions) from it. +Often there will also be a list of conditional rules for categorising +transactions based on their descriptions. +Here\[aq]s an overview of the CSV rules; these are described more fully +below, after the examples: +.PP +.TS +tab(@); +l l. +T{ +\f[B]\f[CB]skip\f[B]\f[R] +T}@T{ +skip one or more header lines or matched CSV records +T} +T{ +\f[B]\f[CB]fields\f[B]\f[R] +T}@T{ +name CSV fields, assign them to hledger fields +T} +T{ +\f[B]field assignment\f[R] +T}@T{ +assign a value to one hledger field, with interpolation +T} +T{ +\f[B]\f[CB]if\f[B]\f[R] +T}@T{ +apply some rules to matched CSV records +T} +T{ +\f[B]\f[CB]end\f[B]\f[R] +T}@T{ +skip the remaining CSV records +T} +T{ +\f[B]\f[CB]date-format\f[B]\f[R] +T}@T{ +describe the format of CSV dates +T} +T{ +\f[B]\f[CB]newest-first\f[B]\f[R] +T}@T{ +disambiguate record order when there\[aq]s only one date +T} +T{ +\f[B]\f[CB]include\f[B]\f[R] +T}@T{ +inline another CSV rules file +T} +.TE +.PP +There\[aq]s also a Convert CSV files tutorial on hledger.org. +.SH EXAMPLES +.PP +Here are some sample hledger CSV rules files. +See also the full collection at: +.PD 0 +.P +.PD +https://github.com/simonmichael/hledger/tree/master/examples/csv +.SS Basic +.PP +At minimum, the rules file must identify the date and amount fields, and +often it also specifies the date format and how many header lines there +are. +Here\[aq]s a simple CSV file and a rules file for it: +.IP +.nf +\f[C] +Date, Description, Id, Amount +12/11/2019, Foo, 123, 10.23 +\f[R] +.fi +.IP +.nf +\f[C] +# basic.csv.rules +skip 1 +fields date, description, _, amount +date-format %d/%m/%Y +\f[R] +.fi +.IP +.nf +\f[C] +$ hledger print -f basic.csv +2019/11/12 Foo + expenses:unknown 10.23 + income:unknown -10.23 +\f[R] +.fi +.PP +Default account names are chosen, since we didn\[aq]t set them. +.SS Bank of Ireland +.PP +Here\[aq]s a CSV with two amount fields (Debit and Credit), and a +balance field, which we can use to add balance assertions, which is not +necessary but provides extra error checking: +.IP +.nf +\f[C] +Date,Details,Debit,Credit,Balance +07/12/2012,LODGMENT 529898,,10.0,131.21 +07/12/2012,PAYMENT,5,,126 +\f[R] +.fi +.IP +.nf +\f[C] +# bankofireland-checking.csv.rules + +# skip the header line +skip + +# name the csv fields, and assign some of them as journal entry fields +fields date, description, amount-out, amount-in, balance + +# We generate balance assertions by assigning to \[dq]balance\[dq] +# above, but you may sometimes need to remove these because: +# +# - the CSV balance differs from the true balance, +# by up to 0.0000000000005 in my experience +# +# - it is sometimes calculated based on non-chronological ordering, +# eg when multiple transactions clear on the same day + +# date is in UK/Ireland format +date-format %d/%m/%Y + +# set the currency +currency EUR + +# set the base account for all txns +account1 assets:bank:boi:checking +\f[R] +.fi +.IP +.nf +\f[C] +$ hledger -f bankofireland-checking.csv print +2012/12/07 LODGMENT 529898 + assets:bank:boi:checking EUR10.0 = EUR131.2 + income:unknown EUR-10.0 + +2012/12/07 PAYMENT + assets:bank:boi:checking EUR-5.0 = EUR126.0 + expenses:unknown EUR5.0 +\f[R] +.fi +.PP +The balance assertions don\[aq]t raise an error above, because we\[aq]re +reading directly from CSV, but they will be checked if these entries are +imported into a journal file. +.SS Amazon +.PP +Here we convert amazon.com order history, and use an if block to +generate a third posting if there\[aq]s a fee. +(In practice you\[aq]d probably get this data from your bank instead, +but it\[aq]s an example.) +.IP +.nf +\f[C] +\[dq]Date\[dq],\[dq]Type\[dq],\[dq]To/From\[dq],\[dq]Name\[dq],\[dq]Status\[dq],\[dq]Amount\[dq],\[dq]Fees\[dq],\[dq]Transaction ID\[dq] +\[dq]Jul 29, 2012\[dq],\[dq]Payment\[dq],\[dq]To\[dq],\[dq]Foo.\[dq],\[dq]Completed\[dq],\[dq]$20.00\[dq],\[dq]$0.00\[dq],\[dq]16000000000000DGLNJPI1P9B8DKPVHL\[dq] +\[dq]Jul 30, 2012\[dq],\[dq]Payment\[dq],\[dq]To\[dq],\[dq]Adapteva, Inc.\[dq],\[dq]Completed\[dq],\[dq]$25.00\[dq],\[dq]$1.00\[dq],\[dq]17LA58JSKRD4HDGLNJPI1P9B8DKPVHL\[dq] +\f[R] +.fi +.IP +.nf +\f[C] +# amazon-orders.csv.rules + +# skip one header line +skip 1 + +# name the csv fields, and assign the transaction\[aq]s date, amount and code. +# Avoided the \[dq]status\[dq] and \[dq]amount\[dq] hledger field names to prevent confusion. +fields date, _, toorfrom, name, amzstatus, amzamount, fees, code + +# how to parse the date +date-format %b %-d, %Y + +# combine two fields to make the description +description %toorfrom %name + +# save the status as a tag +comment status:%amzstatus + +# set the base account for all transactions +account1 assets:amazon +# leave amount1 blank so it can balance the other(s). +# I\[aq]m assuming amzamount excludes the fees, don\[aq]t remember + +# set a generic account2 +account2 expenses:misc +amount2 %amzamount +# and maybe refine it further: +#include categorisation.rules + +# add a third posting for fees, but only if they are non-zero. +# Commas in the data makes counting fields hard, so count from the right instead. +# (Regex translation: \[dq]a field containing a non-zero dollar amount, +# immediately before the 1 right-most fields\[dq]) +if ,\[rs]$[1-9][.0-9]+(,[\[ha],]*){1}$ + account3 expenses:fees + amount3 %fees +\f[R] +.fi +.IP +.nf +\f[C] +$ hledger -f amazon-orders.csv print +2012/07/29 (16000000000000DGLNJPI1P9B8DKPVHL) To Foo. ; status:Completed + assets:amazon + expenses:misc $20.00 + +2012/07/30 (17LA58JSKRD4HDGLNJPI1P9B8DKPVHL) To Adapteva, Inc. ; status:Completed + assets:amazon + expenses:misc $25.00 + expenses:fees $1.00 +\f[R] +.fi +.SS Paypal +.PP +Here\[aq]s a real-world rules file for (customised) Paypal CSV, with +some Paypal-specific rules, and a second rules file included: +.IP +.nf +\f[C] +\[dq]Date\[dq],\[dq]Time\[dq],\[dq]TimeZone\[dq],\[dq]Name\[dq],\[dq]Type\[dq],\[dq]Status\[dq],\[dq]Currency\[dq],\[dq]Gross\[dq],\[dq]Fee\[dq],\[dq]Net\[dq],\[dq]From Email Address\[dq],\[dq]To Email Address\[dq],\[dq]Transaction ID\[dq],\[dq]Item Title\[dq],\[dq]Item ID\[dq],\[dq]Reference Txn ID\[dq],\[dq]Receipt ID\[dq],\[dq]Balance\[dq],\[dq]Note\[dq] +\[dq]10/01/2019\[dq],\[dq]03:46:20\[dq],\[dq]PDT\[dq],\[dq]Calm Radio\[dq],\[dq]Subscription Payment\[dq],\[dq]Completed\[dq],\[dq]USD\[dq],\[dq]-6.99\[dq],\[dq]0.00\[dq],\[dq]-6.99\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]memberships\[at]calmradio.com\[dq],\[dq]60P57143A8206782E\[dq],\[dq]MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month\[dq],\[dq]\[dq],\[dq]I-R8YLY094FJYR\[dq],\[dq]\[dq],\[dq]-6.99\[dq],\[dq]\[dq] +\[dq]10/01/2019\[dq],\[dq]03:46:20\[dq],\[dq]PDT\[dq],\[dq]\[dq],\[dq]Bank Deposit to PP Account \[dq],\[dq]Pending\[dq],\[dq]USD\[dq],\[dq]6.99\[dq],\[dq]0.00\[dq],\[dq]6.99\[dq],\[dq]\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]0TU1544T080463733\[dq],\[dq]\[dq],\[dq]\[dq],\[dq]60P57143A8206782E\[dq],\[dq]\[dq],\[dq]0.00\[dq],\[dq]\[dq] +\[dq]10/01/2019\[dq],\[dq]08:57:01\[dq],\[dq]PDT\[dq],\[dq]Patreon\[dq],\[dq]PreApproved Payment Bill User Payment\[dq],\[dq]Completed\[dq],\[dq]USD\[dq],\[dq]-7.00\[dq],\[dq]0.00\[dq],\[dq]-7.00\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]support\[at]patreon.com\[dq],\[dq]2722394R5F586712G\[dq],\[dq]Patreon* Membership\[dq],\[dq]\[dq],\[dq]B-0PG93074E7M86381M\[dq],\[dq]\[dq],\[dq]-7.00\[dq],\[dq]\[dq] +\[dq]10/01/2019\[dq],\[dq]08:57:01\[dq],\[dq]PDT\[dq],\[dq]\[dq],\[dq]Bank Deposit to PP Account \[dq],\[dq]Pending\[dq],\[dq]USD\[dq],\[dq]7.00\[dq],\[dq]0.00\[dq],\[dq]7.00\[dq],\[dq]\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]71854087RG994194F\[dq],\[dq]Patreon* Membership\[dq],\[dq]\[dq],\[dq]2722394R5F586712G\[dq],\[dq]\[dq],\[dq]0.00\[dq],\[dq]\[dq] +\[dq]10/19/2019\[dq],\[dq]03:02:12\[dq],\[dq]PDT\[dq],\[dq]Wikimedia Foundation, Inc.\[dq],\[dq]Subscription Payment\[dq],\[dq]Completed\[dq],\[dq]USD\[dq],\[dq]-2.00\[dq],\[dq]0.00\[dq],\[dq]-2.00\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]tle\[at]wikimedia.org\[dq],\[dq]K9U43044RY432050M\[dq],\[dq]Monthly donation to the Wikimedia Foundation\[dq],\[dq]\[dq],\[dq]I-R5C3YUS3285L\[dq],\[dq]\[dq],\[dq]-2.00\[dq],\[dq]\[dq] +\[dq]10/19/2019\[dq],\[dq]03:02:12\[dq],\[dq]PDT\[dq],\[dq]\[dq],\[dq]Bank Deposit to PP Account \[dq],\[dq]Pending\[dq],\[dq]USD\[dq],\[dq]2.00\[dq],\[dq]0.00\[dq],\[dq]2.00\[dq],\[dq]\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]3XJ107139A851061F\[dq],\[dq]\[dq],\[dq]\[dq],\[dq]K9U43044RY432050M\[dq],\[dq]\[dq],\[dq]0.00\[dq],\[dq]\[dq] +\[dq]10/22/2019\[dq],\[dq]05:07:06\[dq],\[dq]PDT\[dq],\[dq]Noble Benefactor\[dq],\[dq]Subscription Payment\[dq],\[dq]Completed\[dq],\[dq]USD\[dq],\[dq]10.00\[dq],\[dq]-0.59\[dq],\[dq]9.41\[dq],\[dq]noble\[at]bene.fac.tor\[dq],\[dq]simon\[at]joyful.com\[dq],\[dq]6L8L1662YP1334033\[dq],\[dq]Joyful Systems\[dq],\[dq]\[dq],\[dq]I-KC9VBGY2GWDB\[dq],\[dq]\[dq],\[dq]9.41\[dq],\[dq]\[dq] +\f[R] +.fi +.IP +.nf +\f[C] +# paypal-custom.csv.rules + +# Tips: +# Export from Activity -> Statements -> Custom -> Activity download +# Suggested transaction type: \[dq]Balance affecting\[dq] +# Paypal\[aq]s default fields in 2018 were: +# \[dq]Date\[dq],\[dq]Time\[dq],\[dq]TimeZone\[dq],\[dq]Name\[dq],\[dq]Type\[dq],\[dq]Status\[dq],\[dq]Currency\[dq],\[dq]Gross\[dq],\[dq]Fee\[dq],\[dq]Net\[dq],\[dq]From Email Address\[dq],\[dq]To Email Address\[dq],\[dq]Transaction ID\[dq],\[dq]Shipping Address\[dq],\[dq]Address Status\[dq],\[dq]Item Title\[dq],\[dq]Item ID\[dq],\[dq]Shipping and Handling Amount\[dq],\[dq]Insurance Amount\[dq],\[dq]Sales Tax\[dq],\[dq]Option 1 Name\[dq],\[dq]Option 1 Value\[dq],\[dq]Option 2 Name\[dq],\[dq]Option 2 Value\[dq],\[dq]Reference Txn ID\[dq],\[dq]Invoice Number\[dq],\[dq]Custom Number\[dq],\[dq]Quantity\[dq],\[dq]Receipt ID\[dq],\[dq]Balance\[dq],\[dq]Address Line 1\[dq],\[dq]Address Line 2/District/Neighborhood\[dq],\[dq]Town/City\[dq],\[dq]State/Province/Region/County/Territory/Prefecture/Republic\[dq],\[dq]Zip/Postal Code\[dq],\[dq]Country\[dq],\[dq]Contact Phone Number\[dq],\[dq]Subject\[dq],\[dq]Note\[dq],\[dq]Country Code\[dq],\[dq]Balance Impact\[dq] +# This rules file assumes the following more detailed fields, configured in \[dq]Customize report fields\[dq]: +# \[dq]Date\[dq],\[dq]Time\[dq],\[dq]TimeZone\[dq],\[dq]Name\[dq],\[dq]Type\[dq],\[dq]Status\[dq],\[dq]Currency\[dq],\[dq]Gross\[dq],\[dq]Fee\[dq],\[dq]Net\[dq],\[dq]From Email Address\[dq],\[dq]To Email Address\[dq],\[dq]Transaction ID\[dq],\[dq]Item Title\[dq],\[dq]Item ID\[dq],\[dq]Reference Txn ID\[dq],\[dq]Receipt ID\[dq],\[dq]Balance\[dq],\[dq]Note\[dq] + +fields date, time, timezone, description_, type, status_, currency, grossamount, feeamount, netamount, fromemail, toemail, code, itemtitle, itemid, referencetxnid, receiptid, balance, note + +skip 1 + +date-format %-m/%-d/%Y + +# ignore some paypal events +if +In Progress +Temporary Hold +Update to + skip + +# add more fields to the description +description %description_ %itemtitle + +# save some other fields as tags +comment itemid:%itemid, fromemail:%fromemail, toemail:%toemail, time:%time, type:%type, status:%status_ + +# convert to short currency symbols +# Note: in conditional block regexps, the line of csv being matched is +# a synthetic one: the unquoted field values, with commas between them. +if ,USD, + currency $ +if ,EUR, + currency E +if ,GBP, + currency P + +# generate postings + +# the first posting will be the money leaving/entering my paypal account +# (negative means leaving my account, in all amount fields) +account1 assets:online:paypal +amount1 %netamount + +# the second posting will be money sent to/received from other party +# (account2 is set below) +amount2 -%grossamount + +# if there\[aq]s a fee (9th field), add a third posting for the money taken by paypal. +# TODO: This regexp fails when fields contain a comma (generates a third posting with zero amount) +if \[ha]([\[ha],]+,){8}[\[ha]0] + account3 expenses:banking:paypal + amount3 -%feeamount + comment3 business: + +# choose an account for the second posting + +# override the default account names: +# if amount (8th field) is positive, it\[aq]s income (a debit) +if \[ha]([\[ha],]+,){7}[0-9] + account2 income:unknown +# if negative, it\[aq]s an expense (a credit) +if \[ha]([\[ha],]+,){7}- + account2 expenses:unknown + +# apply common rules for setting account2 & other tweaks +include common.rules + +# apply some overrides specific to this csv + +# Transfers from/to bank. These are usually marked Pending, +# which can be disregarded in this case. +if +Bank Account +Bank Deposit to PP Account + description %type for %referencetxnid %itemtitle + account2 assets:bank:wf:pchecking + account1 assets:online:paypal + +# Currency conversions +if Currency Conversion + account2 equity:currency conversion +\f[R] +.fi +.IP +.nf +\f[C] +# common.rules + +if +darcs +noble benefactor + account2 revenues:foss donations:darcshub + comment2 business: + +if +Calm Radio + account2 expenses:online:apps + +if +electronic frontier foundation +Patreon +wikimedia +Advent of Code + account2 expenses:dues + +if Google + account2 expenses:online:apps + description google | music +\f[R] +.fi +.IP +.nf +\f[C] +$ hledger -f paypal-custom.csv print +2019/10/01 (60P57143A8206782E) Calm Radio MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month ; itemid:, fromemail:simon\[at]joyful.com, toemail:memberships\[at]calmradio.com, time:03:46:20, type:Subscription Payment, status:Completed + assets:online:paypal $-6.99 = $-6.99 + expenses:online:apps $6.99 + +2019/10/01 (0TU1544T080463733) Bank Deposit to PP Account for 60P57143A8206782E ; itemid:, fromemail:, toemail:simon\[at]joyful.com, time:03:46:20, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $6.99 = $0.00 + assets:bank:wf:pchecking $-6.99 + +2019/10/01 (2722394R5F586712G) Patreon Patreon* Membership ; itemid:, fromemail:simon\[at]joyful.com, toemail:support\[at]patreon.com, time:08:57:01, type:PreApproved Payment Bill User Payment, status:Completed + assets:online:paypal $-7.00 = $-7.00 + expenses:dues $7.00 + +2019/10/01 (71854087RG994194F) Bank Deposit to PP Account for 2722394R5F586712G Patreon* Membership ; itemid:, fromemail:, toemail:simon\[at]joyful.com, time:08:57:01, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $7.00 = $0.00 + assets:bank:wf:pchecking $-7.00 + +2019/10/19 (K9U43044RY432050M) Wikimedia Foundation, Inc. Monthly donation to the Wikimedia Foundation ; itemid:, fromemail:simon\[at]joyful.com, toemail:tle\[at]wikimedia.org, time:03:02:12, type:Subscription Payment, status:Completed + assets:online:paypal $-2.00 = $-2.00 + expenses:dues $2.00 + expenses:banking:paypal ; business: + +2019/10/19 (3XJ107139A851061F) Bank Deposit to PP Account for K9U43044RY432050M ; itemid:, fromemail:, toemail:simon\[at]joyful.com, time:03:02:12, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $2.00 = $0.00 + assets:bank:wf:pchecking $-2.00 + +2019/10/22 (6L8L1662YP1334033) Noble Benefactor Joyful Systems ; itemid:, fromemail:noble\[at]bene.fac.tor, toemail:simon\[at]joyful.com, time:05:07:06, type:Subscription Payment, status:Completed + assets:online:paypal $9.41 = $9.41 + revenues:foss donations:darcshub $-10.00 ; business: + expenses:banking:paypal $0.59 ; business: +\f[R] +.fi +.SH CSV RULES +.PP +The following kinds of rule can appear in the rules file, in any order. +Blank lines and lines beginning with \f[C]#\f[R] or \f[C];\f[R] are +ignored. +.SS \f[C]skip\f[R] +.IP +.nf +\f[C] +skip N +\f[R] +.fi +.PP +The word \[dq]skip\[dq] followed by a number (or no number, meaning 1) +tells hledger to ignore this many non-empty lines preceding the CSV +data. +(Empty/blank lines are skipped automatically.) You\[aq]ll need this +whenever your CSV data contains header lines. +.PP +It also has a second purpose: it can be used inside if blocks to ignore +certain CSV records (described below). +.SS \f[C]fields\f[R] +.IP +.nf +\f[C] +fields FIELDNAME1, FIELDNAME2, ... +\f[R] +.fi +.PP +A fields list (the word \[dq]fields\[dq] followed by comma-separated +field names) is the quick way to assign CSV field values to hledger +fields. +It does two things: +.IP "1." 3 +it names the CSV fields. +This is optional, but can be convenient later for interpolating them. +.IP "2." 3 +when you use a standard hledger field name, it assigns the CSV value to +that part of the hledger transaction. +.PP +Here\[aq]s an example that says \[dq]use the 1st, 2nd and 4th fields as +the transaction\[aq]s date, description and amount; name the last two +fields for later reference; and ignore the others\[dq]: +.IP +.nf +\f[C] +fields date, description, , amount, , , somefield, anotherfield +\f[R] +.fi +.PP +Field names may not contain whitespace. +Fields you don\[aq]t care about can be left unnamed. +Currently there must be least two items (there must be at least one +comma). +.PP +Here are the standard hledger field/pseudo-field names. +For more about the transaction parts they refer to, see the manual for +hledger\[aq]s journal format. +.SS Transaction field names +.PP +\f[C]date\f[R], \f[C]date2\f[R], \f[C]status\f[R], \f[C]code\f[R], +\f[C]description\f[R], \f[C]comment\f[R] can be used to form the +transaction\[aq]s first line. +.SS Posting field names +.PP +\f[C]accountN\f[R], where N is 1 to 9, generates a posting, with that +account name. +Most often there are two postings, so you\[aq]ll want to set +\f[C]account1\f[R] and \f[C]account2\f[R]. +If a posting\[aq]s account name is left unset but its amount is set, a +default account name will be chosen (like expenses:unknown or +income:unknown). +.PP +\f[C]amountN\f[R] sets posting N\[aq]s amount. +Or, \f[C]amount\f[R] with no N sets posting 1\[aq]s. +If the CSV has debits and credits in separate fields, use +\f[C]amountN-in\f[R] and \f[C]amountN-out\f[R] instead. +Or \f[C]amount-in\f[R] and \f[C]amount-out\f[R] with no N for posting 1. +.PP +For convenience and backwards compatibility, if you set the amount of +posting 1 only, a second posting with the negative amount will be +generated automatically. +(Unless the account name is parenthesised indicating an unbalanced +posting.) +.PP +If the CSV has the currency symbol in a separate field, you can use +\f[C]currencyN\f[R] to prepend it to posting N\[aq]s amount. +\f[C]currency\f[R] with no N affects ALL postings. +.PP +\f[C]balanceN\f[R] sets a balance assertion amount (or if the posting +amount is left empty, a balance assignment). +.PP +Finally, \f[C]commentN\f[R] sets a comment on the Nth posting. +Comments can also contain tags, as usual. +.PP +See TIPS below for more about setting amounts and currency. +.SS field assignment +.IP +.nf +\f[C] +HLEDGERFIELDNAME FIELDVALUE +\f[R] +.fi +.PP +Instead of or in addition to a fields list, you can use a \[dq]field +assignment\[dq] rule to set the value of a single hledger field, by +writing its name (any of the standard hledger field names above) +followed by a text value. +The value may contain interpolated CSV fields, referenced by their +1-based position in the CSV record (\f[C]%N\f[R]), or by the name they +were given in the fields list (\f[C]%CSVFIELDNAME\f[R]). +Some examples: +.IP +.nf +\f[C] +# set the amount to the 4th CSV field, with \[dq] USD\[dq] appended +amount %4 USD + +# combine three fields to make a comment, containing note: and date: tags +comment note: %somefield - %anotherfield, date: %1 +\f[R] +.fi +.PP +Interpolation strips outer whitespace (so a CSV value like +\f[C]\[dq] 1 \[dq]\f[R] becomes \f[C]1\f[R] when interpolated) (#1051). +See TIPS below for more about referencing other fields. +.SS \f[C]if\f[R] +.IP +.nf +\f[C] +if PATTERN + RULE + +if +PATTERN +PATTERN +PATTERN + RULE + RULE +\f[R] +.fi +.PP +Conditional blocks (\[dq]if blocks\[dq]) are a block of rules that are +applied only to CSV records which match certain patterns. +They are often used for customising account names based on transaction +descriptions. +.PP +A single pattern can be written on the same line as the \[dq]if\[dq]; or +multiple patterns can be written on the following lines, non-indented. +Multiple patterns are OR\[aq]d (any one of them can match). +Patterns are case-insensitive regular expressions which try to match +anywhere within the whole CSV record (POSIX extended regular expressions +with some additions, see +https://hledger.org/hledger.html#regular-expressions). +Note the CSV record they see is close to, but not identical to, the one +in the CSV file; enclosing double quotes will be removed, and the +separator character is always comma. +.PP +It\[aq]s not yet easy to match within a specific field. +If the data does not contain commas, you can hack it with a regular +expression like: +.IP +.nf +\f[C] +# match \[dq]foo\[dq] in the fourth field +if \[ha]([\[ha],]*,){3}foo +\f[R] +.fi +.PP +After the patterns there should be one or more rules to apply, all +indented by at least one space. +Three kinds of rule are allowed in conditional blocks: +.IP \[bu] 2 +field assignments (to set a hledger field) +.IP \[bu] 2 +skip (to skip the matched CSV record) +.IP \[bu] 2 +end (to skip all remaining CSV records). +.PP +Examples: +.IP +.nf +\f[C] +# if the CSV record contains \[dq]groceries\[dq], set account2 to \[dq]expenses:groceries\[dq] +if groceries + account2 expenses:groceries +\f[R] +.fi +.IP +.nf +\f[C] +# if the CSV record contains any of these patterns, set account2 and comment as shown +if +monthly service fee +atm transaction fee +banking thru software + account2 expenses:business:banking + comment XXX deductible ? check it +\f[R] +.fi +.SS \f[C]end\f[R] +.PP +This rule can be used inside if blocks (only), to make hledger stop +reading this CSV file and move on to the next input file, or to command +execution. +Eg: +.IP +.nf +\f[C] +# ignore everything following the first empty record +if ,,,, + end +\f[R] +.fi +.SS \f[C]date-format\f[R] +.IP +.nf +\f[C] +date-format DATEFMT +\f[R] +.fi +.PP +This is a helper for the \f[C]date\f[R] (and \f[C]date2\f[R]) fields. +If your CSV dates are not formatted like \f[C]YYYY-MM-DD\f[R], +\f[C]YYYY/MM/DD\f[R] or \f[C]YYYY.MM.DD\f[R], you\[aq]ll need to add a +date-format rule describing them with a strptime date parsing pattern, +which must parse the CSV date value completely. +Some examples: +.IP +.nf +\f[C] +# MM/DD/YY +date-format %m/%d/%y +\f[R] +.fi +.IP +.nf +\f[C] +# D/M/YYYY +# The - makes leading zeros optional. +date-format %-d/%-m/%Y +\f[R] +.fi +.IP +.nf +\f[C] +# YYYY-Mmm-DD +date-format %Y-%h-%d +\f[R] +.fi +.IP +.nf +\f[C] +# M/D/YYYY HH:MM AM some other junk +# Note the time and junk must be fully parsed, though only the date is used. +date-format %-m/%-d/%Y %l:%M %p some other junk +\f[R] +.fi +.PP +For the supported strptime syntax, see: +.PD 0 +.P +.PD +https://hackage.haskell.org/package/time/docs/Data-Time-Format.html#v:formatTime +.SS \f[C]newest-first\f[R] +.PP +hledger always sorts the generated transactions by date. +Transactions on the same date should appear in the same order as their +CSV records, as hledger can usually auto-detect whether the CSV\[aq]s +normal order is oldest first or newest first. +But if all of the following are true: +.IP \[bu] 2 +the CSV might sometimes contain just one day of data (all records having +the same date) +.IP \[bu] 2 +the CSV records are normally in reverse chronological order (newest at +the top) +.IP \[bu] 2 +and you care about preserving the order of same-day transactions +.PP +then, you should add the \f[C]newest-first\f[R] rule as a hint. +Eg: +.IP +.nf +\f[C] +# tell hledger explicitly that the CSV is normally newest first +newest-first +\f[R] +.fi +.SS \f[C]include\f[R] +.IP +.nf +\f[C] +include RULESFILE +\f[R] +.fi +.PP +This includes the contents of another CSV rules file at this point. +\f[C]RULESFILE\f[R] is an absolute file path or a path relative to the +current file\[aq]s directory. +This can be useful for sharing common rules between several rules files, +eg: +.IP +.nf +\f[C] +# someaccount.csv.rules + +## someaccount-specific rules +fields date,description,amount +account1 assets:someaccount +account2 expenses:misc + +## common rules +include categorisation.rules +\f[R] +.fi +.SH TIPS +.SS Valid CSV +.PP +hledger accepts CSV conforming to RFC 4180. +When CSV values are enclosed in quotes, note: +.IP \[bu] 2 +they must be double quotes (not single quotes) +.IP \[bu] 2 +spaces outside the quotes are not allowed +.SS Other separator characters +.PP +With the \f[C]--separator \[aq]CHAR\[aq]\f[R] option (experimental), +hledger will expect the separator to be CHAR instead of a comma. +Ie it will read other \[dq]Character Separated Values\[dq] formats, such +as TSV (Tab Separated Values). +Note: on the command line, use a real tab character in quotes, not Eg: +.IP +.nf +\f[C] +$ hledger -f foo.tsv --separator \[aq] \[aq] print +\f[R] +.fi +.SS Reading multiple CSV files +.PP +If you use multiple \f[C]-f\f[R] options to read multiple CSV files at +once, hledger will look for a correspondingly-named rules file for each +CSV file. +But if you use the \f[C]--rules-file\f[R] option, that rules file will +be used for all the CSV files. +.SS Valid transactions +.PP +After reading a CSV file, hledger post-processes and validates the +generated journal entries as it would for a journal file - balancing +them, applying balance assignments, and canonicalising amount styles. +Any errors at this stage will be reported in the usual way, displaying +the problem entry. +.PP +There is one exception: balance assertions, if you have generated them, +will not be checked, since normally these will work only when the CSV +data is part of the main journal. +If you do need to check balance assertions generated from CSV right +away, pipe into another hledger: +.IP +.nf +\f[C] +$ hledger -f file.csv print | hledger -f- print +\f[R] +.fi +.SS Deduplicating, importing +.PP +When you download a CSV file periodically, eg to get your latest bank +transactions, the new file may overlap with the old one, containing some +of the same records. +.PP +The import command will (a) detect the new transactions, and (b) append +just those transactions to your main journal. +It is idempotent, so you don\[aq]t have to remember how many times you +ran it or with which version of the CSV. +(It keeps state in a hidden \f[C].latest.FILE.csv\f[R] file.) This is +the easiest way to import CSV data. +Eg: +.IP +.nf +\f[C] +# download the latest CSV files, then run this command. +# Note, no -f flags needed here. +$ hledger import *.csv [--dry] +\f[R] +.fi +.PP +This method works for most CSV files. +(Where records have a stable chronological order, and new records appear +only at the new end.) +.PP +A number of other tools and workflows, hledger-specific and otherwise, +exist for converting, deduplicating, classifying and managing CSV data. +See: +.IP \[bu] 2 +https://hledger.org -> sidebar -> real world setups +.IP \[bu] 2 +https://plaintextaccounting.org -> data import/conversion +.SS Setting amounts +.PP +A posting amount can be set in one of these ways: +.IP \[bu] 2 +by assigning (with a fields list or field assigment) to +\f[C]amountN\f[R] (posting N\[aq]s amount) or \f[C]amount\f[R] (posting +1\[aq]s amount) +.IP \[bu] 2 +by assigning to \f[C]amountN-in\f[R] and \f[C]amountN-out\f[R] (or +\f[C]amount-in\f[R] and \f[C]amount-out\f[R]). +For each CSV record, whichever of these has a non-zero value will be +used, with appropriate sign. +If both contain a non-zero value, this may not work. +.IP \[bu] 2 +by assigning to \f[C]balanceN\f[R] (or \f[C]balance\f[R]) instead of the +above, setting the amount indirectly via a balance assignment. +If you do this the default account name may be wrong, so you should set +that explicitly. +.PP +There is some special handling for an amount\[aq]s sign: +.IP \[bu] 2 +If an amount value is parenthesised, it will be de-parenthesised and +sign-flipped. +.IP \[bu] 2 +If an amount value begins with a double minus sign, those cancel out and +are removed. +.IP \[bu] 2 +If an amount value begins with a plus sign, that will be removed +.SS Setting currency/commodity +.PP +If the currency/commodity symbol is included in the CSV\[aq]s amount +field(s), you don\[aq]t have to do anything special. +.PP +If the currency is provided as a separate CSV field, you can either: +.IP \[bu] 2 +assign that to \f[C]currency\f[R], which adds it to all posting amounts. +The symbol will prepended to the amount quantity (on the left side). +If you write a trailing space after the symbol, there will be a space +between symbol and amount (an exception to the usual whitespace +stripping). +.IP \[bu] 2 +or assign it to \f[C]currencyN\f[R] which adds it to posting N\[aq]s +amount only. +.IP \[bu] 2 +or for more control, construct the amount from symbol and quantity using +field assignment, eg: +.RS 2 +.IP +.nf +\f[C] +fields date,description,currency,quantity +# add currency symbol on the right: +amount %quantity %currency +\f[R] +.fi +.RE +.SS Referencing other fields +.PP +In field assignments, you can interpolate only CSV fields, not hledger +fields. +In the example below, there\[aq]s both a CSV field and a hledger field +named amount1, but %amount1 always means the CSV field, not the hledger +field: +.IP +.nf +\f[C] +# Name the third CSV field \[dq]amount1\[dq] +fields date,description,amount1 + +# Set hledger\[aq]s amount1 to the CSV amount1 field followed by USD +amount1 %amount1 USD + +# Set comment to the CSV amount1 (not the amount1 assigned above) +comment %amount1 +\f[R] +.fi +.PP +Here, since there\[aq]s no CSV amount1 field, %amount1 will produce a +literal \[dq]amount1\[dq]: +.IP +.nf +\f[C] +fields date,description,csvamount +amount1 %csvamount USD +# Can\[aq]t interpolate amount1 here +comment %amount1 +\f[R] +.fi +.PP +When there are multiple field assignments to the same hledger field, +only the last one takes effect. +Here, comment\[aq]s value will be be B, or C if \[dq]something\[dq] is +matched, but never A: +.IP +.nf +\f[C] +comment A +comment B +if something + comment C +\f[R] +.fi +.SS How CSV rules are evaluated +.PP +Here\[aq]s how to think of CSV rules being evaluated (if you really need +to). +First, +.IP \[bu] 2 +\f[C]include\f[R] - all includes are inlined, from top to bottom, depth +first. +(At each include point the file is inlined and scanned for further +includes, recursively, before proceeding.) +.PP +Then \[dq]global\[dq] rules are evaluated, top to bottom. +If a rule is repeated, the last one wins: +.IP \[bu] 2 +\f[C]skip\f[R] (at top level) +.IP \[bu] 2 +\f[C]date-format\f[R] +.IP \[bu] 2 +\f[C]newest-first\f[R] +.IP \[bu] 2 +\f[C]fields\f[R] - names the CSV fields, optionally sets up initial +assignments to hledger fields +.PP +Then for each CSV record in turn: +.IP \[bu] 2 +test all \f[C]if\f[R] blocks. +If any of them contain a \f[C]end\f[R] rule, skip all remaining CSV +records. +Otherwise if any of them contain a \f[C]skip\f[R] rule, skip that many +CSV records. +If there are multiple matched \f[C]skip\f[R] rules, the first one wins. +.IP \[bu] 2 +collect all field assignments at top level and in matched \f[C]if\f[R] +blocks. +When there are multiple assignments for a field, keep only the last one. +.IP \[bu] 2 +compute a value for each hledger field - either the one that was +assigned to it (and interpolate the %CSVFIELDNAME references), or a +default +.IP \[bu] 2 +generate a synthetic hledger transaction from these values. +.PP +This is all part of the CSV reader, one of several readers hledger can +use to parse input files. +When all files have been read successfully, the transactions are passed +as input to whichever hledger command the user specified. + + +.SH "REPORTING BUGS" +Report bugs at http://bugs.hledger.org +(or on the #hledger IRC channel or hledger mail list) + +.SH AUTHORS +Simon Michael and contributors + +.SH COPYRIGHT + +Copyright (C) 2007-2019 Simon Michael. .br Released under GNU GPL v3 or later. diff --git a/hledger_csv.info b/hledger_csv.info --- a/hledger_csv.info +++ b/hledger_csv.info @@ -1,384 +1,944 @@ This is hledger_csv.info, produced by makeinfo version 6.5 from stdin.  -File: hledger_csv.info, Node: Top, Next: CSV RULES, Up: (dir) - -hledger_csv(5) hledger 1.15.2 -***************************** - -hledger can read CSV (comma-separated value) files as if they were -journal files, automatically converting each CSV record into a -transaction. (To learn about _writing_ CSV, see CSV output.) - - Converting CSV to transactions requires some special conversion -rules. These do several things: - - * they describe the layout and format of the CSV data - * they can customize the generated journal entries using a simple - templating language - * they can add refinements based on patterns in the CSV data, eg - categorizing transactions with more detailed account names. - - When reading a CSV file named 'FILE.csv', hledger looks for a -conversion rules file named 'FILE.csv.rules' in the same directory. You -can override this with the '--rules-file' option. If the rules file -does not exist, hledger will auto-create one with some example rules, -which you'll need to adjust. - - At minimum, the rules file must identify the date and amount fields. -It's often necessary to specify the date format, and the number of -header lines to skip, also. Eg: - -fields date, _, _, amount -date-format %d/%m/%Y -skip 1 - - A more complete example: - -# hledger CSV rules for amazon.com order history - -# sample: -# "Date","Type","To/From","Name","Status","Amount","Fees","Transaction ID" -# "Jul 29, 2012","Payment","To","Adapteva, Inc.","Completed","$25.00","$0.00","17LA58JSK6PRD4HDGLNJQPI1PB9N8DKPVHL" - -# skip one header line -skip 1 - -# name the csv fields (and assign the transaction's date, amount and code) -fields date, _, toorfrom, name, amzstatus, amount, fees, code - -# how to parse the date -date-format %b %-d, %Y - -# combine two fields to make the description -description %toorfrom %name - -# save these fields as tags -comment status:%amzstatus, fees:%fees - -# set the base account for all transactions -account1 assets:amazon - -# flip the sign on the amount -amount -%amount - - For more examples, see Convert CSV files. - -* Menu: - -* CSV RULES:: -* CSV TIPS:: - - -File: hledger_csv.info, Node: CSV RULES, Next: CSV TIPS, Prev: Top, Up: Top - -1 CSV RULES -*********** - -The following seven kinds of rule can appear in the rules file, in any -order. Blank lines and lines beginning with '#' or ';' are ignored. - -* Menu: - -* skip:: -* date-format:: -* field list:: -* field assignment:: -* conditional block:: -* include:: -* newest-first:: - - -File: hledger_csv.info, Node: skip, Next: date-format, Up: CSV RULES - -1.1 skip -======== - -'skip'_'N'_ - - Skip this number of CSV records at the beginning. You'll need this -whenever your CSV data contains header lines. Eg: - -# ignore the first CSV line -skip 1 - - -File: hledger_csv.info, Node: date-format, Next: field list, Prev: skip, Up: CSV RULES - -1.2 date-format -=============== - -'date-format'_'DATEFMT'_ - - When your CSV date fields are not formatted like 'YYYY/MM/DD' (or -'YYYY-MM-DD' or 'YYYY.MM.DD'), you'll need to specify the format. -DATEFMT is a strptime-like date parsing pattern, which must parse the -date field values completely. Examples: - -# for dates like "11/06/2013": -date-format %m/%d/%Y - -# for dates like "6/11/2013" (note the - to make leading zeros optional): -date-format %-d/%-m/%Y - -# for dates like "2013-Nov-06": -date-format %Y-%h-%d - -# for dates like "11/6/2013 11:32 PM": -date-format %-m/%-d/%Y %l:%M %p - - -File: hledger_csv.info, Node: field list, Next: field assignment, Prev: date-format, Up: CSV RULES - -1.3 field list -============== - -'fields'_'FIELDNAME1'_, _'FIELDNAME2'_... - - This (a) names the CSV fields, in order (names may not contain -whitespace; uninteresting names may be left blank), and (b) assigns them -to journal entry fields if you use any of these standard field names: -'date', 'date2', 'status', 'code', 'description', 'comment', 'account1', -'account2', 'amount', 'amount-in', 'amount-out', 'currency', 'balance', -'balance1', 'balance2'. Eg: - -# use the 1st, 2nd and 4th CSV fields as the entry's date, description and amount, -# and give the 7th and 8th fields meaningful names for later reference: -# -# CSV field: -# 1 2 3 4 5 6 7 8 -# entry field: -fields date, description, , amount, , , somefield, anotherfield - - -File: hledger_csv.info, Node: field assignment, Next: conditional block, Prev: field list, Up: CSV RULES - -1.4 field assignment -==================== - -_'ENTRYFIELDNAME'_ _'FIELDVALUE'_ - - This sets a journal entry field (one of the standard names above) to -the given text value, which can include CSV field values interpolated by -name ('%CSVFIELDNAME') or 1-based position ('%N'). Eg: - -# set the amount to the 4th CSV field with "USD " prepended -amount USD %4 - -# combine three fields to make a comment (containing two tags) -comment note: %somefield - %anotherfield, date: %1 - - Field assignments can be used instead of or in addition to a field -list. - - Note, interpolation strips any outer whitespace, so a CSV value like -'" 1 "' becomes '1' when interpolated (#1051). - - -File: hledger_csv.info, Node: conditional block, Next: include, Prev: field assignment, Up: CSV RULES - -1.5 conditional block -===================== - -'if' _'PATTERN'_ - _'FIELDASSIGNMENTS'_... - - 'if' -_'PATTERN'_ -_'PATTERN'_... - _'FIELDASSIGNMENTS'_... - - This applies one or more field assignments, only to those CSV records -matched by one of the PATTERNs. The patterns are case-insensitive -regular expressions which match anywhere within the whole CSV record -(it's not yet possible to match within a specific field). When there -are multiple patterns they can be written on separate lines, unindented. -The field assignments are on separate lines indented by at least one -space. Examples: - -# if the CSV record contains "groceries", set account2 to "expenses:groceries" -if groceries - account2 expenses:groceries - -# if the CSV record contains any of these patterns, set account2 and comment as shown -if -monthly service fee -atm transaction fee -banking thru software - account2 expenses:business:banking - comment XXX deductible ? check it - - -File: hledger_csv.info, Node: include, Next: newest-first, Prev: conditional block, Up: CSV RULES - -1.6 include -=========== - -'include'_'RULESFILE'_ - - Include another rules file at this point. 'RULESFILE' is either an -absolute file path or a path relative to the current file's directory. -Eg: - -# rules reused with several CSV files -include common.rules - - -File: hledger_csv.info, Node: newest-first, Prev: include, Up: CSV RULES - -1.7 newest-first -================ - -'newest-first' - - Consider adding this rule if all of the following are true: you might -be processing just one day of data, your CSV records are in reverse -chronological order (newest first), and you care about preserving the -order of same-day transactions. It usually isn't needed, because -hledger autodetects the CSV order, but when all CSV records have the -same date it will assume they are oldest first. - - -File: hledger_csv.info, Node: CSV TIPS, Prev: CSV RULES, Up: Top - -2 CSV TIPS -********** - -* Menu: - -* CSV ordering:: -* CSV accounts:: -* CSV amounts:: -* CSV balance assertions/assignments:: -* Reading multiple CSV files:: -* Valid CSV:: - - -File: hledger_csv.info, Node: CSV ordering, Next: CSV accounts, Up: CSV TIPS - -2.1 CSV ordering -================ - -The generated journal entries will be sorted by date. The order of -same-day entries will be preserved (except in the special case where you -might need 'newest-first', see above). - - -File: hledger_csv.info, Node: CSV accounts, Next: CSV amounts, Prev: CSV ordering, Up: CSV TIPS - -2.2 CSV accounts -================ - -Each journal entry will have two postings, to 'account1' and 'account2' -respectively. It's not yet possible to generate entries with more than -two postings. It's conventional and recommended to use 'account1' for -the account whose CSV we are reading. - - -File: hledger_csv.info, Node: CSV amounts, Next: CSV balance assertions/assignments, Prev: CSV accounts, Up: CSV TIPS - -2.3 CSV amounts -=============== - -A transaction amount must be set, in one of these ways: - - * with an 'amount' field assignment, which sets the first posting's - amount - - * (When the CSV has debit and credit amounts in separate fields:) - with field assignments for the 'amount-in' and 'amount-out' pseudo - fields (both of them). Whichever one has a value will be used, - with appropriate sign. If both contain a value, it might not work - so well. - - * or implicitly by means of a balance assignment (see below). - - There is some special handling for sign in amounts: - - * If an amount value is parenthesised, it will be de-parenthesised - and sign-flipped. - * If an amount value begins with a double minus sign, those will - cancel out and be removed. - - If the currency/commodity symbol is provided as a separate CSV field, -assign it to the 'currency' pseudo field; the symbol will be prepended -to the amount (TODO: when there is an amount). Or, you can use an -'amount' field assignment for more control, eg: - -fields date,description,currency,amount -amount %amount %currency - - -File: hledger_csv.info, Node: CSV balance assertions/assignments, Next: Reading multiple CSV files, Prev: CSV amounts, Up: CSV TIPS - -2.4 CSV balance assertions/assignments -====================================== - -If the CSV includes a running balance, you can assign that to one of the -pseudo fields 'balance' (or 'balance1') or 'balance2'. This will -generate a balance assertion (or if the amount is left empty, a balance -assignment), on the first or second posting, whenever the running -balance field is non-empty. (TODO: #1000) - - -File: hledger_csv.info, Node: Reading multiple CSV files, Next: Valid CSV, Prev: CSV balance assertions/assignments, Up: CSV TIPS - -2.5 Reading multiple CSV files -============================== - -You can read multiple CSV files at once using multiple '-f' arguments on -the command line, and hledger will look for a correspondingly-named -rules file for each. Note if you use the '--rules-file' option, this -one rules file will be used for all the CSV files being read. - - -File: hledger_csv.info, Node: Valid CSV, Prev: Reading multiple CSV files, Up: CSV TIPS - -2.6 Valid CSV -============= - -hledger follows RFC 4180, with the addition of a customisable separator -character. - - Some things to note: - - When quoting fields, - - * you must use double quotes, not single quotes - * spaces outside the quotes are not allowed. - - -Tag Table: -Node: Top72 -Node: CSV RULES2165 -Ref: #csv-rules2273 -Node: skip2536 -Ref: #skip2630 -Node: date-format2802 -Ref: #date-format2929 -Node: field list3479 -Ref: #field-list3616 -Node: field assignment4346 -Ref: #field-assignment4501 -Node: conditional block5125 -Ref: #conditional-block5279 -Node: include6175 -Ref: #include6305 -Node: newest-first6536 -Ref: #newest-first6650 -Node: CSV TIPS7061 -Ref: #csv-tips7155 -Node: CSV ordering7299 -Ref: #csv-ordering7417 -Node: CSV accounts7598 -Ref: #csv-accounts7736 -Node: CSV amounts7990 -Ref: #csv-amounts8148 -Node: CSV balance assertions/assignments9228 -Ref: #csv-balance-assertionsassignments9446 -Node: Reading multiple CSV files9767 -Ref: #reading-multiple-csv-files9967 -Node: Valid CSV10241 -Ref: #valid-csv10364 +File: hledger_csv.info, Node: Top, Next: EXAMPLES, Up: (dir) + +hledger_csv(5) hledger 1.16 +*************************** + +hledger can read CSV (comma-separated value, or character-separated +value) files as if they were journal files, automatically converting +each CSV record into a transaction. (To learn about _writing_ CSV, see +CSV output.) + + We describe each CSV file's format with a corresponding _rules file_. +By default this is named like the CSV file with a '.rules' extension +added. Eg when reading 'FILE.csv', hledger also looks for +'FILE.csv.rules' in the same directory. You can specify a different +rules file with the '--rules-file' option. If a rules file is not +found, hledger will create a sample rules file, which you'll need to +adjust. + + This file contains rules describing the CSV data (header line, fields +layout, date format etc.), and how to construct hledger journal entries +(transactions) from it. Often there will also be a list of conditional +rules for categorising transactions based on their descriptions. Here's +an overview of the CSV rules; these are described more fully below, +after the examples: + +*'skip'* skip one or more header lines or matched CSV records +*'fields'* name CSV fields, assign them to hledger fields +*field assign a value to one hledger field, with interpolation +assignment* +*'if'* apply some rules to matched CSV records +*'end'* skip the remaining CSV records +*'date-format'* describe the format of CSV dates +*'newest-first'* disambiguate record order when there's only one date +*'include'* inline another CSV rules file + + There's also a Convert CSV files tutorial on hledger.org. + +* Menu: + +* EXAMPLES:: +* CSV RULES:: +* TIPS:: + + +File: hledger_csv.info, Node: EXAMPLES, Next: CSV RULES, Prev: Top, Up: Top + +1 EXAMPLES +********** + +Here are some sample hledger CSV rules files. See also the full +collection at: +https://github.com/simonmichael/hledger/tree/master/examples/csv + +* Menu: + +* Basic:: +* Bank of Ireland:: +* Amazon:: +* Paypal:: + + +File: hledger_csv.info, Node: Basic, Next: Bank of Ireland, Up: EXAMPLES + +1.1 Basic +========= + +At minimum, the rules file must identify the date and amount fields, and +often it also specifies the date format and how many header lines there +are. Here's a simple CSV file and a rules file for it: + +Date, Description, Id, Amount +12/11/2019, Foo, 123, 10.23 + +# basic.csv.rules +skip 1 +fields date, description, _, amount +date-format %d/%m/%Y + +$ hledger print -f basic.csv +2019/11/12 Foo + expenses:unknown 10.23 + income:unknown -10.23 + + Default account names are chosen, since we didn't set them. + + +File: hledger_csv.info, Node: Bank of Ireland, Next: Amazon, Prev: Basic, Up: EXAMPLES + +1.2 Bank of Ireland +=================== + +Here's a CSV with two amount fields (Debit and Credit), and a balance +field, which we can use to add balance assertions, which is not +necessary but provides extra error checking: + +Date,Details,Debit,Credit,Balance +07/12/2012,LODGMENT 529898,,10.0,131.21 +07/12/2012,PAYMENT,5,,126 + +# bankofireland-checking.csv.rules + +# skip the header line +skip + +# name the csv fields, and assign some of them as journal entry fields +fields date, description, amount-out, amount-in, balance + +# We generate balance assertions by assigning to "balance" +# above, but you may sometimes need to remove these because: +# +# - the CSV balance differs from the true balance, +# by up to 0.0000000000005 in my experience +# +# - it is sometimes calculated based on non-chronological ordering, +# eg when multiple transactions clear on the same day + +# date is in UK/Ireland format +date-format %d/%m/%Y + +# set the currency +currency EUR + +# set the base account for all txns +account1 assets:bank:boi:checking + +$ hledger -f bankofireland-checking.csv print +2012/12/07 LODGMENT 529898 + assets:bank:boi:checking EUR10.0 = EUR131.2 + income:unknown EUR-10.0 + +2012/12/07 PAYMENT + assets:bank:boi:checking EUR-5.0 = EUR126.0 + expenses:unknown EUR5.0 + + The balance assertions don't raise an error above, because we're +reading directly from CSV, but they will be checked if these entries are +imported into a journal file. + + +File: hledger_csv.info, Node: Amazon, Next: Paypal, Prev: Bank of Ireland, Up: EXAMPLES + +1.3 Amazon +========== + +Here we convert amazon.com order history, and use an if block to +generate a third posting if there's a fee. (In practice you'd probably +get this data from your bank instead, but it's an example.) + +"Date","Type","To/From","Name","Status","Amount","Fees","Transaction ID" +"Jul 29, 2012","Payment","To","Foo.","Completed","$20.00","$0.00","16000000000000DGLNJPI1P9B8DKPVHL" +"Jul 30, 2012","Payment","To","Adapteva, Inc.","Completed","$25.00","$1.00","17LA58JSKRD4HDGLNJPI1P9B8DKPVHL" + +# amazon-orders.csv.rules + +# skip one header line +skip 1 + +# name the csv fields, and assign the transaction's date, amount and code. +# Avoided the "status" and "amount" hledger field names to prevent confusion. +fields date, _, toorfrom, name, amzstatus, amzamount, fees, code + +# how to parse the date +date-format %b %-d, %Y + +# combine two fields to make the description +description %toorfrom %name + +# save the status as a tag +comment status:%amzstatus + +# set the base account for all transactions +account1 assets:amazon +# leave amount1 blank so it can balance the other(s). +# I'm assuming amzamount excludes the fees, don't remember + +# set a generic account2 +account2 expenses:misc +amount2 %amzamount +# and maybe refine it further: +#include categorisation.rules + +# add a third posting for fees, but only if they are non-zero. +# Commas in the data makes counting fields hard, so count from the right instead. +# (Regex translation: "a field containing a non-zero dollar amount, +# immediately before the 1 right-most fields") +if ,\$[1-9][.0-9]+(,[^,]*){1}$ + account3 expenses:fees + amount3 %fees + +$ hledger -f amazon-orders.csv print +2012/07/29 (16000000000000DGLNJPI1P9B8DKPVHL) To Foo. ; status:Completed + assets:amazon + expenses:misc $20.00 + +2012/07/30 (17LA58JSKRD4HDGLNJPI1P9B8DKPVHL) To Adapteva, Inc. ; status:Completed + assets:amazon + expenses:misc $25.00 + expenses:fees $1.00 + + +File: hledger_csv.info, Node: Paypal, Prev: Amazon, Up: EXAMPLES + +1.4 Paypal +========== + +Here's a real-world rules file for (customised) Paypal CSV, with some +Paypal-specific rules, and a second rules file included: + +"Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Item Title","Item ID","Reference Txn ID","Receipt ID","Balance","Note" +"10/01/2019","03:46:20","PDT","Calm Radio","Subscription Payment","Completed","USD","-6.99","0.00","-6.99","simon@joyful.com","memberships@calmradio.com","60P57143A8206782E","MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month","","I-R8YLY094FJYR","","-6.99","" +"10/01/2019","03:46:20","PDT","","Bank Deposit to PP Account ","Pending","USD","6.99","0.00","6.99","","simon@joyful.com","0TU1544T080463733","","","60P57143A8206782E","","0.00","" +"10/01/2019","08:57:01","PDT","Patreon","PreApproved Payment Bill User Payment","Completed","USD","-7.00","0.00","-7.00","simon@joyful.com","support@patreon.com","2722394R5F586712G","Patreon* Membership","","B-0PG93074E7M86381M","","-7.00","" +"10/01/2019","08:57:01","PDT","","Bank Deposit to PP Account ","Pending","USD","7.00","0.00","7.00","","simon@joyful.com","71854087RG994194F","Patreon* Membership","","2722394R5F586712G","","0.00","" +"10/19/2019","03:02:12","PDT","Wikimedia Foundation, Inc.","Subscription Payment","Completed","USD","-2.00","0.00","-2.00","simon@joyful.com","tle@wikimedia.org","K9U43044RY432050M","Monthly donation to the Wikimedia Foundation","","I-R5C3YUS3285L","","-2.00","" +"10/19/2019","03:02:12","PDT","","Bank Deposit to PP Account ","Pending","USD","2.00","0.00","2.00","","simon@joyful.com","3XJ107139A851061F","","","K9U43044RY432050M","","0.00","" +"10/22/2019","05:07:06","PDT","Noble Benefactor","Subscription Payment","Completed","USD","10.00","-0.59","9.41","noble@bene.fac.tor","simon@joyful.com","6L8L1662YP1334033","Joyful Systems","","I-KC9VBGY2GWDB","","9.41","" + +# paypal-custom.csv.rules + +# Tips: +# Export from Activity -> Statements -> Custom -> Activity download +# Suggested transaction type: "Balance affecting" +# Paypal's default fields in 2018 were: +# "Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Shipping Address","Address Status","Item Title","Item ID","Shipping and Handling Amount","Insurance Amount","Sales Tax","Option 1 Name","Option 1 Value","Option 2 Name","Option 2 Value","Reference Txn ID","Invoice Number","Custom Number","Quantity","Receipt ID","Balance","Address Line 1","Address Line 2/District/Neighborhood","Town/City","State/Province/Region/County/Territory/Prefecture/Republic","Zip/Postal Code","Country","Contact Phone Number","Subject","Note","Country Code","Balance Impact" +# This rules file assumes the following more detailed fields, configured in "Customize report fields": +# "Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Item Title","Item ID","Reference Txn ID","Receipt ID","Balance","Note" + +fields date, time, timezone, description_, type, status_, currency, grossamount, feeamount, netamount, fromemail, toemail, code, itemtitle, itemid, referencetxnid, receiptid, balance, note + +skip 1 + +date-format %-m/%-d/%Y + +# ignore some paypal events +if +In Progress +Temporary Hold +Update to + skip + +# add more fields to the description +description %description_ %itemtitle + +# save some other fields as tags +comment itemid:%itemid, fromemail:%fromemail, toemail:%toemail, time:%time, type:%type, status:%status_ + +# convert to short currency symbols +# Note: in conditional block regexps, the line of csv being matched is +# a synthetic one: the unquoted field values, with commas between them. +if ,USD, + currency $ +if ,EUR, + currency E +if ,GBP, + currency P + +# generate postings + +# the first posting will be the money leaving/entering my paypal account +# (negative means leaving my account, in all amount fields) +account1 assets:online:paypal +amount1 %netamount + +# the second posting will be money sent to/received from other party +# (account2 is set below) +amount2 -%grossamount + +# if there's a fee (9th field), add a third posting for the money taken by paypal. +# TODO: This regexp fails when fields contain a comma (generates a third posting with zero amount) +if ^([^,]+,){8}[^0] + account3 expenses:banking:paypal + amount3 -%feeamount + comment3 business: + +# choose an account for the second posting + +# override the default account names: +# if amount (8th field) is positive, it's income (a debit) +if ^([^,]+,){7}[0-9] + account2 income:unknown +# if negative, it's an expense (a credit) +if ^([^,]+,){7}- + account2 expenses:unknown + +# apply common rules for setting account2 & other tweaks +include common.rules + +# apply some overrides specific to this csv + +# Transfers from/to bank. These are usually marked Pending, +# which can be disregarded in this case. +if +Bank Account +Bank Deposit to PP Account + description %type for %referencetxnid %itemtitle + account2 assets:bank:wf:pchecking + account1 assets:online:paypal + +# Currency conversions +if Currency Conversion + account2 equity:currency conversion + +# common.rules + +if +darcs +noble benefactor + account2 revenues:foss donations:darcshub + comment2 business: + +if +Calm Radio + account2 expenses:online:apps + +if +electronic frontier foundation +Patreon +wikimedia +Advent of Code + account2 expenses:dues + +if Google + account2 expenses:online:apps + description google | music + +$ hledger -f paypal-custom.csv print +2019/10/01 (60P57143A8206782E) Calm Radio MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month ; itemid:, fromemail:simon@joyful.com, toemail:memberships@calmradio.com, time:03:46:20, type:Subscription Payment, status:Completed + assets:online:paypal $-6.99 = $-6.99 + expenses:online:apps $6.99 + +2019/10/01 (0TU1544T080463733) Bank Deposit to PP Account for 60P57143A8206782E ; itemid:, fromemail:, toemail:simon@joyful.com, time:03:46:20, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $6.99 = $0.00 + assets:bank:wf:pchecking $-6.99 + +2019/10/01 (2722394R5F586712G) Patreon Patreon* Membership ; itemid:, fromemail:simon@joyful.com, toemail:support@patreon.com, time:08:57:01, type:PreApproved Payment Bill User Payment, status:Completed + assets:online:paypal $-7.00 = $-7.00 + expenses:dues $7.00 + +2019/10/01 (71854087RG994194F) Bank Deposit to PP Account for 2722394R5F586712G Patreon* Membership ; itemid:, fromemail:, toemail:simon@joyful.com, time:08:57:01, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $7.00 = $0.00 + assets:bank:wf:pchecking $-7.00 + +2019/10/19 (K9U43044RY432050M) Wikimedia Foundation, Inc. Monthly donation to the Wikimedia Foundation ; itemid:, fromemail:simon@joyful.com, toemail:tle@wikimedia.org, time:03:02:12, type:Subscription Payment, status:Completed + assets:online:paypal $-2.00 = $-2.00 + expenses:dues $2.00 + expenses:banking:paypal ; business: + +2019/10/19 (3XJ107139A851061F) Bank Deposit to PP Account for K9U43044RY432050M ; itemid:, fromemail:, toemail:simon@joyful.com, time:03:02:12, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $2.00 = $0.00 + assets:bank:wf:pchecking $-2.00 + +2019/10/22 (6L8L1662YP1334033) Noble Benefactor Joyful Systems ; itemid:, fromemail:noble@bene.fac.tor, toemail:simon@joyful.com, time:05:07:06, type:Subscription Payment, status:Completed + assets:online:paypal $9.41 = $9.41 + revenues:foss donations:darcshub $-10.00 ; business: + expenses:banking:paypal $0.59 ; business: + + +File: hledger_csv.info, Node: CSV RULES, Next: TIPS, Prev: EXAMPLES, Up: Top + +2 CSV RULES +*********** + +The following kinds of rule can appear in the rules file, in any order. +Blank lines and lines beginning with '#' or ';' are ignored. + +* Menu: + +* skip:: +* fields:: +* field assignment:: +* if:: +* end:: +* date-format:: +* newest-first:: +* include:: + + +File: hledger_csv.info, Node: skip, Next: fields, Up: CSV RULES + +2.1 'skip' +========== + +skip N + + The word "skip" followed by a number (or no number, meaning 1) tells +hledger to ignore this many non-empty lines preceding the CSV data. +(Empty/blank lines are skipped automatically.) You'll need this +whenever your CSV data contains header lines. + + It also has a second purpose: it can be used inside if blocks to +ignore certain CSV records (described below). + + +File: hledger_csv.info, Node: fields, Next: field assignment, Prev: skip, Up: CSV RULES + +2.2 'fields' +============ + +fields FIELDNAME1, FIELDNAME2, ... + + A fields list (the word "fields" followed by comma-separated field +names) is the quick way to assign CSV field values to hledger fields. +It does two things: + + 1. it names the CSV fields. This is optional, but can be convenient + later for interpolating them. + + 2. when you use a standard hledger field name, it assigns the CSV + value to that part of the hledger transaction. + + Here's an example that says "use the 1st, 2nd and 4th fields as the +transaction's date, description and amount; name the last two fields for +later reference; and ignore the others": + +fields date, description, , amount, , , somefield, anotherfield + + Field names may not contain whitespace. Fields you don't care about +can be left unnamed. Currently there must be least two items (there +must be at least one comma). + + Here are the standard hledger field/pseudo-field names. For more +about the transaction parts they refer to, see the manual for hledger's +journal format. + +* Menu: + +* Transaction field names:: +* Posting field names:: + + +File: hledger_csv.info, Node: Transaction field names, Next: Posting field names, Up: fields + +2.2.1 Transaction field names +----------------------------- + +'date', 'date2', 'status', 'code', 'description', 'comment' can be used +to form the transaction's first line. + + +File: hledger_csv.info, Node: Posting field names, Prev: Transaction field names, Up: fields + +2.2.2 Posting field names +------------------------- + +'accountN', where N is 1 to 9, generates a posting, with that account +name. Most often there are two postings, so you'll want to set +'account1' and 'account2'. If a posting's account name is left unset +but its amount is set, a default account name will be chosen (like +expenses:unknown or income:unknown). + + 'amountN' sets posting N's amount. Or, 'amount' with no N sets +posting 1's. If the CSV has debits and credits in separate fields, use +'amountN-in' and 'amountN-out' instead. Or 'amount-in' and 'amount-out' +with no N for posting 1. + + For convenience and backwards compatibility, if you set the amount of +posting 1 only, a second posting with the negative amount will be +generated automatically. (Unless the account name is parenthesised +indicating an unbalanced posting.) + + If the CSV has the currency symbol in a separate field, you can use +'currencyN' to prepend it to posting N's amount. 'currency' with no N +affects ALL postings. + + 'balanceN' sets a balance assertion amount (or if the posting amount +is left empty, a balance assignment). + + Finally, 'commentN' sets a comment on the Nth posting. Comments can +also contain tags, as usual. + + See TIPS below for more about setting amounts and currency. + + +File: hledger_csv.info, Node: field assignment, Next: if, Prev: fields, Up: CSV RULES + +2.3 field assignment +==================== + +HLEDGERFIELDNAME FIELDVALUE + + Instead of or in addition to a fields list, you can use a "field +assignment" rule to set the value of a single hledger field, by writing +its name (any of the standard hledger field names above) followed by a +text value. The value may contain interpolated CSV fields, referenced +by their 1-based position in the CSV record ('%N'), or by the name they +were given in the fields list ('%CSVFIELDNAME'). Some examples: + +# set the amount to the 4th CSV field, with " USD" appended +amount %4 USD + +# combine three fields to make a comment, containing note: and date: tags +comment note: %somefield - %anotherfield, date: %1 + + Interpolation strips outer whitespace (so a CSV value like '" 1 "' +becomes '1' when interpolated) (#1051). See TIPS below for more about +referencing other fields. + + +File: hledger_csv.info, Node: if, Next: end, Prev: field assignment, Up: CSV RULES + +2.4 'if' +======== + +if PATTERN + RULE + +if +PATTERN +PATTERN +PATTERN + RULE + RULE + + Conditional blocks ("if blocks") are a block of rules that are +applied only to CSV records which match certain patterns. They are +often used for customising account names based on transaction +descriptions. + + A single pattern can be written on the same line as the "if"; or +multiple patterns can be written on the following lines, non-indented. +Multiple patterns are OR'd (any one of them can match). Patterns are +case-insensitive regular expressions which try to match anywhere within +the whole CSV record (POSIX extended regular expressions with some +additions, see https://hledger.org/hledger.html#regular-expressions). +Note the CSV record they see is close to, but not identical to, the one +in the CSV file; enclosing double quotes will be removed, and the +separator character is always comma. + + It's not yet easy to match within a specific field. If the data does +not contain commas, you can hack it with a regular expression like: + +# match "foo" in the fourth field +if ^([^,]*,){3}foo + + After the patterns there should be one or more rules to apply, all +indented by at least one space. Three kinds of rule are allowed in +conditional blocks: + + * field assignments (to set a hledger field) + * skip (to skip the matched CSV record) + * end (to skip all remaining CSV records). + + Examples: + +# if the CSV record contains "groceries", set account2 to "expenses:groceries" +if groceries + account2 expenses:groceries + +# if the CSV record contains any of these patterns, set account2 and comment as shown +if +monthly service fee +atm transaction fee +banking thru software + account2 expenses:business:banking + comment XXX deductible ? check it + + +File: hledger_csv.info, Node: end, Next: date-format, Prev: if, Up: CSV RULES + +2.5 'end' +========= + +This rule can be used inside if blocks (only), to make hledger stop +reading this CSV file and move on to the next input file, or to command +execution. Eg: + +# ignore everything following the first empty record +if ,,,, + end + + +File: hledger_csv.info, Node: date-format, Next: newest-first, Prev: end, Up: CSV RULES + +2.6 'date-format' +================= + +date-format DATEFMT + + This is a helper for the 'date' (and 'date2') fields. If your CSV +dates are not formatted like 'YYYY-MM-DD', 'YYYY/MM/DD' or 'YYYY.MM.DD', +you'll need to add a date-format rule describing them with a strptime +date parsing pattern, which must parse the CSV date value completely. +Some examples: + +# MM/DD/YY +date-format %m/%d/%y + +# D/M/YYYY +# The - makes leading zeros optional. +date-format %-d/%-m/%Y + +# YYYY-Mmm-DD +date-format %Y-%h-%d + +# M/D/YYYY HH:MM AM some other junk +# Note the time and junk must be fully parsed, though only the date is used. +date-format %-m/%-d/%Y %l:%M %p some other junk + + For the supported strptime syntax, see: +https://hackage.haskell.org/package/time/docs/Data-Time-Format.html#v:formatTime + + +File: hledger_csv.info, Node: newest-first, Next: include, Prev: date-format, Up: CSV RULES + +2.7 'newest-first' +================== + +hledger always sorts the generated transactions by date. Transactions +on the same date should appear in the same order as their CSV records, +as hledger can usually auto-detect whether the CSV's normal order is +oldest first or newest first. But if all of the following are true: + + * the CSV might sometimes contain just one day of data (all records + having the same date) + * the CSV records are normally in reverse chronological order (newest + at the top) + * and you care about preserving the order of same-day transactions + + then, you should add the 'newest-first' rule as a hint. Eg: + +# tell hledger explicitly that the CSV is normally newest first +newest-first + + +File: hledger_csv.info, Node: include, Prev: newest-first, Up: CSV RULES + +2.8 'include' +============= + +include RULESFILE + + This includes the contents of another CSV rules file at this point. +'RULESFILE' is an absolute file path or a path relative to the current +file's directory. This can be useful for sharing common rules between +several rules files, eg: + +# someaccount.csv.rules + +## someaccount-specific rules +fields date,description,amount +account1 assets:someaccount +account2 expenses:misc + +## common rules +include categorisation.rules + + +File: hledger_csv.info, Node: TIPS, Prev: CSV RULES, Up: Top + +3 TIPS +****** + +* Menu: + +* Valid CSV:: +* Other separator characters:: +* Reading multiple CSV files:: +* Valid transactions:: +* Deduplicating importing:: +* Setting amounts:: +* Setting currency/commodity:: +* Referencing other fields:: +* How CSV rules are evaluated:: + + +File: hledger_csv.info, Node: Valid CSV, Next: Other separator characters, Up: TIPS + +3.1 Valid CSV +============= + +hledger accepts CSV conforming to RFC 4180. When CSV values are +enclosed in quotes, note: + + * they must be double quotes (not single quotes) + * spaces outside the quotes are not allowed + + +File: hledger_csv.info, Node: Other separator characters, Next: Reading multiple CSV files, Prev: Valid CSV, Up: TIPS + +3.2 Other separator characters +============================== + +With the '--separator 'CHAR'' option (experimental), hledger will expect +the separator to be CHAR instead of a comma. Ie it will read other +"Character Separated Values" formats, such as TSV (Tab Separated +Values). Note: on the command line, use a real tab character in quotes, +not + +$ hledger -f foo.tsv --separator ' ' print + + +File: hledger_csv.info, Node: Reading multiple CSV files, Next: Valid transactions, Prev: Other separator characters, Up: TIPS + +3.3 Reading multiple CSV files +============================== + +If you use multiple '-f' options to read multiple CSV files at once, +hledger will look for a correspondingly-named rules file for each CSV +file. But if you use the '--rules-file' option, that rules file will be +used for all the CSV files. + + +File: hledger_csv.info, Node: Valid transactions, Next: Deduplicating importing, Prev: Reading multiple CSV files, Up: TIPS + +3.4 Valid transactions +====================== + +After reading a CSV file, hledger post-processes and validates the +generated journal entries as it would for a journal file - balancing +them, applying balance assignments, and canonicalising amount styles. +Any errors at this stage will be reported in the usual way, displaying +the problem entry. + + There is one exception: balance assertions, if you have generated +them, will not be checked, since normally these will work only when the +CSV data is part of the main journal. If you do need to check balance +assertions generated from CSV right away, pipe into another hledger: + +$ hledger -f file.csv print | hledger -f- print + + +File: hledger_csv.info, Node: Deduplicating importing, Next: Setting amounts, Prev: Valid transactions, Up: TIPS + +3.5 Deduplicating, importing +============================ + +When you download a CSV file periodically, eg to get your latest bank +transactions, the new file may overlap with the old one, containing some +of the same records. + + The import command will (a) detect the new transactions, and (b) +append just those transactions to your main journal. It is idempotent, +so you don't have to remember how many times you ran it or with which +version of the CSV. (It keeps state in a hidden '.latest.FILE.csv' +file.) This is the easiest way to import CSV data. Eg: + +# download the latest CSV files, then run this command. +# Note, no -f flags needed here. +$ hledger import *.csv [--dry] + + This method works for most CSV files. (Where records have a stable +chronological order, and new records appear only at the new end.) + + A number of other tools and workflows, hledger-specific and +otherwise, exist for converting, deduplicating, classifying and managing +CSV data. See: + + * https://hledger.org -> sidebar -> real world setups + * https://plaintextaccounting.org -> data import/conversion + + +File: hledger_csv.info, Node: Setting amounts, Next: Setting currency/commodity, Prev: Deduplicating importing, Up: TIPS + +3.6 Setting amounts +=================== + +A posting amount can be set in one of these ways: + + * by assigning (with a fields list or field assigment) to 'amountN' + (posting N's amount) or 'amount' (posting 1's amount) + + * by assigning to 'amountN-in' and 'amountN-out' (or 'amount-in' and + 'amount-out'). For each CSV record, whichever of these has a + non-zero value will be used, with appropriate sign. If both + contain a non-zero value, this may not work. + + * by assigning to 'balanceN' (or 'balance') instead of the above, + setting the amount indirectly via a balance assignment. If you do + this the default account name may be wrong, so you should set that + explicitly. + + There is some special handling for an amount's sign: + + * If an amount value is parenthesised, it will be de-parenthesised + and sign-flipped. + * If an amount value begins with a double minus sign, those cancel + out and are removed. + * If an amount value begins with a plus sign, that will be removed + + +File: hledger_csv.info, Node: Setting currency/commodity, Next: Referencing other fields, Prev: Setting amounts, Up: TIPS + +3.7 Setting currency/commodity +============================== + +If the currency/commodity symbol is included in the CSV's amount +field(s), you don't have to do anything special. + + If the currency is provided as a separate CSV field, you can either: + + * assign that to 'currency', which adds it to all posting amounts. + The symbol will prepended to the amount quantity (on the left + side). If you write a trailing space after the symbol, there will + be a space between symbol and amount (an exception to the usual + whitespace stripping). + + * or assign it to 'currencyN' which adds it to posting N's amount + only. + + * or for more control, construct the amount from symbol and quantity + using field assignment, eg: + + fields date,description,currency,quantity + # add currency symbol on the right: + amount %quantity %currency + + +File: hledger_csv.info, Node: Referencing other fields, Next: How CSV rules are evaluated, Prev: Setting currency/commodity, Up: TIPS + +3.8 Referencing other fields +============================ + +In field assignments, you can interpolate only CSV fields, not hledger +fields. In the example below, there's both a CSV field and a hledger +field named amount1, but %amount1 always means the CSV field, not the +hledger field: + +# Name the third CSV field "amount1" +fields date,description,amount1 + +# Set hledger's amount1 to the CSV amount1 field followed by USD +amount1 %amount1 USD + +# Set comment to the CSV amount1 (not the amount1 assigned above) +comment %amount1 + + Here, since there's no CSV amount1 field, %amount1 will produce a +literal "amount1": + +fields date,description,csvamount +amount1 %csvamount USD +# Can't interpolate amount1 here +comment %amount1 + + When there are multiple field assignments to the same hledger field, +only the last one takes effect. Here, comment's value will be be B, or +C if "something" is matched, but never A: + +comment A +comment B +if something + comment C + + +File: hledger_csv.info, Node: How CSV rules are evaluated, Prev: Referencing other fields, Up: TIPS + +3.9 How CSV rules are evaluated +=============================== + +Here's how to think of CSV rules being evaluated (if you really need +to). First, + + * 'include' - all includes are inlined, from top to bottom, depth + first. (At each include point the file is inlined and scanned for + further includes, recursively, before proceeding.) + + Then "global" rules are evaluated, top to bottom. If a rule is +repeated, the last one wins: + + * 'skip' (at top level) + * 'date-format' + * 'newest-first' + * 'fields' - names the CSV fields, optionally sets up initial + assignments to hledger fields + + Then for each CSV record in turn: + + * test all 'if' blocks. If any of them contain a 'end' rule, skip + all remaining CSV records. Otherwise if any of them contain a + 'skip' rule, skip that many CSV records. If there are multiple + matched 'skip' rules, the first one wins. + * collect all field assignments at top level and in matched 'if' + blocks. When there are multiple assignments for a field, keep only + the last one. + * compute a value for each hledger field - either the one that was + assigned to it (and interpolate the %CSVFIELDNAME references), or a + default + * generate a synthetic hledger transaction from these values. + + This is all part of the CSV reader, one of several readers hledger +can use to parse input files. When all files have been read +successfully, the transactions are passed as input to whichever hledger +command the user specified. + + +Tag Table: +Node: Top72 +Node: EXAMPLES1829 +Ref: #examples1935 +Node: Basic2143 +Ref: #basic2243 +Node: Bank of Ireland2785 +Ref: #bank-of-ireland2920 +Node: Amazon4383 +Ref: #amazon4501 +Node: Paypal6434 +Ref: #paypal6528 +Node: CSV RULES14411 +Ref: #csv-rules14520 +Node: skip14765 +Ref: #skip14858 +Node: fields15233 +Ref: #fields15355 +Node: Transaction field names16422 +Ref: #transaction-field-names16582 +Node: Posting field names16693 +Ref: #posting-field-names16845 +Node: field assignment18077 +Ref: #field-assignment18213 +Node: if19031 +Ref: #if19140 +Node: end20856 +Ref: #end20962 +Node: date-format21186 +Ref: #date-format21318 +Node: newest-first22067 +Ref: #newest-first22205 +Node: include22888 +Ref: #include22996 +Node: TIPS23440 +Ref: #tips23522 +Node: Valid CSV23771 +Ref: #valid-csv23890 +Node: Other separator characters24082 +Ref: #other-separator-characters24270 +Node: Reading multiple CSV files24599 +Ref: #reading-multiple-csv-files24796 +Node: Valid transactions25037 +Ref: #valid-transactions25215 +Node: Deduplicating importing25843 +Ref: #deduplicating-importing26022 +Node: Setting amounts27055 +Ref: #setting-amounts27224 +Node: Setting currency/commodity28210 +Ref: #setting-currencycommodity28402 +Node: Referencing other fields29205 +Ref: #referencing-other-fields29405 +Node: How CSV rules are evaluated30302 +Ref: #how-csv-rules-are-evaluated30473  End Tag Table diff --git a/hledger_csv.txt b/hledger_csv.txt --- a/hledger_csv.txt +++ b/hledger_csv.txt @@ -7,48 +7,136 @@ CSV - how hledger reads CSV data, and the CSV rules file format DESCRIPTION - hledger can read CSV (comma-separated value) files as if they were - journal files, automatically converting each CSV record into a transac- - tion. (To learn about writing CSV, see CSV output.) + hledger can read CSV (comma-separated value, or character-separated + value) files as if they were journal files, automatically converting + each CSV record into a transaction. (To learn about writing CSV, see + CSV output.) - Converting CSV to transactions requires some special conversion rules. - These do several things: + We describe each CSV file's format with a corresponding rules file. By + default this is named like the CSV file with a .rules extension added. + Eg when reading FILE.csv, hledger also looks for FILE.csv.rules in the + same directory. You can specify a different rules file with the + --rules-file option. If a rules file is not found, hledger will create + a sample rules file, which you'll need to adjust. - o they describe the layout and format of the CSV data + This file contains rules describing the CSV data (header line, fields + layout, date format etc.), and how to construct hledger journal entries + (transactions) from it. Often there will also be a list of conditional + rules for categorising transactions based on their descriptions. + Here's an overview of the CSV rules; these are described more fully be- + low, after the examples: - o they can customize the generated journal entries using a simple tem- - plating language + skip skip one or more header + lines or matched CSV + records + fields name CSV fields, assign + them to hledger fields + field assignment assign a value to one + hledger field, with inter- + polation + if apply some rules to + matched CSV records + end skip the remaining CSV + records + date-format describe the format of CSV + dates + newest-first disambiguate record order + when there's only one date + include inline another CSV rules + file - o they can add refinements based on patterns in the CSV data, eg cate- - gorizing transactions with more detailed account names. + There's also a Convert CSV files tutorial on hledger.org. - When reading a CSV file named FILE.csv, hledger looks for a conversion - rules file named FILE.csv.rules in the same directory. You can over- - ride this with the --rules-file option. If the rules file does not ex- - ist, hledger will auto-create one with some example rules, which you'll - need to adjust. +EXAMPLES + Here are some sample hledger CSV rules files. See also the full col- + lection at: + https://github.com/simonmichael/hledger/tree/master/examples/csv - At minimum, the rules file must identify the date and amount fields. - It's often necessary to specify the date format, and the number of - header lines to skip, also. Eg: + Basic + At minimum, the rules file must identify the date and amount fields, + and often it also specifies the date format and how many header lines + there are. Here's a simple CSV file and a rules file for it: - fields date, _, _, amount + Date, Description, Id, Amount + 12/11/2019, Foo, 123, 10.23 + + # basic.csv.rules + skip 1 + fields date, description, _, amount date-format %d/%m/%Y - skip 1 - A more complete example: + $ hledger print -f basic.csv + 2019/11/12 Foo + expenses:unknown 10.23 + income:unknown -10.23 - # hledger CSV rules for amazon.com order history + Default account names are chosen, since we didn't set them. - # sample: - # "Date","Type","To/From","Name","Status","Amount","Fees","Transaction ID" - # "Jul 29, 2012","Payment","To","Adapteva, Inc.","Completed","$25.00","$0.00","17LA58JSK6PRD4HDGLNJQPI1PB9N8DKPVHL" + Bank of Ireland + Here's a CSV with two amount fields (Debit and Credit), and a balance + field, which we can use to add balance assertions, which is not neces- + sary but provides extra error checking: + Date,Details,Debit,Credit,Balance + 07/12/2012,LODGMENT 529898,,10.0,131.21 + 07/12/2012,PAYMENT,5,,126 + + # bankofireland-checking.csv.rules + + # skip the header line + skip + + # name the csv fields, and assign some of them as journal entry fields + fields date, description, amount-out, amount-in, balance + + # We generate balance assertions by assigning to "balance" + # above, but you may sometimes need to remove these because: + # + # - the CSV balance differs from the true balance, + # by up to 0.0000000000005 in my experience + # + # - it is sometimes calculated based on non-chronological ordering, + # eg when multiple transactions clear on the same day + + # date is in UK/Ireland format + date-format %d/%m/%Y + + # set the currency + currency EUR + + # set the base account for all txns + account1 assets:bank:boi:checking + + $ hledger -f bankofireland-checking.csv print + 2012/12/07 LODGMENT 529898 + assets:bank:boi:checking EUR10.0 = EUR131.2 + income:unknown EUR-10.0 + + 2012/12/07 PAYMENT + assets:bank:boi:checking EUR-5.0 = EUR126.0 + expenses:unknown EUR5.0 + + The balance assertions don't raise an error above, because we're read- + ing directly from CSV, but they will be checked if these entries are + imported into a journal file. + + Amazon + Here we convert amazon.com order history, and use an if block to gener- + ate a third posting if there's a fee. (In practice you'd probably get + this data from your bank instead, but it's an example.) + + "Date","Type","To/From","Name","Status","Amount","Fees","Transaction ID" + "Jul 29, 2012","Payment","To","Foo.","Completed","$20.00","$0.00","16000000000000DGLNJPI1P9B8DKPVHL" + "Jul 30, 2012","Payment","To","Adapteva, Inc.","Completed","$25.00","$1.00","17LA58JSKRD4HDGLNJPI1P9B8DKPVHL" + + # amazon-orders.csv.rules + # skip one header line skip 1 - # name the csv fields (and assign the transaction's date, amount and code) - fields date, _, toorfrom, name, amzstatus, amount, fees, code + # name the csv fields, and assign the transaction's date, amount and code. + # Avoided the "status" and "amount" hledger field names to prevent confusion. + fields date, _, toorfrom, name, amzstatus, amzamount, fees, code # how to parse the date date-format %b %-d, %Y @@ -56,104 +144,328 @@ # combine two fields to make the description description %toorfrom %name - # save these fields as tags - comment status:%amzstatus, fees:%fees + # save the status as a tag + comment status:%amzstatus # set the base account for all transactions account1 assets:amazon + # leave amount1 blank so it can balance the other(s). + # I'm assuming amzamount excludes the fees, don't remember - # flip the sign on the amount - amount -%amount + # set a generic account2 + account2 expenses:misc + amount2 %amzamount + # and maybe refine it further: + #include categorisation.rules - For more examples, see Convert CSV files. + # add a third posting for fees, but only if they are non-zero. + # Commas in the data makes counting fields hard, so count from the right instead. + # (Regex translation: "a field containing a non-zero dollar amount, + # immediately before the 1 right-most fields") + if ,\$[1-9][.0-9]+(,[^,]*){1}$ + account3 expenses:fees + amount3 %fees + $ hledger -f amazon-orders.csv print + 2012/07/29 (16000000000000DGLNJPI1P9B8DKPVHL) To Foo. ; status:Completed + assets:amazon + expenses:misc $20.00 + + 2012/07/30 (17LA58JSKRD4HDGLNJPI1P9B8DKPVHL) To Adapteva, Inc. ; status:Completed + assets:amazon + expenses:misc $25.00 + expenses:fees $1.00 + + Paypal + Here's a real-world rules file for (customised) Paypal CSV, with some + Paypal-specific rules, and a second rules file included: + + "Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Item Title","Item ID","Reference Txn ID","Receipt ID","Balance","Note" + "10/01/2019","03:46:20","PDT","Calm Radio","Subscription Payment","Completed","USD","-6.99","0.00","-6.99","simon@joyful.com","memberships@calmradio.com","60P57143A8206782E","MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month","","I-R8YLY094FJYR","","-6.99","" + "10/01/2019","03:46:20","PDT","","Bank Deposit to PP Account ","Pending","USD","6.99","0.00","6.99","","simon@joyful.com","0TU1544T080463733","","","60P57143A8206782E","","0.00","" + "10/01/2019","08:57:01","PDT","Patreon","PreApproved Payment Bill User Payment","Completed","USD","-7.00","0.00","-7.00","simon@joyful.com","support@patreon.com","2722394R5F586712G","Patreon* Membership","","B-0PG93074E7M86381M","","-7.00","" + "10/01/2019","08:57:01","PDT","","Bank Deposit to PP Account ","Pending","USD","7.00","0.00","7.00","","simon@joyful.com","71854087RG994194F","Patreon* Membership","","2722394R5F586712G","","0.00","" + "10/19/2019","03:02:12","PDT","Wikimedia Foundation, Inc.","Subscription Payment","Completed","USD","-2.00","0.00","-2.00","simon@joyful.com","tle@wikimedia.org","K9U43044RY432050M","Monthly donation to the Wikimedia Foundation","","I-R5C3YUS3285L","","-2.00","" + "10/19/2019","03:02:12","PDT","","Bank Deposit to PP Account ","Pending","USD","2.00","0.00","2.00","","simon@joyful.com","3XJ107139A851061F","","","K9U43044RY432050M","","0.00","" + "10/22/2019","05:07:06","PDT","Noble Benefactor","Subscription Payment","Completed","USD","10.00","-0.59","9.41","noble@bene.fac.tor","simon@joyful.com","6L8L1662YP1334033","Joyful Systems","","I-KC9VBGY2GWDB","","9.41","" + + # paypal-custom.csv.rules + + # Tips: + # Export from Activity -> Statements -> Custom -> Activity download + # Suggested transaction type: "Balance affecting" + # Paypal's default fields in 2018 were: + # "Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Shipping Address","Address Status","Item Title","Item ID","Shipping and Handling Amount","Insurance Amount","Sales Tax","Option 1 Name","Option 1 Value","Option 2 Name","Option 2 Value","Reference Txn ID","Invoice Number","Custom Number","Quantity","Receipt ID","Balance","Address Line 1","Address Line 2/District/Neighborhood","Town/City","State/Province/Region/County/Territory/Prefecture/Republic","Zip/Postal Code","Country","Contact Phone Number","Subject","Note","Country Code","Balance Impact" + # This rules file assumes the following more detailed fields, configured in "Customize report fields": + # "Date","Time","TimeZone","Name","Type","Status","Currency","Gross","Fee","Net","From Email Address","To Email Address","Transaction ID","Item Title","Item ID","Reference Txn ID","Receipt ID","Balance","Note" + + fields date, time, timezone, description_, type, status_, currency, grossamount, feeamount, netamount, fromemail, toemail, code, itemtitle, itemid, referencetxnid, receiptid, balance, note + + skip 1 + + date-format %-m/%-d/%Y + + # ignore some paypal events + if + In Progress + Temporary Hold + Update to + skip + + # add more fields to the description + description %description_ %itemtitle + + # save some other fields as tags + comment itemid:%itemid, fromemail:%fromemail, toemail:%toemail, time:%time, type:%type, status:%status_ + + # convert to short currency symbols + # Note: in conditional block regexps, the line of csv being matched is + # a synthetic one: the unquoted field values, with commas between them. + if ,USD, + currency $ + if ,EUR, + currency E + if ,GBP, + currency P + + # generate postings + + # the first posting will be the money leaving/entering my paypal account + # (negative means leaving my account, in all amount fields) + account1 assets:online:paypal + amount1 %netamount + + # the second posting will be money sent to/received from other party + # (account2 is set below) + amount2 -%grossamount + + # if there's a fee (9th field), add a third posting for the money taken by paypal. + # TODO: This regexp fails when fields contain a comma (generates a third posting with zero amount) + if ^([^,]+,){8}[^0] + account3 expenses:banking:paypal + amount3 -%feeamount + comment3 business: + + # choose an account for the second posting + + # override the default account names: + # if amount (8th field) is positive, it's income (a debit) + if ^([^,]+,){7}[0-9] + account2 income:unknown + # if negative, it's an expense (a credit) + if ^([^,]+,){7}- + account2 expenses:unknown + + # apply common rules for setting account2 & other tweaks + include common.rules + + # apply some overrides specific to this csv + + # Transfers from/to bank. These are usually marked Pending, + # which can be disregarded in this case. + if + Bank Account + Bank Deposit to PP Account + description %type for %referencetxnid %itemtitle + account2 assets:bank:wf:pchecking + account1 assets:online:paypal + + # Currency conversions + if Currency Conversion + account2 equity:currency conversion + + # common.rules + + if + darcs + noble benefactor + account2 revenues:foss donations:darcshub + comment2 business: + + if + Calm Radio + account2 expenses:online:apps + + if + electronic frontier foundation + Patreon + wikimedia + Advent of Code + account2 expenses:dues + + if Google + account2 expenses:online:apps + description google | music + + $ hledger -f paypal-custom.csv print + 2019/10/01 (60P57143A8206782E) Calm Radio MONTHLY - $1 for the first 2 Months: Me - Order 99309. Item total: $1.00 USD first 2 months, then $6.99 / Month ; itemid:, fromemail:simon@joyful.com, toemail:memberships@calmradio.com, time:03:46:20, type:Subscription Payment, status:Completed + assets:online:paypal $-6.99 = $-6.99 + expenses:online:apps $6.99 + + 2019/10/01 (0TU1544T080463733) Bank Deposit to PP Account for 60P57143A8206782E ; itemid:, fromemail:, toemail:simon@joyful.com, time:03:46:20, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $6.99 = $0.00 + assets:bank:wf:pchecking $-6.99 + + 2019/10/01 (2722394R5F586712G) Patreon Patreon* Membership ; itemid:, fromemail:simon@joyful.com, toemail:support@patreon.com, time:08:57:01, type:PreApproved Payment Bill User Payment, status:Completed + assets:online:paypal $-7.00 = $-7.00 + expenses:dues $7.00 + + 2019/10/01 (71854087RG994194F) Bank Deposit to PP Account for 2722394R5F586712G Patreon* Membership ; itemid:, fromemail:, toemail:simon@joyful.com, time:08:57:01, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $7.00 = $0.00 + assets:bank:wf:pchecking $-7.00 + + 2019/10/19 (K9U43044RY432050M) Wikimedia Foundation, Inc. Monthly donation to the Wikimedia Foundation ; itemid:, fromemail:simon@joyful.com, toemail:tle@wikimedia.org, time:03:02:12, type:Subscription Payment, status:Completed + assets:online:paypal $-2.00 = $-2.00 + expenses:dues $2.00 + expenses:banking:paypal ; business: + + 2019/10/19 (3XJ107139A851061F) Bank Deposit to PP Account for K9U43044RY432050M ; itemid:, fromemail:, toemail:simon@joyful.com, time:03:02:12, type:Bank Deposit to PP Account, status:Pending + assets:online:paypal $2.00 = $0.00 + assets:bank:wf:pchecking $-2.00 + + 2019/10/22 (6L8L1662YP1334033) Noble Benefactor Joyful Systems ; itemid:, fromemail:noble@bene.fac.tor, toemail:simon@joyful.com, time:05:07:06, type:Subscription Payment, status:Completed + assets:online:paypal $9.41 = $9.41 + revenues:foss donations:darcshub $-10.00 ; business: + expenses:banking:paypal $0.59 ; business: + CSV RULES - The following seven kinds of rule can appear in the rules file, in any - order. Blank lines and lines beginning with # or ; are ignored. + The following kinds of rule can appear in the rules file, in any order. + Blank lines and lines beginning with # or ; are ignored. skip - skipN + skip N - Skip this number of CSV records at the beginning. You'll need this - whenever your CSV data contains header lines. Eg: + The word "skip" followed by a number (or no number, meaning 1) tells + hledger to ignore this many non-empty lines preceding the CSV data. + (Empty/blank lines are skipped automatically.) You'll need this when- + ever your CSV data contains header lines. - # ignore the first CSV line - skip 1 + It also has a second purpose: it can be used inside if blocks to ignore + certain CSV records (described below). - date-format - date-formatDATEFMT + fields + fields FIELDNAME1, FIELDNAME2, ... - When your CSV date fields are not formatted like YYYY/MM/DD (or YYYY- - MM-DD or YYYY.MM.DD), you'll need to specify the format. DATEFMT is a - strptime-like date parsing pattern, which must parse the date field - values completely. Examples: + A fields list (the word "fields" followed by comma-separated field + names) is the quick way to assign CSV field values to hledger fields. + It does two things: - # for dates like "11/06/2013": - date-format %m/%d/%Y + 1. it names the CSV fields. This is optional, but can be convenient + later for interpolating them. - # for dates like "6/11/2013" (note the - to make leading zeros optional): - date-format %-d/%-m/%Y + 2. when you use a standard hledger field name, it assigns the CSV value + to that part of the hledger transaction. - # for dates like "2013-Nov-06": - date-format %Y-%h-%d + Here's an example that says "use the 1st, 2nd and 4th fields as the + transaction's date, description and amount; name the last two fields + for later reference; and ignore the others": - # for dates like "11/6/2013 11:32 PM": - date-format %-m/%-d/%Y %l:%M %p + fields date, description, , amount, , , somefield, anotherfield - field list - fieldsFIELDNAME1, FIELDNAME2... + Field names may not contain whitespace. Fields you don't care about + can be left unnamed. Currently there must be least two items (there + must be at least one comma). - This (a) names the CSV fields, in order (names may not contain white- - space; uninteresting names may be left blank), and (b) assigns them to - journal entry fields if you use any of these standard field names: - date, date2, status, code, description, comment, account1, account2, - amount, amount-in, amount-out, currency, balance, balance1, balance2. - Eg: + Here are the standard hledger field/pseudo-field names. For more about + the transaction parts they refer to, see the manual for hledger's jour- + nal format. - # use the 1st, 2nd and 4th CSV fields as the entry's date, description and amount, - # and give the 7th and 8th fields meaningful names for later reference: - # - # CSV field: - # 1 2 3 4 5 6 7 8 - # entry field: - fields date, description, , amount, , , somefield, anotherfield + Transaction field names + date, date2, status, code, description, comment can be used to form the + transaction's first line. + Posting field names + accountN, where N is 1 to 9, generates a posting, with that account + name. Most often there are two postings, so you'll want to set ac- + count1 and account2. If a posting's account name is left unset but its + amount is set, a default account name will be chosen (like expenses:un- + known or income:unknown). + + amountN sets posting N's amount. Or, amount with no N sets posting + 1's. If the CSV has debits and credits in separate fields, use + amountN-in and amountN-out instead. Or amount-in and amount-out with + no N for posting 1. + + For convenience and backwards compatibility, if you set the amount of + posting 1 only, a second posting with the negative amount will be gen- + erated automatically. (Unless the account name is parenthesised indi- + cating an unbalanced posting.) + + If the CSV has the currency symbol in a separate field, you can use + currencyN to prepend it to posting N's amount. currency with no N af- + fects ALL postings. + + balanceN sets a balance assertion amount (or if the posting amount is + left empty, a balance assignment). + + Finally, commentN sets a comment on the Nth posting. Comments can also + contain tags, as usual. + + See TIPS below for more about setting amounts and currency. + field assignment - ENTRYFIELDNAME FIELDVALUE + HLEDGERFIELDNAME FIELDVALUE - This sets a journal entry field (one of the standard names above) to - the given text value, which can include CSV field values interpolated - by name (%CSVFIELDNAME) or 1-based position (%N). Eg: + Instead of or in addition to a fields list, you can use a "field as- + signment" rule to set the value of a single hledger field, by writing + its name (any of the standard hledger field names above) followed by a + text value. The value may contain interpolated CSV fields, referenced + by their 1-based position in the CSV record (%N), or by the name they + were given in the fields list (%CSVFIELDNAME). Some examples: - # set the amount to the 4th CSV field with "USD " prepended - amount USD %4 + # set the amount to the 4th CSV field, with " USD" appended + amount %4 USD - # combine three fields to make a comment (containing two tags) + # combine three fields to make a comment, containing note: and date: tags comment note: %somefield - %anotherfield, date: %1 - Field assignments can be used instead of or in addition to a field - list. + Interpolation strips outer whitespace (so a CSV value like " 1 " be- + comes 1 when interpolated) (#1051). See TIPS below for more about ref- + erencing other fields. - Note, interpolation strips any outer whitespace, so a CSV value like " - 1 " becomes 1 when interpolated (#1051). + if + if PATTERN + RULE - conditional block - if PATTERN - FIELDASSIGNMENTS... + if + PATTERN + PATTERN + PATTERN + RULE + RULE - if - PATTERN - PATTERN... - FIELDASSIGNMENTS... + Conditional blocks ("if blocks") are a block of rules that are applied + only to CSV records which match certain patterns. They are often used + for customising account names based on transaction descriptions. - This applies one or more field assignments, only to those CSV records - matched by one of the PATTERNs. The patterns are case-insensitive reg- - ular expressions which match anywhere within the whole CSV record (it's - not yet possible to match within a specific field). When there are - multiple patterns they can be written on separate lines, unindented. - The field assignments are on separate lines indented by at least one - space. Examples: + A single pattern can be written on the same line as the "if"; or multi- + ple patterns can be written on the following lines, non-indented. Mul- + tiple patterns are OR'd (any one of them can match). Patterns are + case-insensitive regular expressions which try to match anywhere within + the whole CSV record (POSIX extended regular expressions with some ad- + ditions, see https://hledger.org/hledger.html#regular-expressions). + Note the CSV record they see is close to, but not identical to, the one + in the CSV file; enclosing double quotes will be removed, and the sepa- + rator character is always comma. + It's not yet easy to match within a specific field. If the data does + not contain commas, you can hack it with a regular expression like: + + # match "foo" in the fourth field + if ^([^,]*,){3}foo + + After the patterns there should be one or more rules to apply, all in- + dented by at least one space. Three kinds of rule are allowed in con- + ditional blocks: + + o field assignments (to set a hledger field) + + o skip (to skip the matched CSV record) + + o end (to skip all remaining CSV records). + + Examples: + # if the CSV record contains "groceries", set account2 to "expenses:groceries" if groceries account2 expenses:groceries @@ -166,95 +478,267 @@ account2 expenses:business:banking comment XXX deductible ? check it - include - includeRULESFILE + end + This rule can be used inside if blocks (only), to make hledger stop + reading this CSV file and move on to the next input file, or to command + execution. Eg: - Include another rules file at this point. RULESFILE is either an abso- - lute file path or a path relative to the current file's directory. Eg: + # ignore everything following the first empty record + if ,,,, + end - # rules reused with several CSV files - include common.rules + date-format + date-format DATEFMT + This is a helper for the date (and date2) fields. If your CSV dates + are not formatted like YYYY-MM-DD, YYYY/MM/DD or YYYY.MM.DD, you'll + need to add a date-format rule describing them with a strptime date + parsing pattern, which must parse the CSV date value completely. Some + examples: + + # MM/DD/YY + date-format %m/%d/%y + + # D/M/YYYY + # The - makes leading zeros optional. + date-format %-d/%-m/%Y + + # YYYY-Mmm-DD + date-format %Y-%h-%d + + # M/D/YYYY HH:MM AM some other junk + # Note the time and junk must be fully parsed, though only the date is used. + date-format %-m/%-d/%Y %l:%M %p some other junk + + For the supported strptime syntax, see: + https://hackage.haskell.org/package/time/docs/Data-Time-For- + mat.html#v:formatTime + newest-first - newest-first + hledger always sorts the generated transactions by date. Transactions + on the same date should appear in the same order as their CSV records, + as hledger can usually auto-detect whether the CSV's normal order is + oldest first or newest first. But if all of the following are true: - Consider adding this rule if all of the following are true: you might - be processing just one day of data, your CSV records are in reverse - chronological order (newest first), and you care about preserving the - order of same-day transactions. It usually isn't needed, because - hledger autodetects the CSV order, but when all CSV records have the - same date it will assume they are oldest first. + o the CSV might sometimes contain just one day of data (all records + having the same date) -CSV TIPS - CSV ordering - The generated journal entries will be sorted by date. The order of - same-day entries will be preserved (except in the special case where - you might need newest-first, see above). + o the CSV records are normally in reverse chronological order (newest + at the top) - CSV accounts - Each journal entry will have two postings, to account1 and account2 re- - spectively. It's not yet possible to generate entries with more than - two postings. It's conventional and recommended to use account1 for - the account whose CSV we are reading. + o and you care about preserving the order of same-day transactions - CSV amounts - A transaction amount must be set, in one of these ways: + then, you should add the newest-first rule as a hint. Eg: - o with an amount field assignment, which sets the first posting's - amount + # tell hledger explicitly that the CSV is normally newest first + newest-first - o (When the CSV has debit and credit amounts in separate fields:) - with field assignments for the amount-in and amount-out pseudo fields - (both of them). Whichever one has a value will be used, with appropri- - ate sign. If both contain a value, it might not work so well. + include + include RULESFILE - o or implicitly by means of a balance assignment (see below). + This includes the contents of another CSV rules file at this point. + RULESFILE is an absolute file path or a path relative to the current + file's directory. This can be useful for sharing common rules between + several rules files, eg: - There is some special handling for sign in amounts: + # someaccount.csv.rules - o If an amount value is parenthesised, it will be de-parenthesised and - sign-flipped. + ## someaccount-specific rules + fields date,description,amount + account1 assets:someaccount + account2 expenses:misc - o If an amount value begins with a double minus sign, those will cancel - out and be removed. + ## common rules + include categorisation.rules - If the currency/commodity symbol is provided as a separate CSV field, - assign it to the currency pseudo field; the symbol will be prepended to - the amount (TODO: when there is an amount). Or, you can use an amount - field assignment for more control, eg: +TIPS + Valid CSV + hledger accepts CSV conforming to RFC 4180. When CSV values are en- + closed in quotes, note: - fields date,description,currency,amount - amount %amount %currency + o they must be double quotes (not single quotes) - CSV balance assertions/assignments - If the CSV includes a running balance, you can assign that to one of - the pseudo fields balance (or balance1) or balance2. This will gener- - ate a balance assertion (or if the amount is left empty, a balance as- - signment), on the first or second posting, whenever the running balance - field is non-empty. (TODO: #1000) + o spaces outside the quotes are not allowed + Other separator characters + With the --separator 'CHAR' option (experimental), hledger will expect + the separator to be CHAR instead of a comma. Ie it will read other + "Character Separated Values" formats, such as TSV (Tab Separated Val- + ues). Note: on the command line, use a real tab character in quotes, + not Eg: + + $ hledger -f foo.tsv --separator ' ' print + Reading multiple CSV files - You can read multiple CSV files at once using multiple -f arguments on - the command line, and hledger will look for a correspondingly-named - rules file for each. Note if you use the --rules-file option, this one - rules file will be used for all the CSV files being read. + If you use multiple -f options to read multiple CSV files at once, + hledger will look for a correspondingly-named rules file for each CSV + file. But if you use the --rules-file option, that rules file will be + used for all the CSV files. - Valid CSV - hledger follows RFC 4180, with the addition of a customisable separator - character. + Valid transactions + After reading a CSV file, hledger post-processes and validates the gen- + erated journal entries as it would for a journal file - balancing them, + applying balance assignments, and canonicalising amount styles. Any + errors at this stage will be reported in the usual way, displaying the + problem entry. - Some things to note: + There is one exception: balance assertions, if you have generated them, + will not be checked, since normally these will work only when the CSV + data is part of the main journal. If you do need to check balance as- + sertions generated from CSV right away, pipe into another hledger: - When quoting fields, + $ hledger -f file.csv print | hledger -f- print - o you must use double quotes, not single quotes + Deduplicating, importing + When you download a CSV file periodically, eg to get your latest bank + transactions, the new file may overlap with the old one, containing + some of the same records. - o spaces outside the quotes are not allowed. + The import command will (a) detect the new transactions, and (b) append + just those transactions to your main journal. It is idempotent, so you + don't have to remember how many times you ran it or with which version + of the CSV. (It keeps state in a hidden .latest.FILE.csv file.) This + is the easiest way to import CSV data. Eg: + # download the latest CSV files, then run this command. + # Note, no -f flags needed here. + $ hledger import *.csv [--dry] + This method works for most CSV files. (Where records have a stable + chronological order, and new records appear only at the new end.) + A number of other tools and workflows, hledger-specific and otherwise, + exist for converting, deduplicating, classifying and managing CSV data. + See: + + o https://hledger.org -> sidebar -> real world setups + + o https://plaintextaccounting.org -> data import/conversion + + Setting amounts + A posting amount can be set in one of these ways: + + o by assigning (with a fields list or field assigment) to amountN + (posting N's amount) or amount (posting 1's amount) + + o by assigning to amountN-in and amountN-out (or amount-in and amount- + out). For each CSV record, whichever of these has a non-zero value + will be used, with appropriate sign. If both contain a non-zero + value, this may not work. + + o by assigning to balanceN (or balance) instead of the above, setting + the amount indirectly via a balance assignment. If you do this the + default account name may be wrong, so you should set that explicitly. + + There is some special handling for an amount's sign: + + o If an amount value is parenthesised, it will be de-parenthesised and + sign-flipped. + + o If an amount value begins with a double minus sign, those cancel out + and are removed. + + o If an amount value begins with a plus sign, that will be removed + + Setting currency/commodity + If the currency/commodity symbol is included in the CSV's amount + field(s), you don't have to do anything special. + + If the currency is provided as a separate CSV field, you can either: + + o assign that to currency, which adds it to all posting amounts. The + symbol will prepended to the amount quantity (on the left side). If + you write a trailing space after the symbol, there will be a space + between symbol and amount (an exception to the usual whitespace + stripping). + + o or assign it to currencyN which adds it to posting N's amount only. + + o or for more control, construct the amount from symbol and quantity + using field assignment, eg: + + fields date,description,currency,quantity + # add currency symbol on the right: + amount %quantity %currency + + Referencing other fields + In field assignments, you can interpolate only CSV fields, not hledger + fields. In the example below, there's both a CSV field and a hledger + field named amount1, but %amount1 always means the CSV field, not the + hledger field: + + # Name the third CSV field "amount1" + fields date,description,amount1 + + # Set hledger's amount1 to the CSV amount1 field followed by USD + amount1 %amount1 USD + + # Set comment to the CSV amount1 (not the amount1 assigned above) + comment %amount1 + + Here, since there's no CSV amount1 field, %amount1 will produce a lit- + eral "amount1": + + fields date,description,csvamount + amount1 %csvamount USD + # Can't interpolate amount1 here + comment %amount1 + + When there are multiple field assignments to the same hledger field, + only the last one takes effect. Here, comment's value will be be B, or + C if "something" is matched, but never A: + + comment A + comment B + if something + comment C + + How CSV rules are evaluated + Here's how to think of CSV rules being evaluated (if you really need + to). First, + + o include - all includes are inlined, from top to bottom, depth first. + (At each include point the file is inlined and scanned for further + includes, recursively, before proceeding.) + + Then "global" rules are evaluated, top to bottom. If a rule is re- + peated, the last one wins: + + o skip (at top level) + + o date-format + + o newest-first + + o fields - names the CSV fields, optionally sets up initial assignments + to hledger fields + + Then for each CSV record in turn: + + o test all if blocks. If any of them contain a end rule, skip all re- + maining CSV records. Otherwise if any of them contain a skip rule, + skip that many CSV records. If there are multiple matched skip + rules, the first one wins. + + o collect all field assignments at top level and in matched if blocks. + When there are multiple assignments for a field, keep only the last + one. + + o compute a value for each hledger field - either the one that was as- + signed to it (and interpolate the %CSVFIELDNAME references), or a de- + fault + + o generate a synthetic hledger transaction from these values. + + This is all part of the CSV reader, one of several readers hledger can + use to parse input files. When all files have been read successfully, + the transactions are passed as input to whichever hledger command the + user specified. + + + REPORTING BUGS - Report bugs at http://bugs.hledger.org (or on the #hledger IRC channel + Report bugs at http://bugs.hledger.org (or on the #hledger IRC channel or hledger mail list) @@ -263,12 +747,12 @@ COPYRIGHT - Copyright (C) 2007-2016 Simon Michael. + Copyright (C) 2007-2019 Simon Michael. Released under GNU GPL v3 or later. SEE ALSO - hledger(1), hledger-ui(1), hledger-web(1), hledger-api(1), + hledger(1), hledger-ui(1), hledger-web(1), hledger-api(1), hledger_csv(5), hledger_journal(5), hledger_timeclock(5), hledger_time- dot(5), ledger(1) @@ -276,4 +760,4 @@ -hledger 1.15.2 September 2019 hledger_csv(5) +hledger 1.16 December 2019 hledger_csv(5) diff --git a/hledger_journal.5 b/hledger_journal.5 --- a/hledger_journal.5 +++ b/hledger_journal.5 @@ -1,6 +1,6 @@ .\"t -.TH "hledger_journal" "5" "September 2019" "hledger 1.15.2" "hledger User Manuals" +.TH "hledger_journal" "5" "December 2019" "hledger 1.16" "hledger User Manuals" @@ -337,119 +337,134 @@ .SS Amounts .PP After the account name, there is usually an amount. -Important: between account name and amount, there must be \f[B]two or -more spaces\f[R]. +(Important: between account name and amount, there must be \f[B]two or +more spaces\f[R].) .PP -Amounts consist of a number and (usually) a currency symbol or commodity -name. -Some examples: +hledger\[aq]s amount format is flexible, supporting several +international formats. +Here are some examples. +Amounts have a number (the \[dq]quantity\[dq]): +.IP +.nf +\f[C] +1 +\f[R] +.fi .PP -\f[C]2.00001\f[R] -.PD 0 -.P -.PD -\f[C]$1\f[R] -.PD 0 -.P -.PD -\f[C]4000 AAPL\f[R] -.PD 0 -.P -.PD -\f[C]3 \[dq]green apples\[dq]\f[R] -.PD 0 -.P -.PD -\f[C]-$1,000,000.00\f[R] -.PD 0 -.P -.PD -\f[C]INR 9,99,99,999.00\f[R] -.PD 0 -.P -.PD -\f[C]EUR -2.000.000,00\f[R] -.PD 0 -.P -.PD -\f[C]1 999 999.9455\f[R] -.PD 0 -.P -.PD -\f[C]EUR 1E3\f[R] -.PD 0 -.P -.PD -\f[C]1000E-6s\f[R] +\&..and usually a currency or commodity name (the \[dq]commodity\[dq]). +This is a symbol, word, or phrase, to the left or right of the quantity, +with or without a separating space: +.IP +.nf +\f[C] +$1 +4000 AAPL +\f[R] +.fi .PP -As you can see, the amount format is somewhat flexible: -.IP \[bu] 2 -amounts are a number (the \[dq]quantity\[dq]) and optionally a currency -symbol/commodity name (the \[dq]commodity\[dq]). -.IP \[bu] 2 -the commodity is a symbol, word, or phrase, on the left or right, with -or without a separating space. -If the commodity contains numbers, spaces or non-word punctuation it -must be enclosed in double quotes. -.IP \[bu] 2 -negative amounts with a commodity on the left can have the minus sign -before or after it -.IP \[bu] 2 -digit groups (thousands, or any other grouping) can be separated by -space or comma or period and should be used as separator between all -groups -.IP \[bu] 2 -decimal part can be separated by comma or period and should be different -from digit groups separator -.IP \[bu] 2 -scientific E-notation is allowed. -Be careful not to use a digit group separator character in scientific -notation, as it\[aq]s not supported and it might get mistaken for a -decimal point. -(Declaring the digit group separator character explicitly with a -commodity directive will prevent this.) +If the commodity name contains spaces, numbers, or punctuation, it must +be enclosed in double quotes: +.IP +.nf +\f[C] +3 \[dq]no. 42 green apples\[dq] +\f[R] +.fi .PP -You can use any of these variations when recording data. -However, there is some ambiguous way of representing numbers like -\f[C]$1.000\f[R] and \f[C]$1,000\f[R] both may mean either one thousand -or one dollar. -By default hledger will assume that this is sole delimiter is used only -for decimals. -On the other hand commodity format declared prior to that line will help -to resolve that ambiguity differently: +Amounts can be negative. +The minus sign can be written before or after a left-side commodity +symbol: .IP .nf \f[C] +-$1 +$-1 +\f[R] +.fi +.PP +Scientific E notation is allowed: +.IP +.nf +\f[C] +1E-6 +EUR 1E3 +\f[R] +.fi +.PP +A decimal mark (decimal point) can be written with a period or a comma: +.IP +.nf +\f[C] +1.23 +1,23456780000009 +\f[R] +.fi +.SS Digit group marks +.PP +In the integer part of the quantity (left of the decimal mark), groups +of digits can optionally be separated by a \[dq]digit group mark\[dq] - +a space, comma, or period (different from the decimal mark): +.IP +.nf +\f[C] + $1,000,000.00 + EUR 2.000.000,00 +INR 9,99,99,999.00 + 1 000 000.9455 +\f[R] +.fi +.PP +Note, a number containing a single group mark and no decimal mark is +ambiguous. +Are these group marks or decimal marks ? +.IP +.nf +\f[C] +1,000 +1.000 +\f[R] +.fi +.PP +hledger will treat them both as decimal marks by default (cf #793). +If you use digit group marks, to prevent confusion and undetected typos +we recommend you write commodity directives at the top of the file to +explicitly declare the decimal mark (and optionally a digit group mark). +Note, these formats (\[dq]amount styles\[dq]) are specific to each +commodity, so if your data uses multiple formats, hledger can handle it: +.IP +.nf +\f[C] commodity $1,000.00 - -2017/12/25 New life of Scrooge - expenses:gifts $1,000 - assets +commodity EUR 1.000,00 +commodity INR 9,99,99,999.00 +commodity 1 000 000.9455 \f[R] .fi +.SS Amount display format .PP -Though journal may contain mixed styles to represent amount, when -hledger displays amounts, it will choose a consistent format for each -commodity. -(Except for price amounts, which are always formatted as written). +For each commodity, hledger chooses a consistent format to use when +displaying amounts. +(Except price amounts, which are always displayed as written). The display format is chosen as follows: .IP \[bu] 2 -if there is a commodity directive specifying the format, that is used +If there is a commodity directive for the commodity, that format is used +(see examples above). .IP \[bu] 2 -otherwise the format is inferred from the first posting amount in that -commodity in the journal, and the precision (number of decimal places) -will be the maximum from all posting amounts in that commmodity +Otherwise the format of the first posting amount in that commodity seen +in the journal is used. +But the number of decimal places (\[dq]precision\[dq]) will be the +maximum from all posting amounts in that commmodity. .IP \[bu] 2 -or if there are no such amounts in the journal, a default format is used +Or if there are no such amounts in the journal, a default format is used (like \f[C]$1000.00\f[R]). .PP -Price amounts and amounts in \f[C]D\f[R] directives usually don\[aq]t -affect amount format inference, but in some situations they can do so +Price amounts, and amounts in \f[C]D\f[R] directives don\[aq]t affect +the amount display format directly, but occasionally they can do so indirectly. (Eg when D\[aq]s default commodity is applied to a commodity-less amount, or when an amountless posting is balanced using a price\[aq]s -commodity, or when -V is used.) If you find this causing problems, set -the desired format with a commodity directive. +commodity, or when -V is used.) If you find this causing problems, use a +commodity directive to set the display format. .SS Virtual Postings .PP When you parenthesise the account name in a posting, we call that a @@ -1118,9 +1133,25 @@ .fi .SS Declaring commodities .PP -The \f[C]commodity\f[R] directive declares commodities which may be used -in the journal, and their display format. +The \f[C]commodity\f[R] directive has several functions: +.IP "1." 3 +It declares commodities which may be used in the journal. +This is currently not enforced, but can serve as documentation. +.IP "2." 3 +It declares what decimal mark character to expect when parsing input - +useful to disambiguate international number formats in your data. +(Without this, hledger will parse both \f[C]1,000\f[R] and +\f[C]1.000\f[R] as 1). +.IP "3." 3 +It declares the amount display format to use in output - decimal and +digit group marks, number of decimal places, symbol placement etc. .PP +You are likely to run into one of the problems solved by commodity +directives, sooner or later, so it\[aq]s a good idea to just always use +them to declare your commodities. +.PP +A commodity directive is just the word \f[C]commodity\f[R] followed by +an amount. It may be written on a single line, like this: .IP .nf @@ -1135,8 +1166,8 @@ .fi .PP or on multiple lines, using the \[dq]format\[dq] subdirective. -In this case the commodity symbol appears twice and should be the same -in both places: +(In this case the commodity symbol appears twice and should be the same +in both places.): .IP .nf \f[C] @@ -1147,24 +1178,14 @@ ; thousands, lakhs and crores comma-separated, ; period as decimal point, and two decimal places. commodity INR - format INR 9,99,99,999.00 + format INR 1,00,00,000.00 \f[R] .fi .PP -Declaring commodites may be useful as documentation, but currently we do -not enforce that only declared commodities may be used. -This directive is mainly useful for customising the preferred display -format for a commodity. -.PP -Normally the display format is inferred from journal entries, but this -can be unpredictable; declaring it with a commodity directive overrides -this and removes ambiguity. -Towards this end, amounts in commodity directives must always be written -with a decimal point (a period or comma, followed by 0 or more decimal -digits). -.PP -Commodity directives do not affect how amounts are parsed; the parser -will read multiple formats. +The quantity of the amount does not matter; only the format is +significant. +The number must include a decimal mark: either a period or a comma, +followed by 0 or more decimal digits. .SS Default commodity .PP The \f[C]D\f[R] directive sets a default commodity (and display format), @@ -1579,12 +1600,46 @@ .SS Periodic transactions .PP Periodic transaction rules describe transactions that recur. -They allow you to generate future transactions for forecasting, without -having to write them out explicitly in the journal (with -\f[C]--forecast\f[R]). -Secondly, they also can be used to define budget goals (with -\f[C]--budget\f[R]). +They allow hledger to generate temporary future transactions to help +with forecasting, so you don\[aq]t have to write out each one in the +journal, and it\[aq]s easy to try out different forecasts. +Secondly, they are also used to define the budgets shown in budget +reports. .PP +Periodic transactions can be a little tricky, so before you use them, +read this whole section - or at least these tips: +.IP "1." 3 +Two spaces accidentally added or omitted will cause you trouble - read +about this below. +.IP "2." 3 +For troubleshooting, show the generated transactions with +\f[C]hledger print --forecast tag:generated\f[R] or +\f[C]hledger register --forecast tag:generated\f[R]. +.IP "3." 3 +Forecasted transactions will begin only after the last non-forecasted +transaction\[aq]s date. +.IP "4." 3 +Forecasted transactions will end 6 months from today, by default. +See below for the exact start/end rules. +.IP "5." 3 +period expressions can be tricky. +Their documentation needs improvement, but is worth studying. +.IP "6." 3 +Some period expressions with a repeating interval must begin on a +natural boundary of that interval. +Eg in \f[C]weekly from DATE\f[R], DATE must be a monday. +\f[C]\[ti] weekly from 2019/10/1\f[R] (a tuesday) will give an error. +.IP "7." 3 +Other period expressions with an interval are automatically expanded to +cover a whole number of that interval. +(This is done to improve reports, but it also affects periodic +transactions. +Yes, it\[aq]s a bit inconsistent with the above.) Eg: +\f[C]\[ti] every 10th day of month from 2020/01\f[R], which is +equivalent to \f[C]\[ti] every 10th day of month from 2020/01/01\f[R], +will be adjusted to start on 2019/12/10. +.SS Periodic rule syntax +.PP A periodic transaction rule looks like a normal journal entry, with the date replaced by a tilde (\f[C]\[ti]\f[R]) followed by a period expression (mnemonic: \f[C]\[ti]\f[R] looks like a recurring sine @@ -1607,7 +1662,7 @@ expression can work (useful or not). They will be relative to today\[aq]s date, unless a Y default year directive is in effect, in which case they will be relative to Y/1/1. -.SS Two spaces after the period expression +.SS Two spaces between period expression and description! .PP If the period expression is followed by a transaction description, these must be separated by \f[B]two or more spaces\f[R]. @@ -1625,6 +1680,14 @@ income:acme inc \f[R] .fi +.PP +So, +.IP \[bu] 2 +Do write two spaces between your period expression and your transaction +description, if any. +.IP \[bu] 2 +Don\[aq]t accidentally write two spaces in the middle of your period +expression. .SS Forecasting with periodic transactions .PP With the \f[C]--forecast\f[R] flag, each periodic transaction rule @@ -1686,8 +1749,8 @@ (and also, a goal of depositing $2000 into checking) every month. Goals and actual performance can then be compared in budget reports. .PP -For more details, see: balance: Budget report and Cookbook: Budgeting -and Forecasting. +For more details, see: balance: Budget report and Budgeting and +Forecasting. .PP .SS Auto postings / transaction modifiers .PP @@ -1829,7 +1892,7 @@ .SH COPYRIGHT -Copyright (C) 2007-2016 Simon Michael. +Copyright (C) 2007-2019 Simon Michael. .br Released under GNU GPL v3 or later. diff --git a/hledger_journal.info b/hledger_journal.info --- a/hledger_journal.info +++ b/hledger_journal.info @@ -4,8 +4,8 @@  File: hledger_journal.info, Node: Top, Next: FILE FORMAT, Up: (dir) -hledger_journal(5) hledger 1.15.2 -********************************* +hledger_journal(5) hledger 1.16 +******************************* hledger's usual data source is a plain text file containing journal entries in hledger journal format. This file represents a standard @@ -337,77 +337,109 @@ 1.7 Amounts =========== -After the account name, there is usually an amount. Important: between -account name and amount, there must be *two or more spaces*. +After the account name, there is usually an amount. (Important: between +account name and amount, there must be *two or more spaces*.) - Amounts consist of a number and (usually) a currency symbol or -commodity name. Some examples: + hledger's amount format is flexible, supporting several international +formats. Here are some examples. Amounts have a number (the +"quantity"): - '2.00001' -'$1' -'4000 AAPL' -'3 "green apples"' -'-$1,000,000.00' -'INR 9,99,99,999.00' -'EUR -2.000.000,00' -'1 999 999.9455' -'EUR 1E3' -'1000E-6s' +1 - As you can see, the amount format is somewhat flexible: + ..and usually a currency or commodity name (the "commodity"). This +is a symbol, word, or phrase, to the left or right of the quantity, with +or without a separating space: - * amounts are a number (the "quantity") and optionally a currency - symbol/commodity name (the "commodity"). - * the commodity is a symbol, word, or phrase, on the left or right, - with or without a separating space. If the commodity contains - numbers, spaces or non-word punctuation it must be enclosed in - double quotes. - * negative amounts with a commodity on the left can have the minus - sign before or after it - * digit groups (thousands, or any other grouping) can be separated by - space or comma or period and should be used as separator between - all groups - * decimal part can be separated by comma or period and should be - different from digit groups separator - * scientific E-notation is allowed. Be careful not to use a digit - group separator character in scientific notation, as it's not - supported and it might get mistaken for a decimal point. - (Declaring the digit group separator character explicitly with a - commodity directive will prevent this.) +$1 +4000 AAPL - You can use any of these variations when recording data. However, -there is some ambiguous way of representing numbers like '$1.000' and -'$1,000' both may mean either one thousand or one dollar. By default -hledger will assume that this is sole delimiter is used only for -decimals. On the other hand commodity format declared prior to that -line will help to resolve that ambiguity differently: + If the commodity name contains spaces, numbers, or punctuation, it +must be enclosed in double quotes: +3 "no. 42 green apples" + + Amounts can be negative. The minus sign can be written before or +after a left-side commodity symbol: + +-$1 +$-1 + + Scientific E notation is allowed: + +1E-6 +EUR 1E3 + + A decimal mark (decimal point) can be written with a period or a +comma: + +1.23 +1,23456780000009 + +* Menu: + +* Digit group marks:: +* Amount display format:: + + +File: hledger_journal.info, Node: Digit group marks, Next: Amount display format, Up: Amounts + +1.7.1 Digit group marks +----------------------- + +In the integer part of the quantity (left of the decimal mark), groups +of digits can optionally be separated by a "digit group mark" - a space, +comma, or period (different from the decimal mark): + + $1,000,000.00 + EUR 2.000.000,00 +INR 9,99,99,999.00 + 1 000 000.9455 + + Note, a number containing a single group mark and no decimal mark is +ambiguous. Are these group marks or decimal marks ? + +1,000 +1.000 + + hledger will treat them both as decimal marks by default (cf #793). +If you use digit group marks, to prevent confusion and undetected typos +we recommend you write commodity directives at the top of the file to +explicitly declare the decimal mark (and optionally a digit group mark). +Note, these formats ("amount styles") are specific to each commodity, so +if your data uses multiple formats, hledger can handle it: + commodity $1,000.00 +commodity EUR 1.000,00 +commodity INR 9,99,99,999.00 +commodity 1 000 000.9455 -2017/12/25 New life of Scrooge - expenses:gifts $1,000 - assets + +File: hledger_journal.info, Node: Amount display format, Prev: Digit group marks, Up: Amounts - Though journal may contain mixed styles to represent amount, when -hledger displays amounts, it will choose a consistent format for each -commodity. (Except for price amounts, which are always formatted as -written). The display format is chosen as follows: +1.7.2 Amount display format +--------------------------- - * if there is a commodity directive specifying the format, that is - used - * otherwise the format is inferred from the first posting amount in - that commodity in the journal, and the precision (number of decimal - places) will be the maximum from all posting amounts in that - commmodity - * or if there are no such amounts in the journal, a default format is +For each commodity, hledger chooses a consistent format to use when +displaying amounts. (Except price amounts, which are always displayed +as written). The display format is chosen as follows: + + * If there is a commodity directive for the commodity, that format is + used (see examples above). + + * Otherwise the format of the first posting amount in that commodity + seen in the journal is used. But the number of decimal places + ("precision") will be the maximum from all posting amounts in that + commmodity. + + * Or if there are no such amounts in the journal, a default format is used (like '$1000.00'). - Price amounts and amounts in 'D' directives usually don't affect -amount format inference, but in some situations they can do so -indirectly. (Eg when D's default commodity is applied to a -commodity-less amount, or when an amountless posting is balanced using a -price's commodity, or when -V is used.) If you find this causing -problems, set the desired format with a commodity directive. + Price amounts, and amounts in 'D' directives don't affect the amount +display format directly, but occasionally they can do so indirectly. +(Eg when D's default commodity is applied to a commodity-less amount, or +when an amountless posting is balanced using a price's commodity, or +when -V is used.) If you find this causing problems, use a commodity +directive to set the display format.  File: hledger_journal.info, Node: Virtual Postings, Next: Balance Assertions, Prev: Amounts, Up: FILE FORMAT @@ -991,11 +1023,27 @@ 1.14.4 Declaring commodities ---------------------------- -The 'commodity' directive declares commodities which may be used in the -journal, and their display format. +The 'commodity' directive has several functions: - It may be written on a single line, like this: + 1. It declares commodities which may be used in the journal. This is + currently not enforced, but can serve as documentation. + 2. It declares what decimal mark character to expect when parsing + input - useful to disambiguate international number formats in your + data. (Without this, hledger will parse both '1,000' and '1.000' + as 1). + + 3. It declares the amount display format to use in output - decimal + and digit group marks, number of decimal places, symbol placement + etc. + + You are likely to run into one of the problems solved by commodity +directives, sooner or later, so it's a good idea to just always use them +to declare your commodities. + + A commodity directive is just the word 'commodity' followed by an +amount. It may be written on a single line, like this: + ; commodity EXAMPLEAMOUNT ; display AAAA amounts with the symbol on the right, space-separated, @@ -1003,9 +1051,9 @@ ; separating thousands with comma. commodity 1,000.0000 AAAA - or on multiple lines, using the "format" subdirective. In this case + or on multiple lines, using the "format" subdirective. (In this case the commodity symbol appears twice and should be the same in both -places: +places.): ; commodity SYMBOL ; format EXAMPLEAMOUNT @@ -1014,21 +1062,11 @@ ; thousands, lakhs and crores comma-separated, ; period as decimal point, and two decimal places. commodity INR - format INR 9,99,99,999.00 - - Declaring commodites may be useful as documentation, but currently we -do not enforce that only declared commodities may be used. This -directive is mainly useful for customising the preferred display format -for a commodity. - - Normally the display format is inferred from journal entries, but -this can be unpredictable; declaring it with a commodity directive -overrides this and removes ambiguity. Towards this end, amounts in -commodity directives must always be written with a decimal point (a -period or comma, followed by 0 or more decimal digits). + format INR 1,00,00,000.00 - Commodity directives do not affect how amounts are parsed; the parser -will read multiple formats. + The quantity of the amount does not matter; only the format is +significant. The number must include a decimal mark: either a period or +a comma, followed by 0 or more decimal digits.  File: hledger_journal.info, Node: Default commodity, Next: Market prices, Prev: Declaring commodities, Up: Directives @@ -1412,12 +1450,52 @@ ========================== Periodic transaction rules describe transactions that recur. They allow -you to generate future transactions for forecasting, without having to -write them out explicitly in the journal (with '--forecast'). Secondly, -they also can be used to define budget goals (with '--budget'). +hledger to generate temporary future transactions to help with +forecasting, so you don't have to write out each one in the journal, and +it's easy to try out different forecasts. Secondly, they are also used +to define the budgets shown in budget reports. - A periodic transaction rule looks like a normal journal entry, with -the date replaced by a tilde ('~') followed by a period expression + Periodic transactions can be a little tricky, so before you use them, +read this whole section - or at least these tips: + + 1. Two spaces accidentally added or omitted will cause you trouble - + read about this below. + 2. For troubleshooting, show the generated transactions with 'hledger + print --forecast tag:generated' or 'hledger register --forecast + tag:generated'. + 3. Forecasted transactions will begin only after the last + non-forecasted transaction's date. + 4. Forecasted transactions will end 6 months from today, by default. + See below for the exact start/end rules. + 5. period expressions can be tricky. Their documentation needs + improvement, but is worth studying. + 6. Some period expressions with a repeating interval must begin on a + natural boundary of that interval. Eg in 'weekly from DATE', DATE + must be a monday. '~ weekly from 2019/10/1' (a tuesday) will give + an error. + 7. Other period expressions with an interval are automatically + expanded to cover a whole number of that interval. (This is done + to improve reports, but it also affects periodic transactions. + Yes, it's a bit inconsistent with the above.) Eg: '~ every 10th + day of month from 2020/01', which is equivalent to '~ every 10th + day of month from 2020/01/01', will be adjusted to start on + 2019/12/10. + +* Menu: + +* Periodic rule syntax:: +* Two spaces between period expression and description!:: +* Forecasting with periodic transactions:: +* Budgeting with periodic transactions:: + + +File: hledger_journal.info, Node: Periodic rule syntax, Next: Two spaces between period expression and description!, Up: Periodic transactions + +1.15.1 Periodic rule syntax +--------------------------- + +A periodic transaction rule looks like a normal journal entry, with the +date replaced by a tilde ('~') followed by a period expression (mnemonic: '~' looks like a recurring sine wave.): ~ monthly @@ -1433,17 +1511,11 @@ date, unless a Y default year directive is in effect, in which case they will be relative to Y/1/1. -* Menu: - -* Two spaces after the period expression:: -* Forecasting with periodic transactions:: -* Budgeting with periodic transactions:: -  -File: hledger_journal.info, Node: Two spaces after the period expression, Next: Forecasting with periodic transactions, Up: Periodic transactions +File: hledger_journal.info, Node: Two spaces between period expression and description!, Next: Forecasting with periodic transactions, Prev: Periodic rule syntax, Up: Periodic transactions -1.15.1 Two spaces after the period expression ---------------------------------------------- +1.15.2 Two spaces between period expression and description! +------------------------------------------------------------ If the period expression is followed by a transaction description, these must be separated by *two or more spaces*. This helps hledger know @@ -1457,10 +1529,17 @@ assets:bank:checking $1500 income:acme inc + So, + + * Do write two spaces between your period expression and your + transaction description, if any. + * Don't accidentally write two spaces in the middle of your period + expression. +  -File: hledger_journal.info, Node: Forecasting with periodic transactions, Next: Budgeting with periodic transactions, Prev: Two spaces after the period expression, Up: Periodic transactions +File: hledger_journal.info, Node: Forecasting with periodic transactions, Next: Budgeting with periodic transactions, Prev: Two spaces between period expression and description!, Up: Periodic transactions -1.15.2 Forecasting with periodic transactions +1.15.3 Forecasting with periodic transactions --------------------------------------------- With the '--forecast' flag, each periodic transaction rule generates @@ -1513,7 +1592,7 @@  File: hledger_journal.info, Node: Budgeting with periodic transactions, Prev: Forecasting with periodic transactions, Up: Periodic transactions -1.15.3 Budgeting with periodic transactions +1.15.4 Budgeting with periodic transactions ------------------------------------------- With the '--budget' flag, currently supported by the balance command, @@ -1523,8 +1602,8 @@ checking) every month. Goals and actual performance can then be compared in budget reports. - For more details, see: balance: Budget report and Cookbook: Budgeting -and Forecasting. + For more details, see: balance: Budget report and Budgeting and +Forecasting.  File: hledger_journal.info, Node: Auto postings / transaction modifiers, Prev: Periodic transactions, Up: FILE FORMAT @@ -1668,113 +1747,119 @@  Tag Table: Node: Top76 -Node: FILE FORMAT2356 -Ref: #file-format2480 -Node: Transactions2783 -Ref: #transactions2904 -Node: Postings3588 -Ref: #postings3715 -Node: Dates4710 -Ref: #dates4825 -Node: Simple dates4890 -Ref: #simple-dates5016 -Node: Secondary dates5382 -Ref: #secondary-dates5536 -Node: Posting dates7099 -Ref: #posting-dates7228 -Node: Status8600 -Ref: #status8720 -Node: Description10428 -Ref: #description10566 -Node: Payee and note10886 -Ref: #payee-and-note11000 -Node: Account names11335 -Ref: #account-names11478 -Node: Amounts11965 -Ref: #amounts12101 -Node: Virtual Postings15118 -Ref: #virtual-postings15277 -Node: Balance Assertions16497 -Ref: #balance-assertions16672 -Node: Assertions and ordering17631 -Ref: #assertions-and-ordering17817 -Node: Assertions and included files18517 -Ref: #assertions-and-included-files18758 -Node: Assertions and multiple -f options19091 -Ref: #assertions-and-multiple--f-options19345 -Node: Assertions and commodities19477 -Ref: #assertions-and-commodities19707 -Node: Assertions and prices20863 -Ref: #assertions-and-prices21075 -Node: Assertions and subaccounts21515 -Ref: #assertions-and-subaccounts21742 -Node: Assertions and virtual postings22066 -Ref: #assertions-and-virtual-postings22306 -Node: Assertions and precision22448 -Ref: #assertions-and-precision22639 -Node: Balance Assignments22906 -Ref: #balance-assignments23087 -Node: Balance assignments and prices24252 -Ref: #balance-assignments-and-prices24424 -Node: Transaction prices24648 -Ref: #transaction-prices24817 -Node: Comments27083 -Ref: #comments27217 -Node: Tags28387 -Ref: #tags28505 -Node: Directives29898 -Ref: #directives30041 -Node: Comment blocks35649 -Ref: #comment-blocks35794 -Node: Including other files35970 -Ref: #including-other-files36150 -Node: Default year36558 -Ref: #default-year36727 -Node: Declaring commodities37134 -Ref: #declaring-commodities37317 -Node: Default commodity38746 -Ref: #default-commodity38922 -Node: Market prices39556 -Ref: #market-prices39721 -Node: Declaring accounts40562 -Ref: #declaring-accounts40738 -Node: Account comments41663 -Ref: #account-comments41826 -Node: Account subdirectives42221 -Ref: #account-subdirectives42416 -Node: Account types42729 -Ref: #account-types42913 -Node: Account display order44555 -Ref: #account-display-order44725 -Node: Rewriting accounts45854 -Ref: #rewriting-accounts46039 -Node: Basic aliases46775 -Ref: #basic-aliases46921 -Node: Regex aliases47625 -Ref: #regex-aliases47797 -Node: Combining aliases48515 -Ref: #combining-aliases48693 -Node: end aliases49969 -Ref: #end-aliases50117 -Node: Default parent account50218 -Ref: #default-parent-account50384 -Node: Periodic transactions51268 -Ref: #periodic-transactions51466 -Node: Two spaces after the period expression52592 -Ref: #two-spaces-after-the-period-expression52837 -Node: Forecasting with periodic transactions53322 -Ref: #forecasting-with-periodic-transactions53612 -Node: Budgeting with periodic transactions55638 -Ref: #budgeting-with-periodic-transactions55877 -Node: Auto postings / transaction modifiers56336 -Ref: #auto-postings-transaction-modifiers56547 -Node: Auto postings and dates58776 -Ref: #auto-postings-and-dates59033 -Node: Auto postings and transaction balancing / inferred amounts / balance assertions59208 -Ref: #auto-postings-and-transaction-balancing-inferred-amounts-balance-assertions59583 -Node: Auto posting tags59961 -Ref: #auto-posting-tags60200 -Node: EDITOR SUPPORT60865 -Ref: #editor-support60983 +Node: FILE FORMAT2352 +Ref: #file-format2476 +Node: Transactions2779 +Ref: #transactions2900 +Node: Postings3584 +Ref: #postings3711 +Node: Dates4706 +Ref: #dates4821 +Node: Simple dates4886 +Ref: #simple-dates5012 +Node: Secondary dates5378 +Ref: #secondary-dates5532 +Node: Posting dates7095 +Ref: #posting-dates7224 +Node: Status8596 +Ref: #status8716 +Node: Description10424 +Ref: #description10562 +Node: Payee and note10882 +Ref: #payee-and-note10996 +Node: Account names11331 +Ref: #account-names11474 +Node: Amounts11961 +Ref: #amounts12097 +Node: Digit group marks13030 +Ref: #digit-group-marks13179 +Node: Amount display format14117 +Ref: #amount-display-format14274 +Node: Virtual Postings15299 +Ref: #virtual-postings15458 +Node: Balance Assertions16678 +Ref: #balance-assertions16853 +Node: Assertions and ordering17812 +Ref: #assertions-and-ordering17998 +Node: Assertions and included files18698 +Ref: #assertions-and-included-files18939 +Node: Assertions and multiple -f options19272 +Ref: #assertions-and-multiple--f-options19526 +Node: Assertions and commodities19658 +Ref: #assertions-and-commodities19888 +Node: Assertions and prices21044 +Ref: #assertions-and-prices21256 +Node: Assertions and subaccounts21696 +Ref: #assertions-and-subaccounts21923 +Node: Assertions and virtual postings22247 +Ref: #assertions-and-virtual-postings22487 +Node: Assertions and precision22629 +Ref: #assertions-and-precision22820 +Node: Balance Assignments23087 +Ref: #balance-assignments23268 +Node: Balance assignments and prices24433 +Ref: #balance-assignments-and-prices24605 +Node: Transaction prices24829 +Ref: #transaction-prices24998 +Node: Comments27264 +Ref: #comments27398 +Node: Tags28568 +Ref: #tags28686 +Node: Directives30079 +Ref: #directives30222 +Node: Comment blocks35830 +Ref: #comment-blocks35975 +Node: Including other files36151 +Ref: #including-other-files36331 +Node: Default year36739 +Ref: #default-year36908 +Node: Declaring commodities37315 +Ref: #declaring-commodities37498 +Node: Default commodity39159 +Ref: #default-commodity39335 +Node: Market prices39969 +Ref: #market-prices40134 +Node: Declaring accounts40975 +Ref: #declaring-accounts41151 +Node: Account comments42076 +Ref: #account-comments42239 +Node: Account subdirectives42634 +Ref: #account-subdirectives42829 +Node: Account types43142 +Ref: #account-types43326 +Node: Account display order44968 +Ref: #account-display-order45138 +Node: Rewriting accounts46267 +Ref: #rewriting-accounts46452 +Node: Basic aliases47188 +Ref: #basic-aliases47334 +Node: Regex aliases48038 +Ref: #regex-aliases48210 +Node: Combining aliases48928 +Ref: #combining-aliases49106 +Node: end aliases50382 +Ref: #end-aliases50530 +Node: Default parent account50631 +Ref: #default-parent-account50797 +Node: Periodic transactions51681 +Ref: #periodic-transactions51879 +Node: Periodic rule syntax53751 +Ref: #periodic-rule-syntax53957 +Node: Two spaces between period expression and description!54661 +Ref: #two-spaces-between-period-expression-and-description54980 +Node: Forecasting with periodic transactions55664 +Ref: #forecasting-with-periodic-transactions55969 +Node: Budgeting with periodic transactions57995 +Ref: #budgeting-with-periodic-transactions58234 +Node: Auto postings / transaction modifiers58683 +Ref: #auto-postings-transaction-modifiers58894 +Node: Auto postings and dates61123 +Ref: #auto-postings-and-dates61380 +Node: Auto postings and transaction balancing / inferred amounts / balance assertions61555 +Ref: #auto-postings-and-transaction-balancing-inferred-amounts-balance-assertions61930 +Node: Auto posting tags62308 +Ref: #auto-posting-tags62547 +Node: EDITOR SUPPORT63212 +Ref: #editor-support63330  End Tag Table diff --git a/hledger_journal.txt b/hledger_journal.txt --- a/hledger_journal.txt +++ b/hledger_journal.txt @@ -246,83 +246,93 @@ Account names can be aliased. Amounts - After the account name, there is usually an amount. Important: between - account name and amount, there must be two or more spaces. + After the account name, there is usually an amount. (Important: be- + tween account name and amount, there must be two or more spaces.) - Amounts consist of a number and (usually) a currency symbol or commod- - ity name. Some examples: + hledger's amount format is flexible, supporting several international + formats. Here are some examples. Amounts have a number (the "quan- + tity"): - 2.00001 - $1 - 4000 AAPL - 3 "green apples" - -$1,000,000.00 - INR 9,99,99,999.00 - EUR -2.000.000,00 - 1 999 999.9455 - EUR 1E3 - 1000E-6s + 1 - As you can see, the amount format is somewhat flexible: + ..and usually a currency or commodity name (the "commodity"). This is + a symbol, word, or phrase, to the left or right of the quantity, with + or without a separating space: - o amounts are a number (the "quantity") and optionally a currency sym- - bol/commodity name (the "commodity"). + $1 + 4000 AAPL - o the commodity is a symbol, word, or phrase, on the left or right, - with or without a separating space. If the commodity contains num- - bers, spaces or non-word punctuation it must be enclosed in double - quotes. + If the commodity name contains spaces, numbers, or punctuation, it must + be enclosed in double quotes: - o negative amounts with a commodity on the left can have the minus sign - before or after it + 3 "no. 42 green apples" - o digit groups (thousands, or any other grouping) can be separated by - space or comma or period and should be used as separator between all - groups + Amounts can be negative. The minus sign can be written before or after + a left-side commodity symbol: - o decimal part can be separated by comma or period and should be dif- - ferent from digit groups separator + -$1 + $-1 - o scientific E-notation is allowed. Be careful not to use a digit - group separator character in scientific notation, as it's not sup- - ported and it might get mistaken for a decimal point. (Declaring the - digit group separator character explicitly with a commodity directive - will prevent this.) + Scientific E notation is allowed: - You can use any of these variations when recording data. However, - there is some ambiguous way of representing numbers like $1.000 and - $1,000 both may mean either one thousand or one dollar. By default - hledger will assume that this is sole delimiter is used only for deci- - mals. On the other hand commodity format declared prior to that line - will help to resolve that ambiguity differently: + 1E-6 + EUR 1E3 - commodity $1,000.00 + A decimal mark (decimal point) can be written with a period or a comma: - 2017/12/25 New life of Scrooge - expenses:gifts $1,000 - assets + 1.23 + 1,23456780000009 - Though journal may contain mixed styles to represent amount, when - hledger displays amounts, it will choose a consistent format for each - commodity. (Except for price amounts, which are always formatted as - written). The display format is chosen as follows: + Digit group marks + In the integer part of the quantity (left of the decimal mark), groups + of digits can optionally be separated by a "digit group mark" - a + space, comma, or period (different from the decimal mark): - o if there is a commodity directive specifying the format, that is used + $1,000,000.00 + EUR 2.000.000,00 + INR 9,99,99,999.00 + 1 000 000.9455 - o otherwise the format is inferred from the first posting amount in - that commodity in the journal, and the precision (number of decimal - places) will be the maximum from all posting amounts in that commmod- - ity + Note, a number containing a single group mark and no decimal mark is + ambiguous. Are these group marks or decimal marks ? - o or if there are no such amounts in the journal, a default format is + 1,000 + 1.000 + + hledger will treat them both as decimal marks by default (cf #793). If + you use digit group marks, to prevent confusion and undetected typos we + recommend you write commodity directives at the top of the file to ex- + plicitly declare the decimal mark (and optionally a digit group mark). + Note, these formats ("amount styles") are specific to each commodity, + so if your data uses multiple formats, hledger can handle it: + + commodity $1,000.00 + commodity EUR 1.000,00 + commodity INR 9,99,99,999.00 + commodity 1 000 000.9455 + + Amount display format + For each commodity, hledger chooses a consistent format to use when + displaying amounts. (Except price amounts, which are always displayed + as written). The display format is chosen as follows: + + o If there is a commodity directive for the commodity, that format is + used (see examples above). + + o Otherwise the format of the first posting amount in that commodity + seen in the journal is used. But the number of decimal places ("pre- + cision") will be the maximum from all posting amounts in that comm- + modity. + + o Or if there are no such amounts in the journal, a default format is used (like $1000.00). - Price amounts and amounts in D directives usually don't affect amount - format inference, but in some situations they can do so indirectly. - (Eg when D's default commodity is applied to a commodity-less amount, - or when an amountless posting is balanced using a price's commodity, or - when -V is used.) If you find this causing problems, set the desired - format with a commodity directive. + Price amounts, and amounts in D directives don't affect the amount dis- + play format directly, but occasionally they can do so indirectly. (Eg + when D's default commodity is applied to a commodity-less amount, or + when an amountless posting is balanced using a price's commodity, or + when -V is used.) If you find this causing problems, use a commodity + directive to set the display format. Virtual Postings When you parenthesise the account name in a posting, we call that a @@ -667,11 +677,6 @@ account any document account names, de- all entries in all text clare account types & dis- files, before or play order after - - - - - alias end rewrite account names following in- aliases line/included en- tries until end of @@ -727,6 +732,8 @@ file.) display how to display amounts of a commodity in reports (symbol side style and spacing, digit groups, decimal separator, decimal places) + + directive which entries and (when there are multiple files) which files scope are affected by a directive @@ -781,9 +788,23 @@ assets Declaring commodities - The commodity directive declares commodities which may be used in the - journal, and their display format. + The commodity directive has several functions: + 1. It declares commodities which may be used in the journal. This is + currently not enforced, but can serve as documentation. + + 2. It declares what decimal mark character to expect when parsing input + - useful to disambiguate international number formats in your data. + (Without this, hledger will parse both 1,000 and 1.000 as 1). + + 3. It declares the amount display format to use in output - decimal and + digit group marks, number of decimal places, symbol placement etc. + + You are likely to run into one of the problems solved by commodity di- + rectives, sooner or later, so it's a good idea to just always use them + to declare your commodities. + + A commodity directive is just the word commodity followed by an amount. It may be written on a single line, like this: ; commodity EXAMPLEAMOUNT @@ -793,9 +814,9 @@ ; separating thousands with comma. commodity 1,000.0000 AAAA - or on multiple lines, using the "format" subdirective. In this case + or on multiple lines, using the "format" subdirective. (In this case the commodity symbol appears twice and should be the same in both - places: + places.): ; commodity SYMBOL ; format EXAMPLEAMOUNT @@ -804,21 +825,11 @@ ; thousands, lakhs and crores comma-separated, ; period as decimal point, and two decimal places. commodity INR - format INR 9,99,99,999.00 - - Declaring commodites may be useful as documentation, but currently we - do not enforce that only declared commodities may be used. This direc- - tive is mainly useful for customising the preferred display format for - a commodity. - - Normally the display format is inferred from journal entries, but this - can be unpredictable; declaring it with a commodity directive overrides - this and removes ambiguity. Towards this end, amounts in commodity di- - rectives must always be written with a decimal point (a period or - comma, followed by 0 or more decimal digits). + format INR 1,00,00,000.00 - Commodity directives do not affect how amounts are parsed; the parser - will read multiple formats. + The quantity of the amount does not matter; only the format is signifi- + cant. The number must include a decimal mark: either a period or a + comma, followed by 0 or more decimal digits. Default commodity The D directive sets a default commodity (and display format), to be @@ -1129,10 +1140,43 @@ Periodic transactions Periodic transaction rules describe transactions that recur. They al- - low you to generate future transactions for forecasting, without having - to write them out explicitly in the journal (with --forecast). Sec- - ondly, they also can be used to define budget goals (with --budget). + low hledger to generate temporary future transactions to help with + forecasting, so you don't have to write out each one in the journal, + and it's easy to try out different forecasts. Secondly, they are also + used to define the budgets shown in budget reports. + Periodic transactions can be a little tricky, so before you use them, + read this whole section - or at least these tips: + + 1. Two spaces accidentally added or omitted will cause you trouble - + read about this below. + + 2. For troubleshooting, show the generated transactions with hledger + print --forecast tag:generated or hledger register --forecast + tag:generated. + + 3. Forecasted transactions will begin only after the last non-fore- + casted transaction's date. + + 4. Forecasted transactions will end 6 months from today, by default. + See below for the exact start/end rules. + + 5. period expressions can be tricky. Their documentation needs im- + provement, but is worth studying. + + 6. Some period expressions with a repeating interval must begin on a + natural boundary of that interval. Eg in weekly from DATE, DATE + must be a monday. ~ weekly from 2019/10/1 (a tuesday) will give an + error. + + 7. Other period expressions with an interval are automatically expanded + to cover a whole number of that interval. (This is done to improve + reports, but it also affects periodic transactions. Yes, it's a bit + inconsistent with the above.) Eg: ~ every 10th day of month from + 2020/01, which is equivalent to ~ every 10th day of month from + 2020/01/01, will be adjusted to start on 2019/12/10. + + Periodic rule syntax A periodic transaction rule looks like a normal journal entry, with the date replaced by a tilde (~) followed by a period expression (mnemonic: ~ looks like a recurring sine wave.): @@ -1150,7 +1194,7 @@ date, unless a Y default year directive is in effect, in which case they will be relative to Y/1/1. - Two spaces after the period expression + Two spaces between period expression and description! If the period expression is followed by a transaction description, these must be separated by two or more spaces. This helps hledger know where the period expression ends, so that descriptions can not acciden- @@ -1163,6 +1207,14 @@ assets:bank:checking $1500 income:acme inc + So, + + o Do write two spaces between your period expression and your transac- + tion description, if any. + + o Don't accidentally write two spaces in the middle of your period ex- + pression. + Forecasting with periodic transactions With the --forecast flag, each periodic transaction rule generates fu- ture transactions recurring at the specified interval. These are not @@ -1220,8 +1272,8 @@ checking) every month. Goals and actual performance can then be com- pared in budget reports. - For more details, see: balance: Budget report and Cookbook: Budgeting - and Forecasting. + For more details, see: balance: Budget report and Budgeting and Fore- + casting. Auto postings / transaction modifiers Transaction modifier rules, AKA auto posting rules, describe changes to @@ -1349,7 +1401,7 @@ COPYRIGHT - Copyright (C) 2007-2016 Simon Michael. + Copyright (C) 2007-2019 Simon Michael. Released under GNU GPL v3 or later. @@ -1362,4 +1414,4 @@ -hledger 1.15.2 September 2019 hledger_journal(5) +hledger 1.16 December 2019 hledger_journal(5) diff --git a/hledger_timeclock.5 b/hledger_timeclock.5 --- a/hledger_timeclock.5 +++ b/hledger_timeclock.5 @@ -1,5 +1,5 @@ -.TH "hledger_timeclock" "5" "September 2019" "hledger 1.15.2" "hledger User Manuals" +.TH "hledger_timeclock" "5" "December 2019" "hledger 1.16" "hledger User Manuals" @@ -80,7 +80,7 @@ .SH COPYRIGHT -Copyright (C) 2007-2016 Simon Michael. +Copyright (C) 2007-2019 Simon Michael. .br Released under GNU GPL v3 or later. diff --git a/hledger_timeclock.info b/hledger_timeclock.info --- a/hledger_timeclock.info +++ b/hledger_timeclock.info @@ -4,8 +4,8 @@  File: hledger_timeclock.info, Node: Top, Up: (dir) -hledger_timeclock(5) hledger 1.15.2 -*********************************** +hledger_timeclock(5) hledger 1.16 +********************************* hledger can read timeclock files. As with Ledger, these are (a subset of) timeclock.el's format, containing clock-in and clock-out entries as diff --git a/hledger_timeclock.txt b/hledger_timeclock.txt --- a/hledger_timeclock.txt +++ b/hledger_timeclock.txt @@ -65,7 +65,7 @@ COPYRIGHT - Copyright (C) 2007-2016 Simon Michael. + Copyright (C) 2007-2019 Simon Michael. Released under GNU GPL v3 or later. @@ -78,4 +78,4 @@ -hledger 1.15.2 September 2019 hledger_timeclock(5) +hledger 1.16 December 2019 hledger_timeclock(5) diff --git a/hledger_timedot.5 b/hledger_timedot.5 --- a/hledger_timedot.5 +++ b/hledger_timedot.5 @@ -1,5 +1,5 @@ -.TH "hledger_timedot" "5" "September 2019" "hledger 1.15.2" "hledger User Manuals" +.TH "hledger_timedot" "5" "December 2019" "hledger 1.16" "hledger User Manuals" @@ -142,7 +142,7 @@ .SH COPYRIGHT -Copyright (C) 2007-2016 Simon Michael. +Copyright (C) 2007-2019 Simon Michael. .br Released under GNU GPL v3 or later. diff --git a/hledger_timedot.info b/hledger_timedot.info --- a/hledger_timedot.info +++ b/hledger_timedot.info @@ -4,8 +4,8 @@  File: hledger_timedot.info, Node: Top, Next: FILE FORMAT, Up: (dir) -hledger_timedot(5) hledger 1.15.2 -********************************* +hledger_timedot(5) hledger 1.16 +******************************* Timedot is a plain text format for logging dated, categorised quantities (of time, usually), supported by hledger. It is convenient for @@ -111,7 +111,7 @@  Tag Table: Node: Top76 -Node: FILE FORMAT812 -Ref: #file-format913 +Node: FILE FORMAT808 +Ref: #file-format909  End Tag Table diff --git a/hledger_timedot.txt b/hledger_timedot.txt --- a/hledger_timedot.txt +++ b/hledger_timedot.txt @@ -111,7 +111,7 @@ COPYRIGHT - Copyright (C) 2007-2016 Simon Michael. + Copyright (C) 2007-2019 Simon Michael. Released under GNU GPL v3 or later. @@ -124,4 +124,4 @@ -hledger 1.15.2 September 2019 hledger_timedot(5) +hledger 1.16 December 2019 hledger_timedot(5) diff --git a/test/doctests.hs b/test/doctests.hs --- a/test/doctests.hs +++ b/test/doctests.hs @@ -15,6 +15,8 @@ $ stack test hledger-lib:test:doctests [--test-arguments '[--verbose] [--slow] [CIFILEPATHSUBSTRINGS]'] -} +-- This file can't be called doctest.hs ("File name does not match module name") + {-# LANGUAGE PackageImports #-} diff --git a/test/easytests.hs b/test/easytests.hs deleted file mode 100644 --- a/test/easytests.hs +++ /dev/null @@ -1,5 +0,0 @@ -{- -Run hledger-lib's easytest tests using the easytest runner. --} -import Hledger -main = run tests_Hledger diff --git a/test/unittest.hs b/test/unittest.hs new file mode 100644 --- /dev/null +++ b/test/unittest.hs @@ -0,0 +1,11 @@ +{- +Run the hledger-lib package's unit tests using the tasty test runner. +-} + +-- package-qualified import to avoid cabal missing-home-modules warning (and double-building ?) +{-# LANGUAGE PackageImports #-} +import "hledger-lib" Hledger (tests_Hledger) + +import Test.Tasty (defaultMain) + +main = defaultMain tests_Hledger