hledger-flow-0.16.1: test/CSVImport/ImportHelperTurtleTests.hs
{-# LANGUAGE OverloadedLists #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
module CSVImport.ImportHelperTurtleTests where
import Control.Concurrent.STM
import Data.Either
import qualified Data.Map.Strict as Map
import qualified Data.Text as T
import Hledger.Flow.Common
import Hledger.Flow.Import.ImportHelpersTurtle (allYearIncludeFiles, groupIncludeFiles, toIncludeFiles, toIncludeLine, yearsIncludeMap)
import Hledger.Flow.Import.Types (TurtleFileBundle)
import Hledger.Flow.PathHelpers (TurtlePath)
import Path
import Test.HUnit
import TestHelpers (defaultOpts)
import TestHelpersTurtle
import Turtle
groupedJaneBogart :: TurtleFileBundle
groupedJaneBogart =
[ ( "./import/jane/bogartbank/checking/2018-include.journal",
["import/jane/bogartbank/checking/3-journal/2018/2018-12-30.journal"]
),
( "./import/jane/bogartbank/checking/2019-include.journal",
["import/jane/bogartbank/checking/3-journal/2019/2019-01-30.journal"]
),
( "./import/jane/bogartbank/savings/2017-include.journal",
["import/jane/bogartbank/savings/3-journal/2017/2017-12-30.journal"]
),
( "./import/jane/bogartbank/savings/2018-include.journal",
["import/jane/bogartbank/savings/3-journal/2018/2018-01-30.journal"]
)
]
groupedJaneOther :: TurtleFileBundle
groupedJaneOther =
[ ( "./import/jane/otherbank/creditcard/2017-include.journal",
["import/jane/otherbank/creditcard/3-journal/2017/2017-12-30.journal"]
),
( "./import/jane/otherbank/creditcard/2018-include.journal",
["import/jane/otherbank/creditcard/3-journal/2018/2018-01-30.journal"]
),
( "./import/jane/otherbank/investments/2018-include.journal",
["import/jane/otherbank/investments/3-journal/2018/2018-12-30.journal"]
),
( "./import/jane/otherbank/investments/2019-include.journal",
["import/jane/otherbank/investments/3-journal/2019/2019-01-30.journal"]
)
]
groupedJohnBogart :: TurtleFileBundle
groupedJohnBogart =
[ ( "./import/john/bogartbank/checking/2018-include.journal",
[ "import/john/bogartbank/checking/3-journal/2018/2018-11-30.journal",
"import/john/bogartbank/checking/3-journal/2018/2018-10-30.journal",
"import/john/bogartbank/checking/3-journal/2018/2018-12-30.journal"
]
),
( "./import/john/bogartbank/checking/2019-include.journal",
[ "import/john/bogartbank/checking/3-journal/2019/2019-01-30.journal",
"import/john/bogartbank/checking/3-journal/2019/2019-02-30.journal"
]
),
( "./import/john/bogartbank/savings/2017-include.journal",
[ "import/john/bogartbank/savings/3-journal/2017/2017-11-30.journal",
"import/john/bogartbank/savings/3-journal/2017/2017-12-30.journal"
]
),
( "./import/john/bogartbank/savings/2018-include.journal",
[ "import/john/bogartbank/savings/3-journal/2018/2018-02-30.journal",
"import/john/bogartbank/savings/3-journal/2018/2018-01-30.journal"
]
)
]
groupedJohnOther :: TurtleFileBundle
groupedJohnOther =
[ ( "./import/john/otherbank/creditcard/2017-include.journal",
["import/john/otherbank/creditcard/3-journal/2017/2017-12-30.journal"]
),
( "./import/john/otherbank/creditcard/2018-include.journal",
["import/john/otherbank/creditcard/3-journal/2018/2018-01-30.journal"]
),
( "./import/john/otherbank/investments/2018-include.journal",
["import/john/otherbank/investments/3-journal/2018/2018-12-30.journal"]
),
( "./import/john/otherbank/investments/2019-include.journal",
["import/john/otherbank/investments/3-journal/2019/2019-01-30.journal"]
)
]
groupedIncludeFiles :: TurtleFileBundle
groupedIncludeFiles =
groupedJaneBogart
<> groupedJaneOther
<> groupedJohnBogart
<> groupedJohnOther
testYearsIncludeMap :: Test
testYearsIncludeMap =
TestCase
( do
let maps = allYearIncludeFiles groupedJohnOther
let yearGrouping =
[ ( "./import/john/otherbank/creditcard/all-years.journal",
[ "./import/john/otherbank/creditcard/2017-include.journal",
"./import/john/otherbank/creditcard/2018-include.journal"
]
),
( "./import/john/otherbank/investments/all-years.journal",
[ "./import/john/otherbank/investments/2018-include.journal",
"./import/john/otherbank/investments/2019-include.journal"
]
)
]
let expected = (groupedJohnOther, yearGrouping)
assertEqual "An augmented map with grouped years per level added" expected maps
)
testYearsIncludeGrouping :: Test
testYearsIncludeGrouping =
TestCase
( do
let yearsMap = yearsIncludeMap (Map.keys groupedJohnOther)
let expected =
[ ( "./import/john/otherbank/creditcard/all-years.journal",
[ "./import/john/otherbank/creditcard/2017-include.journal",
"./import/john/otherbank/creditcard/2018-include.journal"
]
),
( "./import/john/otherbank/investments/all-years.journal",
[ "./import/john/otherbank/investments/2018-include.journal",
"./import/john/otherbank/investments/2019-include.journal"
]
)
]
assertEqual "A basic map with grouped years per level" expected yearsMap
)
testGroupIncludeFilesTinySet :: Test
testGroupIncludeFilesTinySet =
TestCase
( do
let journals1 = ["import/jane/bogartbank/savings/3-journals/2017/2017-12-30.journal"]
let expected1 = [("./import/jane/bogartbank/savings/2017-include.journal", journals1)] :: TurtleFileBundle
let expectedAllYears1 = [("./import/jane/bogartbank/savings/all-years.journal", ["./import/jane/bogartbank/savings/2017-include.journal"])]
let (group1, allYears1) = groupIncludeFiles journals1
assertEqual "groupIncludeFiles small allYears 1" expectedAllYears1 allYears1
assertEqual "groupIncludeFiles small set 1" expected1 group1
let journals2 = [(fst . head . Map.toList) expected1] :: [TurtlePath]
let expected2 = [("./import/jane/bogartbank/2017-include.journal", journals2)] :: TurtleFileBundle
let expectedAllYears2 = [("./import/jane/bogartbank/all-years.journal", ["./import/jane/bogartbank/2017-include.journal"])]
let (group2, allYears2) = groupIncludeFiles journals2
assertEqual "groupIncludeFiles small allYears 2" expectedAllYears2 allYears2
assertEqual "groupIncludeFiles small set 2" expected2 group2
let journals3 = [(fst . head . Map.toList) expected2] :: [TurtlePath]
let expected3 = [("./import/jane/2017-include.journal", journals3)] :: TurtleFileBundle
let expectedAllYears3 = [("./import/jane/all-years.journal", ["./import/jane/2017-include.journal"])]
let (group3, allYears3) = groupIncludeFiles journals3
assertEqual "groupIncludeFiles small allYears 3" expectedAllYears3 allYears3
assertEqual "groupIncludeFiles small set 3" expected3 group3
)
testGroupIncludeFilesSmallSet :: Test
testGroupIncludeFilesSmallSet =
TestCase
( do
let (group1, allYears1) = groupIncludeFiles $ toJournals inputJaneBogart
let expectedAllYears1 =
[ ( "./import/jane/bogartbank/checking/all-years.journal",
[ "./import/jane/bogartbank/checking/2018-include.journal",
"./import/jane/bogartbank/checking/2019-include.journal"
]
),
( "./import/jane/bogartbank/savings/all-years.journal",
[ "./import/jane/bogartbank/savings/2017-include.journal",
"./import/jane/bogartbank/savings/2018-include.journal"
]
)
]
assertEqual "groupIncludeFiles Jane AllYears 1" expectedAllYears1 allYears1
assertEqual "groupIncludeFiles Jane 1" groupedJaneBogart group1
let (group2, allYears2) = groupIncludeFiles (Map.keys group1)
let expectedAllYears2 =
[ ( "./import/jane/bogartbank/all-years.journal",
[ "./import/jane/bogartbank/2017-include.journal",
"./import/jane/bogartbank/2018-include.journal",
"./import/jane/bogartbank/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles Jane AllYears 2" expectedAllYears2 allYears2
let expectedGroup2 =
[ ( "./import/jane/bogartbank/2017-include.journal",
["./import/jane/bogartbank/savings/2017-include.journal"]
),
( "./import/jane/bogartbank/2018-include.journal",
[ "./import/jane/bogartbank/checking/2018-include.journal",
"./import/jane/bogartbank/savings/2018-include.journal"
]
),
( "./import/jane/bogartbank/2019-include.journal",
["./import/jane/bogartbank/checking/2019-include.journal"]
)
]
assertEqual "groupIncludeFiles Jane 2" expectedGroup2 group2
let (group3, allYears3) = groupIncludeFiles (Map.keys group2)
let expectedAllYears3 =
[ ( "./import/jane/all-years.journal",
[ "./import/jane/2017-include.journal",
"./import/jane/2018-include.journal",
"./import/jane/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles Jane AllYears 3" expectedAllYears3 allYears3
let expectedGroup3 =
[ ( "./import/jane/2017-include.journal",
["./import/jane/bogartbank/2017-include.journal"]
),
( "./import/jane/2018-include.journal",
["./import/jane/bogartbank/2018-include.journal"]
),
( "./import/jane/2019-include.journal",
["./import/jane/bogartbank/2019-include.journal"]
)
]
assertEqual "groupIncludeFiles Jane 3" expectedGroup3 group3
let (group4, allYears4) = groupIncludeFiles (Map.keys group3)
let expectedAllYears4 =
[ ( "./import/all-years.journal",
[ "./import/2017-include.journal",
"./import/2018-include.journal",
"./import/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles Jane AllYears 4" expectedAllYears4 allYears4
let expectedGroup4 =
[ ( "./import/2017-include.journal",
["./import/jane/2017-include.journal"]
),
( "./import/2018-include.journal",
["./import/jane/2018-include.journal"]
),
( "./import/2019-include.journal",
["./import/jane/2019-include.journal"]
)
]
assertEqual "groupIncludeFiles Jane 4" expectedGroup4 group4
let (group5, allYears5) = groupIncludeFiles (Map.keys group4)
let expectedAllYears5 = [("./all-years.journal", ["./2017-include.journal", "./2018-include.journal", "./2019-include.journal"])]
assertEqual "groupIncludeFiles Jane AllYears 5" expectedAllYears5 allYears5
let expectedGroup5 =
[ ( "./2017-include.journal",
["./import/2017-include.journal"]
),
( "./2018-include.journal",
["./import/2018-include.journal"]
),
( "./2019-include.journal",
["./import/2019-include.journal"]
)
]
assertEqual "groupIncludeFiles Jane 5" expectedGroup5 group5
)
testGroupIncludeFiles :: Test
testGroupIncludeFiles =
TestCase
( do
let (group1, allYears1) = groupIncludeFiles journalFiles
let expectedAllYears1 =
[ ( "./import/jane/bogartbank/checking/all-years.journal",
[ "./import/jane/bogartbank/checking/2018-include.journal",
"./import/jane/bogartbank/checking/2019-include.journal"
]
),
( "./import/jane/bogartbank/savings/all-years.journal",
[ "./import/jane/bogartbank/savings/2017-include.journal",
"./import/jane/bogartbank/savings/2018-include.journal"
]
),
( "./import/jane/otherbank/creditcard/all-years.journal",
[ "./import/jane/otherbank/creditcard/2017-include.journal",
"./import/jane/otherbank/creditcard/2018-include.journal"
]
),
( "./import/jane/otherbank/investments/all-years.journal",
[ "./import/jane/otherbank/investments/2018-include.journal",
"./import/jane/otherbank/investments/2019-include.journal"
]
),
( "./import/john/bogartbank/checking/all-years.journal",
[ "./import/john/bogartbank/checking/2018-include.journal",
"./import/john/bogartbank/checking/2019-include.journal"
]
),
( "./import/john/bogartbank/savings/all-years.journal",
[ "./import/john/bogartbank/savings/2017-include.journal",
"./import/john/bogartbank/savings/2018-include.journal"
]
),
( "./import/john/otherbank/creditcard/all-years.journal",
[ "./import/john/otherbank/creditcard/2017-include.journal",
"./import/john/otherbank/creditcard/2018-include.journal"
]
),
( "./import/john/otherbank/investments/all-years.journal",
[ "./import/john/otherbank/investments/2018-include.journal",
"./import/john/otherbank/investments/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles Jane AllYears 1" expectedAllYears1 allYears1
assertEqual "groupIncludeFiles 1" groupedIncludeFiles group1
let (group2, allYears2) = groupIncludeFiles (Map.keys group1)
let expectedAllYears2 =
[ ( "./import/jane/bogartbank/all-years.journal",
[ "./import/jane/bogartbank/2017-include.journal",
"./import/jane/bogartbank/2018-include.journal",
"./import/jane/bogartbank/2019-include.journal"
]
),
( "./import/jane/otherbank/all-years.journal",
[ "./import/jane/otherbank/2017-include.journal",
"./import/jane/otherbank/2018-include.journal",
"./import/jane/otherbank/2019-include.journal"
]
),
( "./import/john/bogartbank/all-years.journal",
[ "./import/john/bogartbank/2017-include.journal",
"./import/john/bogartbank/2018-include.journal",
"./import/john/bogartbank/2019-include.journal"
]
),
( "./import/john/otherbank/all-years.journal",
[ "./import/john/otherbank/2017-include.journal",
"./import/john/otherbank/2018-include.journal",
"./import/john/otherbank/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles Jane AllYears 2" expectedAllYears2 allYears2
let expectedGroup2 =
[ ( "./import/jane/bogartbank/2017-include.journal",
["./import/jane/bogartbank/savings/2017-include.journal"]
),
( "./import/jane/bogartbank/2018-include.journal",
[ "./import/jane/bogartbank/checking/2018-include.journal",
"./import/jane/bogartbank/savings/2018-include.journal"
]
),
( "./import/jane/bogartbank/2019-include.journal",
["./import/jane/bogartbank/checking/2019-include.journal"]
),
( "./import/jane/otherbank/2017-include.journal",
["./import/jane/otherbank/creditcard/2017-include.journal"]
),
( "./import/jane/otherbank/2018-include.journal",
[ "./import/jane/otherbank/creditcard/2018-include.journal",
"./import/jane/otherbank/investments/2018-include.journal"
]
),
( "./import/jane/otherbank/2019-include.journal",
["./import/jane/otherbank/investments/2019-include.journal"]
),
( "./import/john/bogartbank/2017-include.journal",
["./import/john/bogartbank/savings/2017-include.journal"]
),
( "./import/john/bogartbank/2018-include.journal",
[ "./import/john/bogartbank/checking/2018-include.journal",
"./import/john/bogartbank/savings/2018-include.journal"
]
),
( "./import/john/bogartbank/2019-include.journal",
["./import/john/bogartbank/checking/2019-include.journal"]
),
( "./import/john/otherbank/2017-include.journal",
["./import/john/otherbank/creditcard/2017-include.journal"]
),
( "./import/john/otherbank/2018-include.journal",
[ "./import/john/otherbank/creditcard/2018-include.journal",
"./import/john/otherbank/investments/2018-include.journal"
]
),
( "./import/john/otherbank/2019-include.journal",
["./import/john/otherbank/investments/2019-include.journal"]
)
]
assertEqual "groupIncludeFiles 2 - diff 1" [] (expectedGroup2 Map.\\ group2)
assertEqual "groupIncludeFiles 2 - diff 2" [] (group2 Map.\\ expectedGroup2)
assertEqual "groupIncludeFiles 2" expectedGroup2 group2
let (group3, allYears3) = groupIncludeFiles (Map.keys group2)
let expectedAllYears3 =
[ ( "./import/jane/all-years.journal",
[ "./import/jane/2017-include.journal",
"./import/jane/2018-include.journal",
"./import/jane/2019-include.journal"
]
),
( "./import/john/all-years.journal",
[ "./import/john/2017-include.journal",
"./import/john/2018-include.journal",
"./import/john/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles Jane AllYears 3" expectedAllYears3 allYears3
let expectedGroup3 =
[ ( "./import/jane/2017-include.journal",
[ "./import/jane/bogartbank/2017-include.journal",
"./import/jane/otherbank/2017-include.journal"
]
),
( "./import/jane/2018-include.journal",
[ "./import/jane/bogartbank/2018-include.journal",
"./import/jane/otherbank/2018-include.journal"
]
),
( "./import/jane/2019-include.journal",
[ "./import/jane/bogartbank/2019-include.journal",
"./import/jane/otherbank/2019-include.journal"
]
),
( "./import/john/2017-include.journal",
[ "./import/john/bogartbank/2017-include.journal",
"./import/john/otherbank/2017-include.journal"
]
),
( "./import/john/2018-include.journal",
[ "./import/john/bogartbank/2018-include.journal",
"./import/john/otherbank/2018-include.journal"
]
),
( "./import/john/2019-include.journal",
[ "./import/john/bogartbank/2019-include.journal",
"./import/john/otherbank/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles 3 - diff 1" [] (expectedGroup3 Map.\\ group3)
assertEqual "groupIncludeFiles 3 - diff 2" [] (group3 Map.\\ expectedGroup3)
assertEqual "groupIncludeFiles 3" expectedGroup3 group3
let (group4, allYears4) = groupIncludeFiles (Map.keys group3)
let expectedAllYears4 =
[ ( "./import/all-years.journal",
[ "./import/2017-include.journal",
"./import/2018-include.journal",
"./import/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles Jane AllYears 4" expectedAllYears4 allYears4
let expectedGroup4 =
[ ( "./import/2017-include.journal",
[ "./import/jane/2017-include.journal",
"./import/john/2017-include.journal"
]
),
( "./import/2018-include.journal",
[ "./import/jane/2018-include.journal",
"./import/john/2018-include.journal"
]
),
( "./import/2019-include.journal",
[ "./import/jane/2019-include.journal",
"./import/john/2019-include.journal"
]
)
]
assertEqual "groupIncludeFiles 4 - diff 1" [] (expectedGroup4 Map.\\ group4)
assertEqual "groupIncludeFiles 4 - diff 2" [] (group4 Map.\\ expectedGroup4)
assertEqual "groupIncludeFiles 4" expectedGroup4 group4
let (group5, allYears5) = groupIncludeFiles (Map.keys group4)
let expectedAllYears5 = [("./all-years.journal", ["./2017-include.journal", "./2018-include.journal", "./2019-include.journal"])]
assertEqual "groupIncludeFiles Jane AllYears 5" expectedAllYears5 allYears5
let expectedGroup5 =
[ ( "./2017-include.journal",
["./import/2017-include.journal"]
),
( "./2018-include.journal",
["./import/2018-include.journal"]
),
( "./2019-include.journal",
["./import/2019-include.journal"]
)
]
assertEqual "groupIncludeFiles 5 - diff 1" [] (expectedGroup5 Map.\\ group5)
assertEqual "groupIncludeFiles 5 - diff 2" [] (group5 Map.\\ expectedGroup5)
assertEqual "groupIncludeFiles 5" expectedGroup5 group5
)
testIncludeYears :: Test
testIncludeYears =
TestCase
( do
let txterr = "Some text without years"
let expectederr = ["Unable to extract years from the following text:", txterr, "Errors:"]
let actualerr = (init . head) $ map T.lines $ lefts [includeYears' txterr] :: [Text]
assertEqual "Get a list of years from an include file - error case" expectederr actualerr
let txt1 =
"### Generated by hledger-flow - DO NOT EDIT ###\n\n"
<> "include import/2014-include.journal\n"
<> "include import/2015-include.journal\n"
<> "include import/2016-include.journal\n"
<> "include import/2017-include.journal\n"
<> "include import/2018-include.journal\n"
<> "include import/2019-include.journal"
let expected1 = Right [2014 .. 2019]
let actual1 = includeYears' txt1
assertEqual "Get a list of years from an include file - success case 1" expected1 actual1
let txt2 = "include 2019-include.journal"
let expected2 = Right [2019]
let actual2 = includeYears' txt2
assertEqual "Get a list of years from an include file - success case 2" expected2 actual2
)
testToIncludeLine :: Test
testToIncludeLine =
TestCase
( do
let expected = "include file1.journal"
let relativeWithTrailingSlash = toIncludeLine "./base/dir/" "./base/dir/file1.journal"
assertEqual "Include line - relative base dir with trailing slash" expected relativeWithTrailingSlash
let relativeNoTrailingSlash = toIncludeLine "./base/dir" "./base/dir/file1.journal"
assertEqual "Include line - relative base dir without a trailing slash" expected relativeNoTrailingSlash
let absoluteWithTrailingSlash = toIncludeLine "/base/dir/" "/base/dir/file1.journal"
assertEqual "Include line - absolute base dir with trailing slash" expected absoluteWithTrailingSlash
let absoluteNoTrailingSlash = toIncludeLine "/base/dir" "/base/dir/file1.journal"
assertEqual "Include line - absolute base dir without a trailing slash" expected absoluteNoTrailingSlash
)
testToIncludeFiles :: Test
testToIncludeFiles =
TestCase
( do
let expected =
[ ( "./import/john/bogartbank/checking/2018-include.journal",
"### Generated by hledger-flow - DO NOT EDIT ###\n\n"
<> "include import/john/bogartbank/checking/3-journal/2018/2018-10-30.journal\n"
<> "include import/john/bogartbank/checking/3-journal/2018/2018-11-30.journal\n"
<> "include import/john/bogartbank/checking/3-journal/2018/2018-12-30.journal\n"
),
( "./import/john/bogartbank/checking/2019-include.journal",
"### Generated by hledger-flow - DO NOT EDIT ###\n\n"
<> "include import/john/bogartbank/checking/3-journal/2019/2019-01-30.journal\n"
<> "include import/john/bogartbank/checking/3-journal/2019/2019-02-30.journal\n"
),
( "./import/john/bogartbank/savings/2017-include.journal",
"### Generated by hledger-flow - DO NOT EDIT ###\n\n"
<> "include import/john/bogartbank/savings/3-journal/2017/2017-11-30.journal\n"
<> "include import/john/bogartbank/savings/3-journal/2017/2017-12-30.journal\n"
),
( "./import/john/bogartbank/savings/2018-include.journal",
"### Generated by hledger-flow - DO NOT EDIT ###\n\n"
<> "include import/john/bogartbank/savings/3-journal/2018/2018-01-30.journal\n"
<> "include import/john/bogartbank/savings/3-journal/2018/2018-02-30.journal\n"
)
]
ch <- newTChanIO
txt <- toIncludeFiles (defaultOpts [absdir|/|]) ch groupedJohnBogart
assertEqual "Convert a grouped map of paths, to a map with text contents for each file" expected txt
)
tests :: Test
tests = TestList [testYearsIncludeMap, testYearsIncludeGrouping, testGroupIncludeFilesTinySet, testGroupIncludeFilesSmallSet, testGroupIncludeFiles, testIncludeYears, testToIncludeLine, testToIncludeFiles]