hledger-flow 0.14.2.0 → 0.14.3.0

raw patch · 19 files changed

+1062/−676 lines, 19 filesdep +filepathdep +gitrev

Dependencies added: filepath, gitrev

Files

• ChangeLog.md +6/−0
• app/Main.hs +4/−3
• hledger-flow.cabal +9/−2
• src/Hledger/Flow/Common.hs +15/−162
• src/Hledger/Flow/Import/CSVImport.hs +3/−2
• src/Hledger/Flow/Import/ImportHelpers.hs +42/−7
• src/Hledger/Flow/Import/ImportHelpersTurtle.hs +168/−0
• src/Hledger/Flow/Import/Types.hs +6/−3
• src/Hledger/Flow/Internals.hs +31/−0
• src/Hledger/Flow/Reports.hs +2/−2
• src/Hledger/Flow/RuntimeOptions.hs +1/−0
• src/Hledger/Flow/Types.hs +1/−10
• test/CSVImport/ImportHelperTests.hs +119/−0
• test/CSVImport/ImportHelperTurtleTests.hs +432/−0
• test/CSVImport/Integration.hs +4/−1
• test/CSVImport/Unit.hs +3/−423
• test/Common/Integration.hs +2/−2
• test/TestHelpers.hs +143/−59
• test/TestHelpersTurtle.hs +71/−0

@@ -1,5 +1,11 @@ # Changelog for [hledger-flow](https://github.com/apauley/hledger-flow) +## 0.14.3++Ensure that generated include files only contain files ending with .journal++Fixes [#92](https://github.com/apauley/hledger-flow/issues/92)+ ## 0.14.2  Add an optional `--start-year` command-line option for imports:

@@ -12,6 +12,7 @@  import Hledger.Flow.PathHelpers (TurtlePath) import Hledger.Flow.Common+import Hledger.Flow.Internals (versionInfo, systemInfo) import Hledger.Flow.BaseDir import qualified Hledger.Flow.RuntimeOptions as RT import Hledger.Flow.Reports@@ -41,7 +42,7 @@ main = do   cmd <- Turtle.options "An hledger workflow focusing on automated statement import and classification:\nhttps://github.com/apauley/hledger-flow#readme" baseCommandParser   case cmd of-    Version                                -> Turtle.stdout $ Turtle.select versionInfo+    Version                                -> T.putStrLn versionInfo     Command mainParams' (Import subParams) -> toRuntimeOptionsImport mainParams' subParams >>= importCSVs     Command mainParams' (Report subParams) -> toRuntimeOptionsReport mainParams' subParams >>= generateReports @@ -57,7 +58,7 @@                            , RT.importRunDir = runDir                            , RT.importStartYear = startYear                            , RT.onlyNewFiles = onlyNewFiles subParams'-                           , RT.hfVersion = versionInfo'+                           , RT.hfVersion = versionInfo                            , RT.hledgerInfo = hli                            , RT.sysInfo = systemInfo                            , RT.verbose = verbosity mainParams' > 0@@ -73,7 +74,7 @@                            , RT.importRunDir = [reldir|.|]                            , RT.importStartYear = Nothing                            , RT.onlyNewFiles = False-                           , RT.hfVersion = versionInfo'+                           , RT.hfVersion = versionInfo                            , RT.hledgerInfo = hli                            , RT.sysInfo = systemInfo                            , RT.verbose = verbosity mainParams' > 0

@@ -4,10 +4,10 @@ -- -- see: https://github.com/sol/hpack ----- hash: cb2578ca8c6701d18ddc78599992215ad8c3b4f13ee2deb248bd30112365d3e4+-- hash: 7dede2e183b932b3b9af01bd855ca6725acdbefbc962fee1ed1eb113df327286  name:           hledger-flow-version:        0.14.2.0+version:        0.14.3.0 synopsis:       An hledger workflow focusing on automated statement import and classification. description:    Please see the README on GitHub at <https://github.com/apauley/hledger-flow#readme> category:       Finance, Console@@ -35,7 +35,9 @@       Hledger.Flow.DocHelpers       Hledger.Flow.Import.CSVImport       Hledger.Flow.Import.ImportHelpers+      Hledger.Flow.Import.ImportHelpersTurtle       Hledger.Flow.Import.Types+      Hledger.Flow.Internals       Hledger.Flow.Logging       Hledger.Flow.PathHelpers       Hledger.Flow.Reports@@ -50,7 +52,9 @@       base >=4.7 && <5     , containers     , exceptions+    , filepath     , foldl+    , gitrev     , path     , path-io     , stm@@ -84,10 +88,13 @@       BaseDir.Integration       Common.Integration       Common.Unit+      CSVImport.ImportHelperTests+      CSVImport.ImportHelperTurtleTests       CSVImport.Integration       CSVImport.Unit       PathHelpers.Unit       TestHelpers+      TestHelpersTurtle       Paths_hledger_flow   hs-source-dirs:       test

@@ -1,7 +1,11 @@ {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}  module Hledger.Flow.Common where +import Path (absfile, relfile)+import qualified Path.IO as Path+ import qualified Turtle import Turtle ((%), (</>), (<.>)) @@ -12,66 +16,41 @@ import qualified GHC.IO.Handle.FD as H  import Data.Char (isDigit)-import Data.Maybe import Data.Either  import qualified Control.Foldl as Fold import qualified Data.Map.Strict as Map  import Data.Function (on)-import qualified Data.List as List (nub, null, sort, sortBy, groupBy)+import qualified Data.List as List (null, sortBy, groupBy) import Data.Ord (comparing)  import Hledger.Flow.Types-import qualified Hledger.Flow.Import.Types as IT  import Hledger.Flow.Logging-import Hledger.Flow.PathHelpers (TurtlePath)-import Hledger.Flow.BaseDir (turtleBaseDir, relativeToBase, relativeToBase')-import Hledger.Flow.DocHelpers (docURL)+import Hledger.Flow.PathHelpers (AbsFile, TurtlePath, fromTurtleAbsFile, pathToTurtle)+import Hledger.Flow.BaseDir (turtleBaseDir, relativeToBase)  import Control.Concurrent.STM -import qualified Data.List.NonEmpty as NE-import Paths_hledger_flow (version)-import qualified Data.Version as Version (showVersion)-import qualified System.Info as Sys--type InputFileBundle = Map.Map TurtlePath [TurtlePath]--versionInfo :: NE.NonEmpty Turtle.Line-versionInfo = Turtle.textToLines versionInfo'--versionInfo' :: T.Text-versionInfo' = T.pack ("hledger-flow " ++ Version.showVersion version ++ " " ++-                       os systemInfo ++ " " ++ arch systemInfo ++ " " ++-                       compilerName systemInfo ++ " " ++-                       Version.showVersion (compilerVersion systemInfo))--systemInfo :: SystemInfo-systemInfo = SystemInfo { os = Sys.os-                        , arch = Sys.arch-                        , compilerName = Sys.compilerName-                        , compilerVersion = Sys.compilerVersion-                        }--hledgerPathFromOption :: Maybe TurtlePath -> IO TurtlePath+hledgerPathFromOption :: Maybe TurtlePath -> IO AbsFile hledgerPathFromOption pathOption = do   case pathOption of     Just h  -> do-      isOnDisk <- Turtle.testfile h-      if isOnDisk then return h else do+      hlAbs <- fromTurtleAbsFile h+      isOnDisk <- Path.doesFileExist hlAbs+      if isOnDisk then return hlAbs else do         let msg = Turtle.format ("Unable to find hledger at "%Turtle.fp) h-        errExit' 1 (T.hPutStrLn H.stderr) msg h+        errExit' 1 (T.hPutStrLn H.stderr) msg hlAbs     Nothing -> do-      maybeH <- Turtle.which "hledger"+      maybeH <- Path.findExecutable [relfile|hledger|]       case maybeH of         Just h  -> return h         Nothing -> do           let msg = "Unable to find hledger in your path.\n"                 <> "You need to either install hledger, or add it to your PATH, or provide the path to an hledger executable.\n\n"                 <> "There are a number of installation options on the hledger website: https://hledger.org/download.html"-          errExit' 1 (T.hPutStrLn H.stderr) msg "/"+          errExit' 1 (T.hPutStrLn H.stderr) msg [absfile|/hledger|]  hledgerVersionFromPath :: TurtlePath -> IO T.Text hledgerVersionFromPath hlp = fmap (T.strip . Turtle.linesToText) (Turtle.single $ shellToList $ Turtle.inproc (Turtle.format Turtle.fp hlp) ["--version"] Turtle.empty)@@ -79,7 +58,7 @@ hledgerInfoFromPath :: Maybe TurtlePath -> IO HledgerInfo hledgerInfoFromPath pathOption = do   hlp <- hledgerPathFromOption pathOption-  hlv <- hledgerVersionFromPath hlp+  hlv <- hledgerVersionFromPath $ pathToTurtle hlp   return $ HledgerInfo hlp hlv  showCmdArgs :: [T.Text] -> T.Text@@ -185,36 +164,9 @@ groupValuesBy :: (Ord k, Ord v) => (v -> k) -> [v] -> Map.Map k [v] groupValuesBy keyFun = groupPairs . pairBy keyFun -initialIncludeFilePath :: TurtlePath -> TurtlePath-initialIncludeFilePath p = (Turtle.parent . Turtle.parent . Turtle.parent) p </> includeFileName p--parentIncludeFilePath :: TurtlePath -> TurtlePath-parentIncludeFilePath p = (Turtle.parent . Turtle.parent) p </> (Turtle.filename p)--allYearsPath :: TurtlePath -> TurtlePath-allYearsPath = allYearsPath' Turtle.directory--allYearsPath' :: (TurtlePath -> TurtlePath) -> TurtlePath -> TurtlePath-allYearsPath' dir p = dir p </> "all-years.journal"- allYearsFileName :: TurtlePath allYearsFileName = "all-years" <.> "journal" -groupIncludeFiles :: [TurtlePath] -> (InputFileBundle, InputFileBundle)-groupIncludeFiles = allYearIncludeFiles . groupIncludeFilesPerYear--groupIncludeFilesPerYear :: [TurtlePath] -> InputFileBundle-groupIncludeFilesPerYear [] = Map.empty-groupIncludeFilesPerYear ps@(p:_) = case extractImportDirs p of-    Right _ -> (groupValuesBy initialIncludeFilePath) ps-    Left  _ -> (groupValuesBy parentIncludeFilePath)  ps--allYearIncludeFiles :: InputFileBundle -> (InputFileBundle, InputFileBundle)-allYearIncludeFiles m = (m, yearsIncludeMap $ Map.keys m)--yearsIncludeMap :: [TurtlePath] -> InputFileBundle-yearsIncludeMap = groupValuesBy allYearsPath- lsDirs :: TurtlePath -> Turtle.Shell TurtlePath lsDirs = onlyDirs . Turtle.ls @@ -276,65 +228,6 @@ shellToList :: Turtle.Shell a -> Turtle.Shell [a] shellToList files = Turtle.fold files Fold.list -includeFileName :: TurtlePath -> TurtlePath-includeFileName = (<.> "journal") . Turtle.fromText . (Turtle.format (Turtle.fp%"-include")) . Turtle.dirname--toIncludeFiles :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> InputFileBundle -> IO (Map.Map TurtlePath T.Text)-toIncludeFiles opts ch m = do-  preMap  <- extraIncludes opts ch (Map.keys m) ["opening.journal"] ["pre-import.journal"] []-  postMap <- extraIncludes opts ch (Map.keys m) ["closing.journal"] ["post-import.journal"] ["prices.journal"]-  return $ (addPreamble . toIncludeFiles' preMap postMap) m--extraIncludes :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> [TurtlePath] -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO InputFileBundle-extraIncludes opts ch = extraIncludes' opts ch Map.empty--extraIncludes' :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> InputFileBundle -> [TurtlePath] -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO InputFileBundle-extraIncludes' _ _ acc [] _ _ _ = return acc-extraIncludes' opts ch acc (file:files) extraSuffixes manualFiles prices = do-  extra <- extraIncludesForFile opts ch file extraSuffixes manualFiles prices-  extraIncludes' opts ch (Map.unionWith (++) acc extra) files extraSuffixes manualFiles prices--extraIncludesForFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> TurtlePath -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO InputFileBundle-extraIncludesForFile opts ch file extraSuffixes manualFiles prices = do-  let dirprefix = Turtle.fromText $ fst $ T.breakOn "-" $ Turtle.format Turtle.fp $ Turtle.basename file-  let fileNames = map (\suff -> Turtle.fromText $ Turtle.format (Turtle.fp%"-"%Turtle.s) dirprefix suff) extraSuffixes-  let suffixFiles = map (Turtle.directory file </>) fileNames-  let suffixDirFiles = map (Turtle.directory file </> "_manual_" </> dirprefix </>) manualFiles-  let priceFiles = map (Turtle.directory file </> ".." </> "prices" </> dirprefix </>) prices-  let extraFiles = suffixFiles ++ suffixDirFiles ++ priceFiles-  filtered <- Turtle.single $ filterPaths Turtle.testfile extraFiles-  let logMsg = Turtle.format ("Looking for possible extra include files for '"%Turtle.fp%"' among these "%Turtle.d%" options: "%Turtle.s%". Found "%Turtle.d%": "%Turtle.s)-               (relativeToBase opts file) (length extraFiles) (Turtle.repr $ relativeFilesAsText opts extraFiles)-               (length filtered) (Turtle.repr $ relativeFilesAsText opts filtered)-  logVerbose opts ch logMsg-  return $ Map.fromList [(file, filtered)]--relativeFilesAsText :: HasBaseDir o => o -> [TurtlePath] -> [T.Text]-relativeFilesAsText opts ps = map ((Turtle.format Turtle.fp) . (relativeToBase opts)) ps--toIncludeFiles' :: InputFileBundle -> InputFileBundle -> InputFileBundle -> Map.Map TurtlePath T.Text-toIncludeFiles' preMap postMap = Map.mapWithKey $ generatedIncludeText preMap postMap--addPreamble :: Map.Map TurtlePath T.Text -> Map.Map TurtlePath T.Text-addPreamble = Map.map (\txt -> includePreamble <> "\n" <> txt)--toIncludeLine :: TurtlePath -> TurtlePath -> T.Text-toIncludeLine base file = Turtle.format ("!include "%Turtle.fp) $ relativeToBase' base file--generatedIncludeText :: InputFileBundle -> InputFileBundle -> TurtlePath -> [TurtlePath] -> T.Text-generatedIncludeText preMap postMap outputFile fs = do-  let preFiles = fromMaybe [] $ Map.lookup outputFile preMap-  let files = List.nub . List.sort $ fs-  let postFiles = fromMaybe [] $ Map.lookup outputFile postMap-  let lns = map (toIncludeLine $ Turtle.directory outputFile) $ preFiles ++ files ++ postFiles-  T.intercalate "\n" $ lns ++ [""]--includePreamble :: T.Text-includePreamble = "### Generated by hledger-flow - DO NOT EDIT ###\n"--groupAndWriteIncludeFiles :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> [TurtlePath] -> IO [TurtlePath]-groupAndWriteIncludeFiles opts ch = writeFileMap opts ch . groupIncludeFiles- writeFiles :: IO (Map.Map TurtlePath T.Text) -> IO [TurtlePath] writeFiles fileMap = do   m <- fileMap@@ -348,26 +241,6 @@ writeTextMap :: Map.Map TurtlePath T.Text -> IO () writeTextMap = Map.foldlWithKey (\a k v -> a <> Turtle.writeTextFile k v) (return ()) -writeFileMap :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> (InputFileBundle, InputFileBundle) -> IO [TurtlePath]-writeFileMap opts ch (m, allYears) = do-  _ <- writeFiles' $ (addPreamble . toIncludeFiles' Map.empty Map.empty) allYears-  writeFiles . (toIncludeFiles opts ch) $ m--writeIncludesUpTo :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> TurtlePath -> [TurtlePath] -> IO [TurtlePath]-writeIncludesUpTo _ _ _ [] = return []-writeIncludesUpTo opts ch stopAt journalFiles = do-  let shouldStop = any (\dir -> dir == stopAt) $ map Turtle.parent journalFiles-  if shouldStop-    then return journalFiles-    else do-      newJournalFiles <- groupAndWriteIncludeFiles opts ch journalFiles-      writeIncludesUpTo opts ch stopAt newJournalFiles--writeToplevelAllYearsInclude :: (HasBaseDir o, HasVerbosity o) => o -> IO [TurtlePath]-writeToplevelAllYearsInclude opts = do-  let allTop = Map.singleton (turtleBaseDir opts </> allYearsFileName) ["import" </> allYearsFileName]-  writeFiles' $ (addPreamble . toIncludeFiles' Map.empty Map.empty) allTop- changeExtension :: T.Text -> TurtlePath -> TurtlePath changeExtension ext path = (Turtle.dropExtension path) <.> ext @@ -377,26 +250,6 @@ changeOutputPath :: TurtlePath -> TurtlePath -> TurtlePath changeOutputPath newOutputLocation srcFile = mconcat $ map changeSrcDir $ Turtle.splitDirectories srcFile   where changeSrcDir file = if file == "1-in/" || file == "2-preprocessed/" then newOutputLocation else file--importDirBreakdown ::  TurtlePath -> [TurtlePath]-importDirBreakdown = importDirBreakdown' []--importDirBreakdown' :: [TurtlePath] -> TurtlePath -> [TurtlePath]-importDirBreakdown' acc path = do-  let dir = Turtle.directory path-  if Turtle.dirname dir == "import" || (Turtle.dirname dir == "")-    then dir:acc-    else importDirBreakdown' (dir:acc) $ Turtle.parent dir--extractImportDirs :: TurtlePath -> Either T.Text IT.ImportDirs-extractImportDirs inputFile = do-  case importDirBreakdown inputFile of-    [bd,owner,bank,account,filestate,year] -> Right $ IT.ImportDirs bd owner bank account filestate year-    _ -> do-      Left $ Turtle.format ("I couldn't find the right number of directories between \"import\" and the input file:\n"%Turtle.fp-                      %"\n\nhledger-flow expects to find input files in this structure:\n"%-                      "import/owner/bank/account/filestate/year/trxfile\n\n"%-                      "Have a look at the documentation for a detailed explanation:\n"%Turtle.s) inputFile (docURL "input-files")  listOwners :: HasBaseDir o => o -> Turtle.Shell TurtlePath listOwners opts = fmap Turtle.basename $ lsDirs $ (turtleBaseDir opts) </> "import"

@@ -13,6 +13,7 @@ import Hledger.Flow.Import.Types import Hledger.Flow.BaseDir (relativeToBase, effectiveRunDir) import Hledger.Flow.Import.ImportHelpers+import Hledger.Flow.Import.ImportHelpersTurtle (extractImportDirs, writeIncludesUpTo, writeToplevelAllYearsInclude) import Hledger.Flow.PathHelpers (TurtlePath, pathToTurtle) import Hledger.Flow.DocHelpers (docURL) import Hledger.Flow.Common@@ -153,7 +154,7 @@   case maybeRulesFile of     Just rf -> do       let relRules = relativeToBase opts rf-      let hledger = Turtle.format Turtle.fp $ FlowTypes.hlPath . hledgerInfo $ opts :: T.Text+      let hledger = Turtle.format Turtle.fp $ pathToTurtle . FlowTypes.hlPath . hledgerInfo $ opts :: T.Text       let args = ["print", "--rules-file", Turtle.format Turtle.fp rf, "--file", Turtle.format Turtle.fp csvSrc, "--output-file", Turtle.format Turtle.fp journalOut]       let cmdLabel = Turtle.format ("importing '"%Turtle.fp%"' using rules file '"%Turtle.fp%"'") relCSV relRules       _ <- timeAndExitOnErr opts ch cmdLabel channelOut channelErr (parAwareProc opts) (hledger, args, Turtle.empty)@@ -204,7 +205,7 @@   let constructArgs = [Turtle.format Turtle.fp csvSrc, "-", Turtle.lineToText bank, Turtle.lineToText account, Turtle.lineToText owner]   let constructCmdText = Turtle.format ("Running: "%Turtle.fp%" "%Turtle.s) relScript (showCmdArgs constructArgs)   let stdLines = inprocWithErrFun (channelErrLn ch) (script, constructArgs, Turtle.empty)-  let hledger = Turtle.format Turtle.fp $ FlowTypes.hlPath . hledgerInfo $ opts :: T.Text+  let hledger = Turtle.format Turtle.fp $ pathToTurtle . FlowTypes.hlPath . hledgerInfo $ opts :: T.Text   let args = ["print", "--ignore-assertions", "--file", "-", "--output-file", Turtle.format Turtle.fp journalOut]   let relSrc = relativeToBase opts csvSrc   let cmdLabel = Turtle.format ("executing '"%Turtle.fp%"' on '"%Turtle.fp%"'") relScript relSrc

@@ -1,21 +1,27 @@ {-# LANGUAGE QuasiQuotes #-} -module Hledger.Flow.Import.ImportHelpers (findInputFiles, findJournalFiles) where+module Hledger.Flow.Import.ImportHelpers (findInputFiles, findJournalFiles, groupIncludesUpTo, includeFileName) where  import Path import Data.Char (isDigit)+import Data.Maybe (fromMaybe)+import System.FilePath (dropTrailingPathSeparator) -import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, findFilesIn)+import Hledger.Flow.Common (groupValuesBy)+import Hledger.Flow.PathHelpers (AbsDir, AbsFile, RelDir, RelFile, findFilesIn, pathSize)+import Hledger.Flow.Import.Types (InputFileBundle) +import qualified Data.Map.Strict as Map+ findInputFiles :: Integer -> AbsDir -> IO [AbsFile] findInputFiles startYear = do-  let excludeDirs = [[Path.reldir|2-preprocessed|], [Path.reldir|3-journal|]] ++ commonExcludeDirs-  findFilesIn (includeYearFilesForParent [Path.reldir|1-in|] startYear) excludeDirs+  let excludeDirs = [[reldir|2-preprocessed|], [reldir|3-journal|]] ++ commonExcludeDirs+  findFilesIn (includeYearFilesForParent [reldir|1-in|] startYear) excludeDirs  findJournalFiles :: AbsDir -> IO [AbsFile] findJournalFiles = do-  let excludeDirs = [[Path.reldir|1-in|], [Path.reldir|2-preprocessed|]] ++ commonExcludeDirs-  findFilesIn (includeYearFilesForParent [Path.reldir|3-journal|] 0) excludeDirs+  let excludeDirs = [[reldir|1-in|], [reldir|2-preprocessed|]] ++ commonExcludeDirs+  findFilesIn (includeYearFilesForParent [reldir|3-journal|] 0) excludeDirs  -- | Include only files directly underneath parentDir/yearDir, e.g. 1-in/2020/* or 3-journal/2020/* includeYearFilesForParent :: RelDir -> Integer -> AbsDir -> Bool@@ -29,4 +35,33 @@ dirToStringNoSlash = init . Path.toFilePath . Path.dirname  commonExcludeDirs :: [RelDir]-commonExcludeDirs = [[Path.reldir|_manual_|], [Path.reldir|__pycache__|]]+commonExcludeDirs = [[reldir|_manual_|], [reldir|__pycache__|]]++groupIncludesUpTo :: RelDir -> [RelFile] -> InputFileBundle+groupIncludesUpTo = groupIncludesUpTo' Map.empty++groupIncludesUpTo' :: InputFileBundle -> RelDir -> [RelFile] -> InputFileBundle+groupIncludesUpTo' acc _ [] = acc+groupIncludesUpTo' acc stopAt journals = do+  let dirs = map parent journals :: [RelDir]+  let shouldStop = stopAt `elem` dirs+  if shouldStop then acc else do+    let grouped = groupIncludeFilesPerYear journals+    groupIncludesUpTo' (acc <> grouped) stopAt (Map.keys grouped)++groupIncludeFilesPerYear :: [RelFile] -> InputFileBundle+groupIncludeFilesPerYear [] = Map.empty+groupIncludeFilesPerYear ps@(p:_) = if pathSize (parent p) == 6+  then groupValuesBy initialIncludeFilePath ps+  else groupValuesBy parentIncludeFilePath ps++initialIncludeFilePath :: RelFile -> RelFile+initialIncludeFilePath p = (parent . parent . parent) p </> includeFileName p++parentIncludeFilePath :: RelFile -> RelFile+parentIncludeFilePath p = (parent . parent) p </> filename p++includeFileName :: RelFile -> RelFile+includeFileName f = do+  let includeFile = (dropTrailingPathSeparator . toFilePath . dirname . parent) f ++ "-include.journal"+  fromMaybe [relfile|unknown-include.journal|] $ parseRelFile includeFile

@@ -0,0 +1,168 @@+{-# LANGUAGE OverloadedStrings #-}++-- | Functions which currently uses TurtlePath and will be replaced with Path eventually+module Hledger.Flow.Import.ImportHelpersTurtle (+    allYearIncludeFiles+  , extractImportDirs+  , extraIncludesForFile+  , groupIncludeFiles+  , groupAndWriteIncludeFiles+  , includePreamble+  , toIncludeFiles+  , toIncludeLine+  , writeIncludesUpTo+  , writeToplevelAllYearsInclude+  , yearsIncludeMap+  )+ where++import Hledger.Flow.PathHelpers (TurtlePath)+import Hledger.Flow.DocHelpers (docURL)+import Hledger.Flow.Common (allYearsFileName, filterPaths, groupValuesBy, writeFiles, writeFiles')+import Hledger.Flow.BaseDir (relativeToBase, relativeToBase', turtleBaseDir)+import Hledger.Flow.Logging (logVerbose)++import Hledger.Flow.Types+import Hledger.Flow.Import.Types++import qualified Data.List as List (nub, sort)+import qualified Data.Text as T+import qualified Data.Map.Strict as Map++import qualified Turtle+import Turtle ((%), (</>), (<.>))++import Control.Concurrent.STM (TChan)+import Data.Maybe (fromMaybe)++extractImportDirs :: TurtlePath -> Either T.Text ImportDirs+extractImportDirs inputFile = do+  case importDirBreakdown inputFile of+    [bd,owner,bank,account,filestate,year] -> Right $ ImportDirs bd owner bank account filestate year+    _ -> do+      Left $ Turtle.format ("I couldn't find the right number of directories between \"import\" and the input file:\n"%Turtle.fp+                      %"\n\nhledger-flow expects to find input files in this structure:\n"%+                      "import/owner/bank/account/filestate/year/trxfile\n\n"%+                      "Have a look at the documentation for a detailed explanation:\n"%Turtle.s) inputFile (docURL "input-files")++importDirBreakdown ::  TurtlePath -> [TurtlePath]+importDirBreakdown = importDirBreakdown' []++importDirBreakdown' :: [TurtlePath] -> TurtlePath -> [TurtlePath]+importDirBreakdown' acc path = do+  let dir = Turtle.directory path+  if Turtle.dirname dir == "import" || (Turtle.dirname dir == "")+    then dir:acc+    else importDirBreakdown' (dir:acc) $ Turtle.parent dir++groupIncludeFiles :: [TurtlePath] -> (TurtleFileBundle, TurtleFileBundle)+groupIncludeFiles = allYearIncludeFiles . groupIncludeFilesPerYear . filter isJournalFile++isJournalFile :: TurtlePath -> Bool+isJournalFile f = Turtle.extension f == Just "journal"++allYearIncludeFiles :: TurtleFileBundle -> (TurtleFileBundle, TurtleFileBundle)+allYearIncludeFiles m = (m, yearsIncludeMap $ Map.keys m)++yearsIncludeMap :: [TurtlePath] -> TurtleFileBundle+yearsIncludeMap = groupValuesBy allYearsPath++allYearsPath :: TurtlePath -> TurtlePath+allYearsPath = allYearsPath' Turtle.directory++allYearsPath' :: (TurtlePath -> TurtlePath) -> TurtlePath -> TurtlePath+allYearsPath' dir p = dir p </> allYearsFileName++includeFileName :: TurtlePath -> TurtlePath+includeFileName = (<.> "journal") . Turtle.fromText . (Turtle.format (Turtle.fp%"-include")) . Turtle.dirname++groupIncludeFilesPerYear :: [TurtlePath] -> TurtleFileBundle+groupIncludeFilesPerYear [] = Map.empty+groupIncludeFilesPerYear ps@(p:_) = case extractImportDirs p of+    Right _ -> groupValuesBy initialIncludeFilePath ps+    Left  _ -> groupValuesBy parentIncludeFilePath ps++initialIncludeFilePath :: TurtlePath -> TurtlePath+initialIncludeFilePath p = (Turtle.parent . Turtle.parent . Turtle.parent) p </> includeFileName p++parentIncludeFilePath :: TurtlePath -> TurtlePath+parentIncludeFilePath p = (Turtle.parent . Turtle.parent) p </> Turtle.filename p++toIncludeFiles :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> TurtleFileBundle -> IO (Map.Map TurtlePath T.Text)+toIncludeFiles opts ch m = do+  preMap  <- extraIncludes opts ch (Map.keys m) ["opening.journal"] ["pre-import.journal"] []+  postMap <- extraIncludes opts ch (Map.keys m) ["closing.journal"] ["post-import.journal"] ["prices.journal"]+  return $ (addPreamble . toIncludeFiles' preMap postMap) m++toIncludeFiles' :: TurtleFileBundle -> TurtleFileBundle -> TurtleFileBundle -> Map.Map TurtlePath T.Text+toIncludeFiles' preMap postMap = Map.mapWithKey $ generatedIncludeText preMap postMap++addPreamble :: Map.Map TurtlePath T.Text -> Map.Map TurtlePath T.Text+addPreamble = Map.map (\txt -> includePreamble <> "\n" <> txt)++toIncludeLine :: TurtlePath -> TurtlePath -> T.Text+toIncludeLine base file = Turtle.format ("!include "%Turtle.fp) $ relativeToBase' base file++generatedIncludeText :: TurtleFileBundle -> TurtleFileBundle -> TurtlePath -> [TurtlePath] -> T.Text+generatedIncludeText preMap postMap outputFile fs = do+  let preFiles = fromMaybe [] $ Map.lookup outputFile preMap+  let files = List.nub . List.sort $ fs+  let postFiles = fromMaybe [] $ Map.lookup outputFile postMap+  let lns = map (toIncludeLine $ Turtle.directory outputFile) $ preFiles ++ files ++ postFiles+  T.intercalate "\n" $ lns ++ [""]++includePreamble :: T.Text+includePreamble = "### Generated by hledger-flow - DO NOT EDIT ###\n"++groupAndWriteIncludeFiles :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> [TurtlePath] -> IO [TurtlePath]+groupAndWriteIncludeFiles opts ch = writeFileMap opts ch . groupIncludeFiles++writeFileMap :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> (TurtleFileBundle, TurtleFileBundle) -> IO [TurtlePath]+writeFileMap opts ch (m, allYears) = do+  _ <- writeFiles' $ (addPreamble . toIncludeFiles' Map.empty Map.empty) allYears+  writeFiles . (toIncludeFiles opts ch) $ m++writeIncludesUpTo :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> TurtlePath -> [TurtlePath] -> IO [TurtlePath]+writeIncludesUpTo _ _ _ [] = return []+writeIncludesUpTo opts ch stopAt journalFiles = do+  let shouldStop = any (\dir -> dir == stopAt) $ map Turtle.parent journalFiles+  if shouldStop+    then return journalFiles+    else do+      newJournalFiles <- groupAndWriteIncludeFiles opts ch journalFiles+      writeIncludesUpTo opts ch stopAt newJournalFiles++writeToplevelAllYearsInclude :: (HasBaseDir o, HasVerbosity o) => o -> IO [TurtlePath]+writeToplevelAllYearsInclude opts = do+  let directivesFile = turtleBaseDir opts </> "directives" <.> "journal"+  directivesExists <- Turtle.testfile directivesFile+  let preMap = if directivesExists then Map.singleton (turtleBaseDir opts </> allYearsFileName) [directivesFile] else Map.empty+  let allTop = Map.singleton (turtleBaseDir opts </> allYearsFileName) ["import" </> allYearsFileName]+  writeFiles' $ (addPreamble . toIncludeFiles' preMap Map.empty) allTop++extraIncludes :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> [TurtlePath] -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO TurtleFileBundle+extraIncludes opts ch = extraIncludes' opts ch Map.empty++extraIncludes' :: (HasBaseDir o, HasVerbosity o) => o -> TChan LogMessage -> TurtleFileBundle -> [TurtlePath] -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO TurtleFileBundle+extraIncludes' _ _ acc [] _ _ _ = return acc+extraIncludes' opts ch acc (file:files) extraSuffixes manualFiles prices = do+  extra <- extraIncludesForFile opts ch file extraSuffixes manualFiles prices+  extraIncludes' opts ch (Map.unionWith (++) acc extra) files extraSuffixes manualFiles prices++extraIncludesForFile :: (HasVerbosity o, HasBaseDir o) => o -> TChan LogMessage -> TurtlePath -> [T.Text] -> [TurtlePath] -> [TurtlePath] -> IO TurtleFileBundle+extraIncludesForFile opts ch file extraSuffixes manualFiles prices = do+  let dirprefix = Turtle.fromText $ fst $ T.breakOn "-" $ Turtle.format Turtle.fp $ Turtle.basename file+  let fileNames = map (Turtle.fromText . Turtle.format (Turtle.fp % "-" % Turtle.s) dirprefix) extraSuffixes+  let suffixFiles = map (Turtle.directory file </>) fileNames+  let suffixDirFiles = map (Turtle.directory file </> "_manual_" </> dirprefix </>) manualFiles+  let priceFiles = map (Turtle.directory file </> ".." </> "prices" </> dirprefix </>) prices+  let extraFiles = suffixFiles ++ suffixDirFiles ++ priceFiles+  filtered <- Turtle.single $ filterPaths Turtle.testfile extraFiles+  let logMsg = Turtle.format ("Looking for possible extra include files for '"%Turtle.fp%"' among these "%Turtle.d%" options: "%Turtle.s%". Found "%Turtle.d%": "%Turtle.s)+               (relativeToBase opts file) (length extraFiles) (Turtle.repr $ relativeFilesAsText opts extraFiles)+               (length filtered) (Turtle.repr $ relativeFilesAsText opts filtered)+  logVerbose opts ch logMsg+  return $ Map.fromList [(file, filtered)]++relativeFilesAsText :: HasBaseDir o => o -> [TurtlePath] -> [T.Text]+relativeFilesAsText opts = map (Turtle.format Turtle.fp . relativeToBase opts)

@@ -1,7 +1,10 @@-module Hledger.Flow.Import.Types-where+module Hledger.Flow.Import.Types where -import Hledger.Flow.PathHelpers (TurtlePath)+import Hledger.Flow.PathHelpers (RelFile, TurtlePath)+import qualified Data.Map.Strict as Map++type TurtleFileBundle = Map.Map TurtlePath [TurtlePath]+type InputFileBundle = Map.Map RelFile [RelFile]  data ImportDirs = ImportDirs { importDir  :: TurtlePath                              , ownerDir   :: TurtlePath

@@ -0,0 +1,31 @@+{-# LANGUAGE TemplateHaskell #-}++module Hledger.Flow.Internals where++import Development.GitRev+import Data.Version (Version, showVersion)+import Paths_hledger_flow (version)++import qualified Data.Text as T+import qualified System.Info as Sys++data SystemInfo = SystemInfo { os :: String+                             , arch :: String+                             , compilerName :: String+                             , compilerVersion :: Version+                             }+                deriving (Show)++versionInfo :: T.Text+versionInfo = T.pack ("hledger-flow " ++ showVersion version ++ " " +++                       os systemInfo ++ " " ++ arch systemInfo ++ " " +++                       compilerName systemInfo ++ " " +++                       showVersion (compilerVersion systemInfo) +++                       " " ++ $(gitHash))++systemInfo :: SystemInfo+systemInfo = SystemInfo { os = Sys.os+                        , arch = Sys.arch+                        , compilerName = Sys.compilerName+                        , compilerVersion = Sys.compilerVersion+                        }

@@ -18,7 +18,7 @@  import qualified Data.Text as T import qualified Hledger.Flow.Types as FlowTypes-import Hledger.Flow.PathHelpers (TurtlePath)+import Hledger.Flow.PathHelpers (TurtlePath, pathToTurtle) import Hledger.Flow.Logging import qualified Data.List as List @@ -112,7 +112,7 @@   let relativeOutputFile = relativeToBase opts outputFile   let reportArgs = ["--file", Turtle.format Turtle.fp journal, "--period", Turtle.repr year] ++ args   let reportDisplayArgs = ["--file", Turtle.format Turtle.fp relativeJournal, "--period", Turtle.repr year] ++ args-  let hledger = Turtle.format Turtle.fp $ FlowTypes.hlPath . hledgerInfo $ opts :: T.Text+  let hledger = Turtle.format Turtle.fp $ pathToTurtle . FlowTypes.hlPath . hledgerInfo $ opts :: T.Text   let cmdLabel = Turtle.format ("hledger "%Turtle.s) $ showCmdArgs reportDisplayArgs   ((exitCode, stdOut, _), _) <- timeAndExitOnErr opts ch cmdLabel dummyLogger channelErr Turtle.procStrictWithErr (hledger, reportArgs, mempty)   if (successCheck stdOut)

@@ -4,6 +4,7 @@ import qualified Data.Text as T import Prelude hiding (putStrLn) import Hledger.Flow.Types+import Hledger.Flow.Internals (SystemInfo)  data RuntimeOptions = RuntimeOptions { baseDir :: BaseDir                                      , importRunDir :: RunDir

@@ -1,11 +1,9 @@-{-# LANGUAGE TypeSynonymInstances #-} {-# LANGUAGE FlexibleInstances #-}  module Hledger.Flow.Types where  import qualified Turtle (ExitCode, NominalDiffTime, Shell, Line) import qualified Data.Text as T-import Data.Version  import Hledger.Flow.PathHelpers @@ -19,17 +17,10 @@ type ProcFun = T.Text -> [T.Text] -> Turtle.Shell Turtle.Line -> IO FullOutput type ProcInput = (T.Text, [T.Text], Turtle.Shell Turtle.Line) -data HledgerInfo = HledgerInfo { hlPath :: TurtlePath+data HledgerInfo = HledgerInfo { hlPath :: AbsFile                                , hlVersion :: T.Text                                }                  deriving (Show)--data SystemInfo = SystemInfo { os :: String-                             , arch :: String-                             , compilerName :: String-                             , compilerVersion :: Version-                             }-                deriving (Show)  class HasVerbosity a where   verbose :: a -> Bool

@@ -0,0 +1,119 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE OverloadedLists #-}+{-# LANGUAGE QuasiQuotes #-}++module CSVImport.ImportHelperTests where++import Test.HUnit++import Hledger.Flow.Import.Types (InputFileBundle)+import Hledger.Flow.Import.ImportHelpers (groupIncludesUpTo, includeFileName)++import Path+import TestHelpers++testToJournal :: Test+testToJournal = TestCase (+  do+    let journal = toJournal [relfile|import/jane/bogartbank/investment/1-in/2020/2020-09-30.csv|]+    let expected = [relfile|import/jane/bogartbank/investment/3-journal/2020/2020-09-30.journal|]+    assertEqual "toJournal" expected journal+  )++testIncludeFileName :: Test+testIncludeFileName = TestCase (+  do+    let includeFile = includeFileName [relfile|import/jane/bogartbank/investment/3-journals/2020/2020-09-30.journal|]+    assertEqual "includeFileName" [relfile|2020-include.journal|] includeFile+  )++testGroupIncludesUpToTinySet :: Test+testGroupIncludesUpToTinySet = TestCase (+  do+    let expected = [+           ([relfile|import/jane/bogartbank/savings/2017-include.journal|], [janeSavingsJournal2017])+         , ([relfile|import/jane/bogartbank/2017-include.journal|], [[relfile|import/jane/bogartbank/savings/2017-include.journal|]])+         , ([relfile|import/jane/2017-include.journal|],            [[relfile|import/jane/bogartbank/2017-include.journal|]])+         ] :: InputFileBundle++    let grouped = groupIncludesUpTo [reldir|import/jane|] [janeSavingsJournal2017]+    assertEqual "groupIncludesUpTo: A single journal file grouping" expected grouped+  )++testGroupIncludesUpToSmallSet :: Test+testGroupIncludesUpToSmallSet = TestCase (+  do+    let expected = [+           ([relfile|import/jane/bogartbank/savings/2017-include.journal|], [janeSavingsJournal2017])+         , ([relfile|import/jane/bogartbank/savings/2018-include.journal|], janeSavingsJournals2018)+         , ([relfile|import/jane/bogartbank/savings/2019-include.journal|], [janeSavingsJournal2019])+         , ([relfile|import/jane/bogartbank/2017-include.journal|], [[relfile|import/jane/bogartbank/savings/2017-include.journal|]])+         , ([relfile|import/jane/bogartbank/2018-include.journal|], [[relfile|import/jane/bogartbank/savings/2018-include.journal|]])+         , ([relfile|import/jane/bogartbank/2019-include.journal|], [[relfile|import/jane/bogartbank/savings/2019-include.journal|]])+         , ([relfile|import/jane/2017-include.journal|],            [[relfile|import/jane/bogartbank/2017-include.journal|]])+         , ([relfile|import/jane/2018-include.journal|],            [[relfile|import/jane/bogartbank/2018-include.journal|]])+         , ([relfile|import/jane/2019-include.journal|],            [[relfile|import/jane/bogartbank/2019-include.journal|]])+         ] :: InputFileBundle++    let grouped = groupIncludesUpTo [reldir|import/jane|] janeSavingsJournals+    assertEqual "groupIncludesUpTo: A small set of journal files - same account over 3 years" expected grouped+  )++testGroupIncludesUpTo :: Test+testGroupIncludesUpTo = TestCase (+  do+    let expected = [+           ([relfile|import/john/bogartbank/savings/2017-include.journal|], johnSavingsJournals2017)+         , ([relfile|import/john/bogartbank/savings/2018-include.journal|], johnSavingsJournals2018)+         , ([relfile|import/john/bogartbank/checking/2018-include.journal|], johnCheckingJournals2018)+         , ([relfile|import/john/bogartbank/checking/2019-include.journal|], johnCheckingJournals2019)++         , ([relfile|import/john/otherbank/creditcard/2017-include.journal|], [johnCCJournal2017])+         , ([relfile|import/john/otherbank/creditcard/2018-include.journal|], [johnCCJournal2018])+         , ([relfile|import/john/otherbank/investments/2018-include.journal|], [johnInvestJournal2018])+         , ([relfile|import/john/otherbank/investments/2019-include.journal|], [johnInvestJournal2019])++         , ([relfile|import/jane/bogartbank/savings/2017-include.journal|], [janeSavingsJournal2017])+         , ([relfile|import/jane/bogartbank/savings/2018-include.journal|], janeSavingsJournals2018)+         , ([relfile|import/jane/bogartbank/savings/2019-include.journal|], [janeSavingsJournal2019])++         , ([relfile|import/jane/otherbank/creditcard/2017-include.journal|], [janeCCJournal2017])+         , ([relfile|import/jane/otherbank/creditcard/2018-include.journal|], [janeCCJournal2018])+         , ([relfile|import/jane/otherbank/investments/2018-include.journal|], [janeInvestJournal2018])+         , ([relfile|import/jane/otherbank/investments/2019-include.journal|], [janeInvestJournal2019])++         , ([relfile|import/john/bogartbank/2017-include.journal|], [[relfile|import/john/bogartbank/savings/2017-include.journal|]])+         , ([relfile|import/john/bogartbank/2018-include.journal|], [[relfile|import/john/bogartbank/checking/2018-include.journal|], [relfile|import/john/bogartbank/savings/2018-include.journal|]])+         , ([relfile|import/john/bogartbank/2019-include.journal|], [[relfile|import/john/bogartbank/checking/2019-include.journal|]])++         , ([relfile|import/john/otherbank/2017-include.journal|], [[relfile|import/john/otherbank/creditcard/2017-include.journal|]])+         , ([relfile|import/john/otherbank/2018-include.journal|], [[relfile|import/john/otherbank/creditcard/2018-include.journal|], [relfile|import/john/otherbank/investments/2018-include.journal|]])+         , ([relfile|import/john/otherbank/2019-include.journal|], [[relfile|import/john/otherbank/investments/2019-include.journal|]])++         , ([relfile|import/jane/bogartbank/2017-include.journal|], [[relfile|import/jane/bogartbank/savings/2017-include.journal|]])+         , ([relfile|import/jane/bogartbank/2018-include.journal|], [[relfile|import/jane/bogartbank/savings/2018-include.journal|]])+         , ([relfile|import/jane/bogartbank/2019-include.journal|], [[relfile|import/jane/bogartbank/savings/2019-include.journal|]])++         , ([relfile|import/jane/otherbank/2017-include.journal|], [[relfile|import/jane/otherbank/creditcard/2017-include.journal|]])+         , ([relfile|import/jane/otherbank/2018-include.journal|], [[relfile|import/jane/otherbank/creditcard/2018-include.journal|], [relfile|import/jane/otherbank/investments/2018-include.journal|]])+         , ([relfile|import/jane/otherbank/2019-include.journal|], [[relfile|import/jane/otherbank/investments/2019-include.journal|]])++         , ([relfile|import/john/2017-include.journal|],            [[relfile|import/john/bogartbank/2017-include.journal|], [relfile|import/john/otherbank/2017-include.journal|]])+         , ([relfile|import/john/2018-include.journal|],            [[relfile|import/john/bogartbank/2018-include.journal|], [relfile|import/john/otherbank/2018-include.journal|]])+         , ([relfile|import/john/2019-include.journal|],            [[relfile|import/john/bogartbank/2019-include.journal|], [relfile|import/john/otherbank/2019-include.journal|]])++         , ([relfile|import/jane/2017-include.journal|],            [[relfile|import/jane/bogartbank/2017-include.journal|], [relfile|import/jane/otherbank/2017-include.journal|]])+         , ([relfile|import/jane/2018-include.journal|],            [[relfile|import/jane/bogartbank/2018-include.journal|], [relfile|import/jane/otherbank/2018-include.journal|]])+         , ([relfile|import/jane/2019-include.journal|],            [[relfile|import/jane/bogartbank/2019-include.journal|], [relfile|import/jane/otherbank/2019-include.journal|]])++         , ([relfile|import/2017-include.journal|],            [[relfile|import/jane/2017-include.journal|], [relfile|import/john/2017-include.journal|]])+         , ([relfile|import/2018-include.journal|],            [[relfile|import/jane/2018-include.journal|], [relfile|import/john/2018-include.journal|]])+         , ([relfile|import/2019-include.journal|],            [[relfile|import/jane/2019-include.journal|], [relfile|import/john/2019-include.journal|]])+         ] :: InputFileBundle++    let grouped = groupIncludesUpTo [reldir|import|] journalFiles+    assertEqual "groupIncludesUpTo: A full set of journal files" expected grouped+  )++tests :: Test+tests = TestList [testToJournal, testIncludeFileName, testGroupIncludesUpToTinySet, testGroupIncludesUpToSmallSet, testGroupIncludesUpTo]

@@ -0,0 +1,432 @@+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE OverloadedLists #-}+{-# LANGUAGE QuasiQuotes #-}++module CSVImport.ImportHelperTurtleTests where++import Test.HUnit+import Path+import Turtle+import qualified Data.Map.Strict as Map+import Control.Concurrent.STM++import TestHelpers (defaultOpts)+import TestHelpersTurtle+import Hledger.Flow.PathHelpers (TurtlePath)+import Hledger.Flow.Common+import Hledger.Flow.Import.Types (TurtleFileBundle)+import Hledger.Flow.Import.ImportHelpersTurtle (allYearIncludeFiles, groupIncludeFiles, toIncludeFiles, toIncludeLine, yearsIncludeMap)++import Data.Either+import qualified Data.Text as T++groupedJaneBogart :: TurtleFileBundle+groupedJaneBogart = [+  ("./import/jane/bogartbank/checking/2018-include.journal",+   ["import/jane/bogartbank/checking/3-journal/2018/2018-12-30.journal"]),+  ("./import/jane/bogartbank/checking/2019-include.journal",+   ["import/jane/bogartbank/checking/3-journal/2019/2019-01-30.journal"]),+  ("./import/jane/bogartbank/savings/2017-include.journal",+   ["import/jane/bogartbank/savings/3-journal/2017/2017-12-30.journal"]),+  ("./import/jane/bogartbank/savings/2018-include.journal",+   ["import/jane/bogartbank/savings/3-journal/2018/2018-01-30.journal"])]++groupedJaneOther :: TurtleFileBundle+groupedJaneOther = [+  ("./import/jane/otherbank/creditcard/2017-include.journal",+   ["import/jane/otherbank/creditcard/3-journal/2017/2017-12-30.journal"]),+  ("./import/jane/otherbank/creditcard/2018-include.journal",+   ["import/jane/otherbank/creditcard/3-journal/2018/2018-01-30.journal"]),+  ("./import/jane/otherbank/investments/2018-include.journal",+   ["import/jane/otherbank/investments/3-journal/2018/2018-12-30.journal"]),+  ("./import/jane/otherbank/investments/2019-include.journal",+   ["import/jane/otherbank/investments/3-journal/2019/2019-01-30.journal"])]++groupedJohnBogart :: TurtleFileBundle+groupedJohnBogart = [+  ("./import/john/bogartbank/checking/2018-include.journal",+   ["import/john/bogartbank/checking/3-journal/2018/2018-11-30.journal",+    "import/john/bogartbank/checking/3-journal/2018/2018-10-30.journal",+    "import/john/bogartbank/checking/3-journal/2018/2018-12-30.journal"]),+  ("./import/john/bogartbank/checking/2019-include.journal",+   ["import/john/bogartbank/checking/3-journal/2019/2019-01-30.journal",+    "import/john/bogartbank/checking/3-journal/2019/2019-02-30.journal"]),+  ("./import/john/bogartbank/savings/2017-include.journal",+   ["import/john/bogartbank/savings/3-journal/2017/2017-11-30.journal",+    "import/john/bogartbank/savings/3-journal/2017/2017-12-30.journal"]),+  ("./import/john/bogartbank/savings/2018-include.journal",+   ["import/john/bogartbank/savings/3-journal/2018/2018-02-30.journal",+    "import/john/bogartbank/savings/3-journal/2018/2018-01-30.journal"])]++groupedJohnOther :: TurtleFileBundle+groupedJohnOther = [+  ("./import/john/otherbank/creditcard/2017-include.journal",+   ["import/john/otherbank/creditcard/3-journal/2017/2017-12-30.journal"]),+  ("./import/john/otherbank/creditcard/2018-include.journal",+   ["import/john/otherbank/creditcard/3-journal/2018/2018-01-30.journal"]),+  ("./import/john/otherbank/investments/2018-include.journal",+   ["import/john/otherbank/investments/3-journal/2018/2018-12-30.journal"]),+  ("./import/john/otherbank/investments/2019-include.journal",+   ["import/john/otherbank/investments/3-journal/2019/2019-01-30.journal"])]++groupedIncludeFiles :: TurtleFileBundle+groupedIncludeFiles = groupedJaneBogart <> groupedJaneOther <>+                      groupedJohnBogart <> groupedJohnOther++testYearsIncludeMap :: Test+testYearsIncludeMap = TestCase (+  do+    let maps = allYearIncludeFiles groupedJohnOther+    let yearGrouping = [("./import/john/otherbank/creditcard/all-years.journal",+                         ["./import/john/otherbank/creditcard/2017-include.journal",+                          "./import/john/otherbank/creditcard/2018-include.journal"]),+                        ("./import/john/otherbank/investments/all-years.journal",+                         ["./import/john/otherbank/investments/2018-include.journal",+                          "./import/john/otherbank/investments/2019-include.journal"])]+    let expected = (groupedJohnOther, yearGrouping)+    assertEqual "An augmented map with grouped years per level added" expected maps+  )++testYearsIncludeGrouping :: Test+testYearsIncludeGrouping = TestCase (+  do+    let yearsMap = yearsIncludeMap (Map.keys groupedJohnOther)+    let expected = [("./import/john/otherbank/creditcard/all-years.journal",+                     ["./import/john/otherbank/creditcard/2017-include.journal",+                      "./import/john/otherbank/creditcard/2018-include.journal"]),+                    ("./import/john/otherbank/investments/all-years.journal",+                     ["./import/john/otherbank/investments/2018-include.journal",+                      "./import/john/otherbank/investments/2019-include.journal"])]+    assertEqual "A basic map with grouped years per level" expected yearsMap+  )++testGroupIncludeFilesTinySet :: Test+testGroupIncludeFilesTinySet = TestCase (+  do+    let journals1 = [   "import/jane/bogartbank/savings/3-journals/2017/2017-12-30.journal"]+    let expected1 = [("./import/jane/bogartbank/savings/2017-include.journal", journals1)] :: TurtleFileBundle+    let expectedAllYears1 = [("./import/jane/bogartbank/savings/all-years.journal", ["./import/jane/bogartbank/savings/2017-include.journal"])]+    let (group1, allYears1) = groupIncludeFiles journals1+    assertEqual "groupIncludeFiles small allYears 1" expectedAllYears1 allYears1+    assertEqual "groupIncludeFiles small set 1" expected1 group1++    let journals2 = [(fst . head . Map.toList) expected1] :: [TurtlePath]+    let expected2 = [("./import/jane/bogartbank/2017-include.journal", journals2)] :: TurtleFileBundle+    let expectedAllYears2 = [("./import/jane/bogartbank/all-years.journal", ["./import/jane/bogartbank/2017-include.journal"])]+    let (group2, allYears2) = groupIncludeFiles journals2+    assertEqual "groupIncludeFiles small allYears 2" expectedAllYears2 allYears2+    assertEqual "groupIncludeFiles small set 2" expected2 group2++    let journals3 = [(fst . head . Map.toList) expected2] :: [TurtlePath]+    let expected3 = [("./import/jane/2017-include.journal", journals3)] :: TurtleFileBundle+    let expectedAllYears3 = [("./import/jane/all-years.journal", ["./import/jane/2017-include.journal"])]+    let (group3, allYears3) = groupIncludeFiles journals3+    assertEqual "groupIncludeFiles small allYears 3" expectedAllYears3 allYears3+    assertEqual "groupIncludeFiles small set 3" expected3 group3+  )++testGroupIncludeFilesSmallSet :: Test+testGroupIncludeFilesSmallSet = TestCase (+  do+    let (group1, allYears1) = groupIncludeFiles (toJournals inputJaneBogart)+    let expectedAllYears1 = [+          ("./import/jane/bogartbank/checking/all-years.journal",+           ["./import/jane/bogartbank/checking/2018-include.journal",+            "./import/jane/bogartbank/checking/2019-include.journal"]),+          ("./import/jane/bogartbank/savings/all-years.journal",+           ["./import/jane/bogartbank/savings/2017-include.journal",+            "./import/jane/bogartbank/savings/2018-include.journal"])]++    assertEqual "groupIncludeFiles Jane AllYears 1" expectedAllYears1 allYears1+    assertEqual "groupIncludeFiles Jane 1" groupedJaneBogart group1++    let (group2, allYears2) = groupIncludeFiles (Map.keys group1)+    let expectedAllYears2 = [+          ("./import/jane/bogartbank/all-years.journal",+           ["./import/jane/bogartbank/2017-include.journal",+            "./import/jane/bogartbank/2018-include.journal",+            "./import/jane/bogartbank/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane AllYears 2" expectedAllYears2 allYears2+    let expectedGroup2 = [+          ("./import/jane/bogartbank/2017-include.journal",+           ["./import/jane/bogartbank/savings/2017-include.journal"]),+          ("./import/jane/bogartbank/2018-include.journal",+           ["./import/jane/bogartbank/checking/2018-include.journal",+            "./import/jane/bogartbank/savings/2018-include.journal"]),+          ("./import/jane/bogartbank/2019-include.journal",+           ["./import/jane/bogartbank/checking/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane 2" expectedGroup2 group2++    let (group3, allYears3) = groupIncludeFiles (Map.keys group2)+    let expectedAllYears3 = [+          ("./import/jane/all-years.journal",+           ["./import/jane/2017-include.journal",+            "./import/jane/2018-include.journal",+            "./import/jane/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane AllYears 3" expectedAllYears3 allYears3+    let expectedGroup3 = [+          ("./import/jane/2017-include.journal",+           ["./import/jane/bogartbank/2017-include.journal"]),+          ("./import/jane/2018-include.journal",+           ["./import/jane/bogartbank/2018-include.journal"]),+          ("./import/jane/2019-include.journal",+           ["./import/jane/bogartbank/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane 3" expectedGroup3 group3++    let (group4, allYears4) = groupIncludeFiles (Map.keys group3)+    let expectedAllYears4 = [+          ("./import/all-years.journal",+           ["./import/2017-include.journal",+            "./import/2018-include.journal",+            "./import/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane AllYears 4" expectedAllYears4 allYears4+    let expectedGroup4 = [+          ("./import/2017-include.journal",+           ["./import/jane/2017-include.journal"]),+          ("./import/2018-include.journal",+           ["./import/jane/2018-include.journal"]),+          ("./import/2019-include.journal",+           ["./import/jane/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane 4" expectedGroup4 group4++    let (group5, allYears5) = groupIncludeFiles (Map.keys group4)+    let expectedAllYears5 = [("./all-years.journal", ["./2017-include.journal", "./2018-include.journal", "./2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane AllYears 5" expectedAllYears5 allYears5+    let expectedGroup5 = [+          ("./2017-include.journal",+           ["./import/2017-include.journal"]),+          ("./2018-include.journal",+           ["./import/2018-include.journal"]),+          ("./2019-include.journal",+           ["./import/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane 5" expectedGroup5 group5+ )++testGroupIncludeFiles :: Test+testGroupIncludeFiles = TestCase (+  do+    let (group1, allYears1) = groupIncludeFiles journalFiles+    let expectedAllYears1 = [+          ("./import/jane/bogartbank/checking/all-years.journal",+            ["./import/jane/bogartbank/checking/2018-include.journal",+             "./import/jane/bogartbank/checking/2019-include.journal"]),+          ("./import/jane/bogartbank/savings/all-years.journal",+            ["./import/jane/bogartbank/savings/2017-include.journal",+             "./import/jane/bogartbank/savings/2018-include.journal"]),+          ("./import/jane/otherbank/creditcard/all-years.journal",+            ["./import/jane/otherbank/creditcard/2017-include.journal",+             "./import/jane/otherbank/creditcard/2018-include.journal"]),+          ("./import/jane/otherbank/investments/all-years.journal",+            ["./import/jane/otherbank/investments/2018-include.journal",+             "./import/jane/otherbank/investments/2019-include.journal"]),+          ("./import/john/bogartbank/checking/all-years.journal",+            ["./import/john/bogartbank/checking/2018-include.journal",+             "./import/john/bogartbank/checking/2019-include.journal"]),+          ("./import/john/bogartbank/savings/all-years.journal",+            ["./import/john/bogartbank/savings/2017-include.journal",+             "./import/john/bogartbank/savings/2018-include.journal"]),+          ("./import/john/otherbank/creditcard/all-years.journal",+            ["./import/john/otherbank/creditcard/2017-include.journal",+             "./import/john/otherbank/creditcard/2018-include.journal"]),+          ("./import/john/otherbank/investments/all-years.journal",+            ["./import/john/otherbank/investments/2018-include.journal",+             "./import/john/otherbank/investments/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane AllYears 1" expectedAllYears1 allYears1+    assertEqual "groupIncludeFiles 1" groupedIncludeFiles group1++    let (group2, allYears2) = groupIncludeFiles (Map.keys group1)+    let expectedAllYears2 = [+          ("./import/jane/bogartbank/all-years.journal",+            ["./import/jane/bogartbank/2017-include.journal",+             "./import/jane/bogartbank/2018-include.journal",+             "./import/jane/bogartbank/2019-include.journal"]),+          ("./import/jane/otherbank/all-years.journal",+            ["./import/jane/otherbank/2017-include.journal",+             "./import/jane/otherbank/2018-include.journal",+             "./import/jane/otherbank/2019-include.journal"]),+          ("./import/john/bogartbank/all-years.journal",+            ["./import/john/bogartbank/2017-include.journal",+             "./import/john/bogartbank/2018-include.journal",+             "./import/john/bogartbank/2019-include.journal"]),+          ("./import/john/otherbank/all-years.journal",+            ["./import/john/otherbank/2017-include.journal",+             "./import/john/otherbank/2018-include.journal",+             "./import/john/otherbank/2019-include.journal"])]++    assertEqual "groupIncludeFiles Jane AllYears 2" expectedAllYears2 allYears2+    let expectedGroup2 = [("./import/jane/bogartbank/2017-include.journal",+                           ["./import/jane/bogartbank/savings/2017-include.journal"]),+                          ("./import/jane/bogartbank/2018-include.journal",+                           ["./import/jane/bogartbank/checking/2018-include.journal",+                            "./import/jane/bogartbank/savings/2018-include.journal"]),+                          ("./import/jane/bogartbank/2019-include.journal",+                           ["./import/jane/bogartbank/checking/2019-include.journal"]),++                          ("./import/jane/otherbank/2017-include.journal",+                           ["./import/jane/otherbank/creditcard/2017-include.journal"]),+                          ("./import/jane/otherbank/2018-include.journal",+                           ["./import/jane/otherbank/creditcard/2018-include.journal",+                            "./import/jane/otherbank/investments/2018-include.journal"]),+                          ("./import/jane/otherbank/2019-include.journal",+                           ["./import/jane/otherbank/investments/2019-include.journal"]),++                          ("./import/john/bogartbank/2017-include.journal",+                           ["./import/john/bogartbank/savings/2017-include.journal"]),+                          ("./import/john/bogartbank/2018-include.journal",+                           ["./import/john/bogartbank/checking/2018-include.journal",+                            "./import/john/bogartbank/savings/2018-include.journal"]),+                          ("./import/john/bogartbank/2019-include.journal",+                           ["./import/john/bogartbank/checking/2019-include.journal"]),++                          ("./import/john/otherbank/2017-include.journal",+                           ["./import/john/otherbank/creditcard/2017-include.journal"]),+                          ("./import/john/otherbank/2018-include.journal",+                           ["./import/john/otherbank/creditcard/2018-include.journal",+                            "./import/john/otherbank/investments/2018-include.journal"]),+                          ("./import/john/otherbank/2019-include.journal",+                           ["./import/john/otherbank/investments/2019-include.journal"])]+    assertEqual "groupIncludeFiles 2 - diff 1" [] (expectedGroup2 Map.\\ group2)+    assertEqual "groupIncludeFiles 2 - diff 2" [] (group2 Map.\\ expectedGroup2)+    assertEqual "groupIncludeFiles 2" expectedGroup2 group2++    let (group3, allYears3) = groupIncludeFiles (Map.keys group2)+    let expectedAllYears3 = [+          ("./import/jane/all-years.journal",+            ["./import/jane/2017-include.journal",+             "./import/jane/2018-include.journal",+             "./import/jane/2019-include.journal"]),+          ("./import/john/all-years.journal",+            ["./import/john/2017-include.journal",+             "./import/john/2018-include.journal",+             "./import/john/2019-include.journal"])]++    assertEqual "groupIncludeFiles Jane AllYears 3" expectedAllYears3 allYears3+    let expectedGroup3 = [("./import/jane/2017-include.journal",+                           ["./import/jane/bogartbank/2017-include.journal",+                            "./import/jane/otherbank/2017-include.journal"]),+                          ("./import/jane/2018-include.journal",+                           ["./import/jane/bogartbank/2018-include.journal",+                            "./import/jane/otherbank/2018-include.journal"]),+                          ("./import/jane/2019-include.journal",+                           ["./import/jane/bogartbank/2019-include.journal",+                            "./import/jane/otherbank/2019-include.journal"]),+                          ("./import/john/2017-include.journal",+                           ["./import/john/bogartbank/2017-include.journal",+                            "./import/john/otherbank/2017-include.journal"]),+                          ("./import/john/2018-include.journal",+                           ["./import/john/bogartbank/2018-include.journal",+                            "./import/john/otherbank/2018-include.journal"]),+                          ("./import/john/2019-include.journal",+                           ["./import/john/bogartbank/2019-include.journal",+                            "./import/john/otherbank/2019-include.journal"])]+    assertEqual "groupIncludeFiles 3 - diff 1" [] (expectedGroup3 Map.\\ group3)+    assertEqual "groupIncludeFiles 3 - diff 2" [] (group3 Map.\\ expectedGroup3)+    assertEqual "groupIncludeFiles 3" expectedGroup3 group3++    let (group4, allYears4) = groupIncludeFiles (Map.keys group3)+    let expectedAllYears4 = [+          ("./import/all-years.journal",+            ["./import/2017-include.journal",+             "./import/2018-include.journal",+             "./import/2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane AllYears 4" expectedAllYears4 allYears4+    let expectedGroup4 = [("./import/2017-include.journal",+                           ["./import/jane/2017-include.journal",+                            "./import/john/2017-include.journal"]),+                          ("./import/2018-include.journal",+                           ["./import/jane/2018-include.journal",+                            "./import/john/2018-include.journal"]),+                          ("./import/2019-include.journal",+                           ["./import/jane/2019-include.journal",+                            "./import/john/2019-include.journal"])]+    assertEqual "groupIncludeFiles 4 - diff 1" [] (expectedGroup4 Map.\\ group4)+    assertEqual "groupIncludeFiles 4 - diff 2" [] (group4 Map.\\ expectedGroup4)+    assertEqual "groupIncludeFiles 4" expectedGroup4 group4++    let (group5, allYears5) = groupIncludeFiles (Map.keys group4)+    let expectedAllYears5 = [("./all-years.journal", ["./2017-include.journal", "./2018-include.journal", "./2019-include.journal"])]+    assertEqual "groupIncludeFiles Jane AllYears 5" expectedAllYears5 allYears5+    let expectedGroup5 = [+          ("./2017-include.journal",+           ["./import/2017-include.journal"]),+          ("./2018-include.journal",+           ["./import/2018-include.journal"]),+          ("./2019-include.journal",+           ["./import/2019-include.journal"])]+    assertEqual "groupIncludeFiles 5 - diff 1" [] (expectedGroup5 Map.\\ group5)+    assertEqual "groupIncludeFiles 5 - diff 2" [] (group5 Map.\\ expectedGroup5)+    assertEqual "groupIncludeFiles 5" expectedGroup5 group5+  )++testIncludeYears :: Test+testIncludeYears = TestCase (+  do+    let txterr = "Some text without years"+    let expectederr = ["Unable to extract years from the following text:", txterr, "Errors:"]+    let actualerr = (init . head) $ map (T.lines) $ lefts [includeYears' txterr] :: [Text]+    assertEqual "Get a list of years from an include file - error case" expectederr actualerr++    let txt1 = "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>+          "!include import/2014-include.journal\n" <>+          "!include import/2015-include.journal\n" <>+          "!include import/2016-include.journal\n" <>+          "!include import/2017-include.journal\n" <>+          "!include import/2018-include.journal\n" <>+          "!include import/2019-include.journal"++    let expected1 = Right [2014..2019]+    let actual1 = includeYears' txt1+    assertEqual "Get a list of years from an include file - success case 1" expected1 actual1++    let txt2 = "!include 2019-include.journal"++    let expected2 = Right [2019]+    let actual2 = includeYears' txt2+    assertEqual "Get a list of years from an include file - success case 2" expected2 actual2+  )++testToIncludeLine :: Test+testToIncludeLine = TestCase (+  do+    let expected = "!include file1.journal"+    let relativeWithTrailingSlash = toIncludeLine "./base/dir/" "./base/dir/file1.journal"+    assertEqual "Include line - relative base dir with trailing slash" expected relativeWithTrailingSlash++    let relativeNoTrailingSlash = toIncludeLine "./base/dir" "./base/dir/file1.journal"+    assertEqual "Include line - relative base dir without a trailing slash" expected relativeNoTrailingSlash++    let absoluteWithTrailingSlash = toIncludeLine "/base/dir/" "/base/dir/file1.journal"+    assertEqual "Include line - absolute base dir with trailing slash" expected absoluteWithTrailingSlash++    let absoluteNoTrailingSlash = toIncludeLine "/base/dir" "/base/dir/file1.journal"+    assertEqual "Include line - absolute base dir without a trailing slash" expected absoluteNoTrailingSlash+  )++testToIncludeFiles :: Test+testToIncludeFiles = TestCase (+  do+    let expected = [+          ("./import/john/bogartbank/checking/2018-include.journal",+           "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>+           "!include import/john/bogartbank/checking/3-journal/2018/2018-10-30.journal\n" <>+           "!include import/john/bogartbank/checking/3-journal/2018/2018-11-30.journal\n" <>+           "!include import/john/bogartbank/checking/3-journal/2018/2018-12-30.journal\n"),+          ("./import/john/bogartbank/checking/2019-include.journal",+           "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>+           "!include import/john/bogartbank/checking/3-journal/2019/2019-01-30.journal\n" <>+           "!include import/john/bogartbank/checking/3-journal/2019/2019-02-30.journal\n"),+          ("./import/john/bogartbank/savings/2017-include.journal",+           "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>+           "!include import/john/bogartbank/savings/3-journal/2017/2017-11-30.journal\n" <>+           "!include import/john/bogartbank/savings/3-journal/2017/2017-12-30.journal\n"),+          ("./import/john/bogartbank/savings/2018-include.journal",+           "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>+           "!include import/john/bogartbank/savings/3-journal/2018/2018-01-30.journal\n" <>+           "!include import/john/bogartbank/savings/3-journal/2018/2018-02-30.journal\n")]++    ch <- newTChanIO+    txt <- toIncludeFiles (defaultOpts [absdir|/|]) ch groupedJohnBogart+    assertEqual "Convert a grouped map of paths, to a map with text contents for each file" expected txt)++tests :: Test+tests = TestList [testYearsIncludeMap, testYearsIncludeGrouping, testGroupIncludeFilesTinySet, testGroupIncludeFilesSmallSet, testGroupIncludeFiles, testIncludeYears, testToIncludeLine, testToIncludeFiles]

@@ -8,9 +8,12 @@ import qualified Data.Map.Strict as Map import qualified Data.List as List (sort) -import TestHelpers+import TestHelpers (defaultOpts)+import TestHelpersTurtle (journalFiles, hiddenFiles, touchAll, extraFiles) import Hledger.Flow.Common+import Hledger.Flow.Import.ImportHelpersTurtle (extraIncludesForFile, groupAndWriteIncludeFiles, includePreamble, toIncludeFiles) import Hledger.Flow.PathHelpers+ import Control.Concurrent.STM  testExtraIncludesForFile :: Test

@@ -1,429 +1,9 @@-{-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE OverloadedLists #-}-{-# LANGUAGE QuasiQuotes #-}- module CSVImport.Unit where  import Test.HUnit-import Path-import Turtle-import qualified Data.Map.Strict as Map-import Control.Concurrent.STM -import TestHelpers-import Hledger.Flow.PathHelpers (TurtlePath)-import Hledger.Flow.Common--import Data.Either-import qualified Data.Text as T--groupedJaneBogart :: Map.Map TurtlePath [TurtlePath]-groupedJaneBogart = [-  ("./import/jane/bogartbank/checking/2018-include.journal",-   ["import/jane/bogartbank/checking/3-journal/2018/2018-12-30.journal"]),-  ("./import/jane/bogartbank/checking/2019-include.journal",-   ["import/jane/bogartbank/checking/3-journal/2019/2019-01-30.journal"]),-  ("./import/jane/bogartbank/savings/2017-include.journal",-   ["import/jane/bogartbank/savings/3-journal/2017/2017-12-30.journal"]),-  ("./import/jane/bogartbank/savings/2018-include.journal",-   ["import/jane/bogartbank/savings/3-journal/2018/2018-01-30.journal"])]--groupedJaneOther :: Map.Map TurtlePath [TurtlePath]-groupedJaneOther = [-  ("./import/jane/otherbank/creditcard/2017-include.journal",-   ["import/jane/otherbank/creditcard/3-journal/2017/2017-12-30.journal"]),-  ("./import/jane/otherbank/creditcard/2018-include.journal",-   ["import/jane/otherbank/creditcard/3-journal/2018/2018-01-30.journal"]),-  ("./import/jane/otherbank/investments/2018-include.journal",-   ["import/jane/otherbank/investments/3-journal/2018/2018-12-30.journal"]),-  ("./import/jane/otherbank/investments/2019-include.journal",-   ["import/jane/otherbank/investments/3-journal/2019/2019-01-30.journal"])]--groupedJohnBogart :: Map.Map TurtlePath [TurtlePath]-groupedJohnBogart = [-  ("./import/john/bogartbank/checking/2018-include.journal",-   ["import/john/bogartbank/checking/3-journal/2018/2018-11-30.journal",-    "import/john/bogartbank/checking/3-journal/2018/2018-10-30.journal",-    "import/john/bogartbank/checking/3-journal/2018/2018-12-30.journal"]),-  ("./import/john/bogartbank/checking/2019-include.journal",-   ["import/john/bogartbank/checking/3-journal/2019/2019-01-30.journal",-    "import/john/bogartbank/checking/3-journal/2019/2019-02-30.journal"]),-  ("./import/john/bogartbank/savings/2017-include.journal",-   ["import/john/bogartbank/savings/3-journal/2017/2017-11-30.journal",-    "import/john/bogartbank/savings/3-journal/2017/2017-12-30.journal"]),-  ("./import/john/bogartbank/savings/2018-include.journal",-   ["import/john/bogartbank/savings/3-journal/2018/2018-02-30.journal",-    "import/john/bogartbank/savings/3-journal/2018/2018-01-30.journal"])]--groupedJohnOther :: Map.Map TurtlePath [TurtlePath]-groupedJohnOther = [-  ("./import/john/otherbank/creditcard/2017-include.journal",-   ["import/john/otherbank/creditcard/3-journal/2017/2017-12-30.journal"]),-  ("./import/john/otherbank/creditcard/2018-include.journal",-   ["import/john/otherbank/creditcard/3-journal/2018/2018-01-30.journal"]),-  ("./import/john/otherbank/investments/2018-include.journal",-   ["import/john/otherbank/investments/3-journal/2018/2018-12-30.journal"]),-  ("./import/john/otherbank/investments/2019-include.journal",-   ["import/john/otherbank/investments/3-journal/2019/2019-01-30.journal"])]--groupedIncludeFiles :: Map.Map TurtlePath [TurtlePath]-groupedIncludeFiles = groupedJaneBogart <> groupedJaneOther <>-                      groupedJohnBogart <> groupedJohnOther--testYearsIncludeMap :: Test-testYearsIncludeMap = TestCase (-  do-    let maps = allYearIncludeFiles groupedJohnOther-    let yearGrouping = [("./import/john/otherbank/creditcard/all-years.journal",-                         ["./import/john/otherbank/creditcard/2017-include.journal",-                          "./import/john/otherbank/creditcard/2018-include.journal"]),-                        ("./import/john/otherbank/investments/all-years.journal",-                         ["./import/john/otherbank/investments/2018-include.journal",-                          "./import/john/otherbank/investments/2019-include.journal"])]-    let expected = (groupedJohnOther, yearGrouping)-    assertEqual "An augmented map with grouped years per level added" expected maps-  )--testYearsIncludeGrouping :: Test-testYearsIncludeGrouping = TestCase (-  do-    let yearsMap = yearsIncludeMap (Map.keys groupedJohnOther)-    let expected = [("./import/john/otherbank/creditcard/all-years.journal",-                     ["./import/john/otherbank/creditcard/2017-include.journal",-                      "./import/john/otherbank/creditcard/2018-include.journal"]),-                    ("./import/john/otherbank/investments/all-years.journal",-                     ["./import/john/otherbank/investments/2018-include.journal",-                      "./import/john/otherbank/investments/2019-include.journal"])]-    assertEqual "A basic map with grouped years per level" expected yearsMap-  )--testGroupIncludeFilesTinySet :: Test-testGroupIncludeFilesTinySet = TestCase (-  do-    let journals1 = [   "import/jane/bogartbank/savings/3-journals/2017/2017-12-30.journal"]-    let expected1 = [("./import/jane/bogartbank/savings/2017-include.journal", journals1)] :: Map.Map TurtlePath [TurtlePath]-    let expectedAllYears1 = [("./import/jane/bogartbank/savings/all-years.journal", ["./import/jane/bogartbank/savings/2017-include.journal"])]-    let (group1, allYears1) = groupIncludeFiles journals1-    assertEqual "groupIncludeFiles small allYears 1" expectedAllYears1 allYears1-    assertEqual "groupIncludeFiles small set 1" expected1 group1--    let journals2 = [(fst . head . Map.toList) expected1] :: [TurtlePath]-    let expected2 = [("./import/jane/bogartbank/2017-include.journal", journals2)] :: Map.Map TurtlePath [TurtlePath]-    let expectedAllYears2 = [("./import/jane/bogartbank/all-years.journal", ["./import/jane/bogartbank/2017-include.journal"])]-    let (group2, allYears2) = groupIncludeFiles journals2-    assertEqual "groupIncludeFiles small allYears 2" expectedAllYears2 allYears2-    assertEqual "groupIncludeFiles small set 2" expected2 group2--    let journals3 = [(fst . head . Map.toList) expected2] :: [TurtlePath]-    let expected3 = [("./import/jane/2017-include.journal", journals3)] :: Map.Map TurtlePath [TurtlePath]-    let expectedAllYears3 = [("./import/jane/all-years.journal", ["./import/jane/2017-include.journal"])]-    let (group3, allYears3) = groupIncludeFiles journals3-    assertEqual "groupIncludeFiles small allYears 3" expectedAllYears3 allYears3-    assertEqual "groupIncludeFiles small set 3" expected3 group3-  )--testGroupIncludeFilesSmallSet :: Test-testGroupIncludeFilesSmallSet = TestCase (-  do-    let (group1, allYears1) = groupIncludeFiles (toJournals inputJaneBogart)-    let expectedAllYears1 = [-          ("./import/jane/bogartbank/checking/all-years.journal",-           ["./import/jane/bogartbank/checking/2018-include.journal",-            "./import/jane/bogartbank/checking/2019-include.journal"]),-          ("./import/jane/bogartbank/savings/all-years.journal",-           ["./import/jane/bogartbank/savings/2017-include.journal",-            "./import/jane/bogartbank/savings/2018-include.journal"])]--    assertEqual "groupIncludeFiles Jane AllYears 1" expectedAllYears1 allYears1-    assertEqual "groupIncludeFiles Jane 1" groupedJaneBogart group1--    let (group2, allYears2) = groupIncludeFiles (Map.keys group1)-    let expectedAllYears2 = [-          ("./import/jane/bogartbank/all-years.journal",-           ["./import/jane/bogartbank/2017-include.journal",-            "./import/jane/bogartbank/2018-include.journal",-            "./import/jane/bogartbank/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane AllYears 2" expectedAllYears2 allYears2-    let expectedGroup2 = [-          ("./import/jane/bogartbank/2017-include.journal",-           ["./import/jane/bogartbank/savings/2017-include.journal"]),-          ("./import/jane/bogartbank/2018-include.journal",-           ["./import/jane/bogartbank/checking/2018-include.journal",-            "./import/jane/bogartbank/savings/2018-include.journal"]),-          ("./import/jane/bogartbank/2019-include.journal",-           ["./import/jane/bogartbank/checking/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane 2" expectedGroup2 group2--    let (group3, allYears3) = groupIncludeFiles (Map.keys group2)-    let expectedAllYears3 = [-          ("./import/jane/all-years.journal",-           ["./import/jane/2017-include.journal",-            "./import/jane/2018-include.journal",-            "./import/jane/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane AllYears 3" expectedAllYears3 allYears3-    let expectedGroup3 = [-          ("./import/jane/2017-include.journal",-           ["./import/jane/bogartbank/2017-include.journal"]),-          ("./import/jane/2018-include.journal",-           ["./import/jane/bogartbank/2018-include.journal"]),-          ("./import/jane/2019-include.journal",-           ["./import/jane/bogartbank/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane 3" expectedGroup3 group3--    let (group4, allYears4) = groupIncludeFiles (Map.keys group3)-    let expectedAllYears4 = [-          ("./import/all-years.journal",-           ["./import/2017-include.journal",-            "./import/2018-include.journal",-            "./import/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane AllYears 4" expectedAllYears4 allYears4-    let expectedGroup4 = [-          ("./import/2017-include.journal",-           ["./import/jane/2017-include.journal"]),-          ("./import/2018-include.journal",-           ["./import/jane/2018-include.journal"]),-          ("./import/2019-include.journal",-           ["./import/jane/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane 4" expectedGroup4 group4--    let (group5, allYears5) = groupIncludeFiles (Map.keys group4)-    let expectedAllYears5 = [("./all-years.journal", ["./2017-include.journal", "./2018-include.journal", "./2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane AllYears 5" expectedAllYears5 allYears5-    let expectedGroup5 = [-          ("./2017-include.journal",-           ["./import/2017-include.journal"]),-          ("./2018-include.journal",-           ["./import/2018-include.journal"]),-          ("./2019-include.journal",-           ["./import/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane 5" expectedGroup5 group5- )--testGroupIncludeFiles :: Test-testGroupIncludeFiles = TestCase (-  do-    let (group1, allYears1) = groupIncludeFiles journalFiles-    let expectedAllYears1 = [-          ("./import/jane/bogartbank/checking/all-years.journal",-            ["./import/jane/bogartbank/checking/2018-include.journal",-             "./import/jane/bogartbank/checking/2019-include.journal"]),-          ("./import/jane/bogartbank/savings/all-years.journal",-            ["./import/jane/bogartbank/savings/2017-include.journal",-             "./import/jane/bogartbank/savings/2018-include.journal"]),-          ("./import/jane/otherbank/creditcard/all-years.journal",-            ["./import/jane/otherbank/creditcard/2017-include.journal",-             "./import/jane/otherbank/creditcard/2018-include.journal"]),-          ("./import/jane/otherbank/investments/all-years.journal",-            ["./import/jane/otherbank/investments/2018-include.journal",-             "./import/jane/otherbank/investments/2019-include.journal"]),-          ("./import/john/bogartbank/checking/all-years.journal",-            ["./import/john/bogartbank/checking/2018-include.journal",-             "./import/john/bogartbank/checking/2019-include.journal"]),-          ("./import/john/bogartbank/savings/all-years.journal",-            ["./import/john/bogartbank/savings/2017-include.journal",-             "./import/john/bogartbank/savings/2018-include.journal"]),-          ("./import/john/otherbank/creditcard/all-years.journal",-            ["./import/john/otherbank/creditcard/2017-include.journal",-             "./import/john/otherbank/creditcard/2018-include.journal"]),-          ("./import/john/otherbank/investments/all-years.journal",-            ["./import/john/otherbank/investments/2018-include.journal",-             "./import/john/otherbank/investments/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane AllYears 1" expectedAllYears1 allYears1-    assertEqual "groupIncludeFiles 1" groupedIncludeFiles group1--    let (group2, allYears2) = groupIncludeFiles (Map.keys group1)-    let expectedAllYears2 = [-          ("./import/jane/bogartbank/all-years.journal",-            ["./import/jane/bogartbank/2017-include.journal",-             "./import/jane/bogartbank/2018-include.journal",-             "./import/jane/bogartbank/2019-include.journal"]),-          ("./import/jane/otherbank/all-years.journal",-            ["./import/jane/otherbank/2017-include.journal",-             "./import/jane/otherbank/2018-include.journal",-             "./import/jane/otherbank/2019-include.journal"]),-          ("./import/john/bogartbank/all-years.journal",-            ["./import/john/bogartbank/2017-include.journal",-             "./import/john/bogartbank/2018-include.journal",-             "./import/john/bogartbank/2019-include.journal"]),-          ("./import/john/otherbank/all-years.journal",-            ["./import/john/otherbank/2017-include.journal",-             "./import/john/otherbank/2018-include.journal",-             "./import/john/otherbank/2019-include.journal"])]--    assertEqual "groupIncludeFiles Jane AllYears 2" expectedAllYears2 allYears2-    let expectedGroup2 = [("./import/jane/bogartbank/2017-include.journal",-                           ["./import/jane/bogartbank/savings/2017-include.journal"]),-                          ("./import/jane/bogartbank/2018-include.journal",-                           ["./import/jane/bogartbank/checking/2018-include.journal",-                            "./import/jane/bogartbank/savings/2018-include.journal"]),-                          ("./import/jane/bogartbank/2019-include.journal",-                           ["./import/jane/bogartbank/checking/2019-include.journal"]),--                          ("./import/jane/otherbank/2017-include.journal",-                           ["./import/jane/otherbank/creditcard/2017-include.journal"]),-                          ("./import/jane/otherbank/2018-include.journal",-                           ["./import/jane/otherbank/creditcard/2018-include.journal",-                            "./import/jane/otherbank/investments/2018-include.journal"]),-                          ("./import/jane/otherbank/2019-include.journal",-                           ["./import/jane/otherbank/investments/2019-include.journal"]),--                          ("./import/john/bogartbank/2017-include.journal",-                           ["./import/john/bogartbank/savings/2017-include.journal"]),-                          ("./import/john/bogartbank/2018-include.journal",-                           ["./import/john/bogartbank/checking/2018-include.journal",-                            "./import/john/bogartbank/savings/2018-include.journal"]),-                          ("./import/john/bogartbank/2019-include.journal",-                           ["./import/john/bogartbank/checking/2019-include.journal"]),--                          ("./import/john/otherbank/2017-include.journal",-                           ["./import/john/otherbank/creditcard/2017-include.journal"]),-                          ("./import/john/otherbank/2018-include.journal",-                           ["./import/john/otherbank/creditcard/2018-include.journal",-                            "./import/john/otherbank/investments/2018-include.journal"]),-                          ("./import/john/otherbank/2019-include.journal",-                           ["./import/john/otherbank/investments/2019-include.journal"])]-    assertEqual "groupIncludeFiles 2 - diff 1" [] (expectedGroup2 Map.\\ group2)-    assertEqual "groupIncludeFiles 2 - diff 2" [] (group2 Map.\\ expectedGroup2)-    assertEqual "groupIncludeFiles 2" expectedGroup2 group2--    let (group3, allYears3) = groupIncludeFiles (Map.keys group2)-    let expectedAllYears3 = [-          ("./import/jane/all-years.journal",-            ["./import/jane/2017-include.journal",-             "./import/jane/2018-include.journal",-             "./import/jane/2019-include.journal"]),-          ("./import/john/all-years.journal",-            ["./import/john/2017-include.journal",-             "./import/john/2018-include.journal",-             "./import/john/2019-include.journal"])]--    assertEqual "groupIncludeFiles Jane AllYears 3" expectedAllYears3 allYears3-    let expectedGroup3 = [("./import/jane/2017-include.journal",-                           ["./import/jane/bogartbank/2017-include.journal",-                            "./import/jane/otherbank/2017-include.journal"]),-                          ("./import/jane/2018-include.journal",-                           ["./import/jane/bogartbank/2018-include.journal",-                            "./import/jane/otherbank/2018-include.journal"]),-                          ("./import/jane/2019-include.journal",-                           ["./import/jane/bogartbank/2019-include.journal",-                            "./import/jane/otherbank/2019-include.journal"]),-                          ("./import/john/2017-include.journal",-                           ["./import/john/bogartbank/2017-include.journal",-                            "./import/john/otherbank/2017-include.journal"]),-                          ("./import/john/2018-include.journal",-                           ["./import/john/bogartbank/2018-include.journal",-                            "./import/john/otherbank/2018-include.journal"]),-                          ("./import/john/2019-include.journal",-                           ["./import/john/bogartbank/2019-include.journal",-                            "./import/john/otherbank/2019-include.journal"])]-    assertEqual "groupIncludeFiles 3 - diff 1" [] (expectedGroup3 Map.\\ group3)-    assertEqual "groupIncludeFiles 3 - diff 2" [] (group3 Map.\\ expectedGroup3)-    assertEqual "groupIncludeFiles 3" expectedGroup3 group3--    let (group4, allYears4) = groupIncludeFiles (Map.keys group3)-    let expectedAllYears4 = [-          ("./import/all-years.journal",-            ["./import/2017-include.journal",-             "./import/2018-include.journal",-             "./import/2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane AllYears 4" expectedAllYears4 allYears4-    let expectedGroup4 = [("./import/2017-include.journal",-                           ["./import/jane/2017-include.journal",-                            "./import/john/2017-include.journal"]),-                          ("./import/2018-include.journal",-                           ["./import/jane/2018-include.journal",-                            "./import/john/2018-include.journal"]),-                          ("./import/2019-include.journal",-                           ["./import/jane/2019-include.journal",-                            "./import/john/2019-include.journal"])]-    assertEqual "groupIncludeFiles 4 - diff 1" [] (expectedGroup4 Map.\\ group4)-    assertEqual "groupIncludeFiles 4 - diff 2" [] (group4 Map.\\ expectedGroup4)-    assertEqual "groupIncludeFiles 4" expectedGroup4 group4--    let (group5, allYears5) = groupIncludeFiles (Map.keys group4)-    let expectedAllYears5 = [("./all-years.journal", ["./2017-include.journal", "./2018-include.journal", "./2019-include.journal"])]-    assertEqual "groupIncludeFiles Jane AllYears 5" expectedAllYears5 allYears5-    let expectedGroup5 = [-          ("./2017-include.journal",-           ["./import/2017-include.journal"]),-          ("./2018-include.journal",-           ["./import/2018-include.journal"]),-          ("./2019-include.journal",-           ["./import/2019-include.journal"])]-    assertEqual "groupIncludeFiles 5 - diff 1" [] (expectedGroup5 Map.\\ group5)-    assertEqual "groupIncludeFiles 5 - diff 2" [] (group5 Map.\\ expectedGroup5)-    assertEqual "groupIncludeFiles 5" expectedGroup5 group5-  )--testIncludeYears :: Test-testIncludeYears = TestCase (-  do-    let txterr = "Some text without years"-    let expectederr = ["Unable to extract years from the following text:", txterr, "Errors:"]-    let actualerr = (init . head) $ map (T.lines) $ lefts [includeYears' txterr] :: [Text]-    assertEqual "Get a list of years from an include file - error case" expectederr actualerr--    let txt1 = "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>-          "!include import/2014-include.journal\n" <>-          "!include import/2015-include.journal\n" <>-          "!include import/2016-include.journal\n" <>-          "!include import/2017-include.journal\n" <>-          "!include import/2018-include.journal\n" <>-          "!include import/2019-include.journal"--    let expected1 = Right [2014..2019]-    let actual1 = includeYears' txt1-    assertEqual "Get a list of years from an include file - success case 1" expected1 actual1--    let txt2 = "!include 2019-include.journal"--    let expected2 = Right [2019]-    let actual2 = includeYears' txt2-    assertEqual "Get a list of years from an include file - success case 2" expected2 actual2-  )--testToIncludeLine :: Test-testToIncludeLine = TestCase (-  do-    let expected = "!include file1.journal"-    let relativeWithTrailingSlash = toIncludeLine "./base/dir/" "./base/dir/file1.journal"-    assertEqual "Include line - relative base dir with trailing slash" expected relativeWithTrailingSlash--    let relativeNoTrailingSlash = toIncludeLine "./base/dir" "./base/dir/file1.journal"-    assertEqual "Include line - relative base dir without a trailing slash" expected relativeNoTrailingSlash--    let absoluteWithTrailingSlash = toIncludeLine "/base/dir/" "/base/dir/file1.journal"-    assertEqual "Include line - absolute base dir with trailing slash" expected absoluteWithTrailingSlash--    let absoluteNoTrailingSlash = toIncludeLine "/base/dir" "/base/dir/file1.journal"-    assertEqual "Include line - absolute base dir without a trailing slash" expected absoluteNoTrailingSlash-  )--testToIncludeFiles :: Test-testToIncludeFiles = TestCase (-  do-    let expected = [-          ("./import/john/bogartbank/checking/2018-include.journal",-           "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>-           "!include import/john/bogartbank/checking/3-journal/2018/2018-10-30.journal\n" <>-           "!include import/john/bogartbank/checking/3-journal/2018/2018-11-30.journal\n" <>-           "!include import/john/bogartbank/checking/3-journal/2018/2018-12-30.journal\n"),-          ("./import/john/bogartbank/checking/2019-include.journal",-           "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>-           "!include import/john/bogartbank/checking/3-journal/2019/2019-01-30.journal\n" <>-           "!include import/john/bogartbank/checking/3-journal/2019/2019-02-30.journal\n"),-          ("./import/john/bogartbank/savings/2017-include.journal",-           "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>-           "!include import/john/bogartbank/savings/3-journal/2017/2017-11-30.journal\n" <>-           "!include import/john/bogartbank/savings/3-journal/2017/2017-12-30.journal\n"),-          ("./import/john/bogartbank/savings/2018-include.journal",-           "### Generated by hledger-flow - DO NOT EDIT ###\n\n" <>-           "!include import/john/bogartbank/savings/3-journal/2018/2018-01-30.journal\n" <>-           "!include import/john/bogartbank/savings/3-journal/2018/2018-02-30.journal\n")]--    ch <- newTChanIO-    txt <- toIncludeFiles (defaultOpts [absdir|/|]) ch groupedJohnBogart-    assertEqual "Convert a grouped map of paths, to a map with text contents for each file" expected txt)+import qualified CSVImport.ImportHelperTests+import qualified CSVImport.ImportHelperTurtleTests  tests :: Test-tests = TestList [testYearsIncludeMap, testYearsIncludeGrouping, testGroupIncludeFilesTinySet, testGroupIncludeFilesSmallSet, testGroupIncludeFiles, testIncludeYears, testToIncludeLine, testToIncludeFiles]+tests = TestList [CSVImport.ImportHelperTests.tests, CSVImport.ImportHelperTurtleTests.tests]

@@ -7,7 +7,7 @@ import Turtle import qualified Data.List as List (sort) -import TestHelpers+import TestHelpersTurtle import Hledger.Flow.PathHelpers (TurtlePath) import Hledger.Flow.Common @@ -23,7 +23,7 @@         let tmpHidden = map (tmpdir </>) hiddenFiles :: [TurtlePath]         let onDisk = List.sort $ tmpJournals ++ tmpExtras ++ tmpHidden         touchAll onDisk-        filtered <- (fmap List.sort) $ shellToList $ onlyFiles $ select onDisk+        filtered <- fmap List.sort $ shellToList $ onlyFiles $ select onDisk         let expected = List.sort $ tmpExtras ++ tmpJournals         liftIO $ assertEqual "Hidden files should be excluded" expected filtered      )

@@ -1,64 +1,20 @@ {-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE OverloadedLists #-} {-# LANGUAGE QuasiQuotes #-}  module TestHelpers where  import Path-import Turtle -import qualified Hledger.Flow.Types as FlowTypes-import Hledger.Flow.PathHelpers (TurtlePath)-import Hledger.Flow.Common-import Hledger.Flow.RuntimeOptions--inputJohnBogart :: [TurtlePath]-inputJohnBogart = [-  "import/john/bogartbank/savings/1-in/2017/2017-11-30.csv",-  "import/john/bogartbank/savings/1-in/2017/2017-12-30.csv",-  "import/john/bogartbank/savings/1-in/2018/2018-02-30.csv",-  "import/john/bogartbank/savings/1-in/2018/2018-01-30.csv",-  "import/john/bogartbank/checking/1-in/2018/2018-11-30.csv",-  "import/john/bogartbank/checking/1-in/2018/2018-10-30.csv",-  "import/john/bogartbank/checking/1-in/2018/2018-12-30.csv",-  "import/john/bogartbank/checking/1-in/2019/2019-01-30.csv",-  "import/john/bogartbank/checking/1-in/2019/2019-02-30.csv"]--inputJohnOther :: [TurtlePath]-inputJohnOther = [-  "import/john/otherbank/creditcard/1-in/2017/2017-12-30.csv",-  "import/john/otherbank/creditcard/1-in/2018/2018-01-30.csv",-  "import/john/otherbank/investments/1-in/2018/2018-12-30.csv",-  "import/john/otherbank/investments/1-in/2019/2019-01-30.csv"]--inputJaneBogart :: [TurtlePath]-inputJaneBogart = [-  "import/jane/bogartbank/savings/1-in/2017/2017-12-30.csv",-  "import/jane/bogartbank/savings/1-in/2018/2018-01-30.csv",-  "import/jane/bogartbank/checking/1-in/2018/2018-12-30.csv",-  "import/jane/bogartbank/checking/1-in/2019/2019-01-30.csv"]--inputJaneOther :: [TurtlePath]-inputJaneOther = [-  "import/jane/otherbank/creditcard/1-in/2017/2017-12-30.csv",-  "import/jane/otherbank/creditcard/1-in/2018/2018-01-30.csv",-  "import/jane/otherbank/investments/1-in/2018/2018-12-30.csv",-  "import/jane/otherbank/investments/1-in/2019/2019-01-30.csv"]--inputFiles :: [TurtlePath]-inputFiles = inputJohnBogart <> inputJohnOther <> inputJaneBogart <> inputJaneOther--journalFiles :: [TurtlePath]-journalFiles = toJournals inputFiles+import Data.Maybe (fromMaybe) -extraFiles :: [TurtlePath]-extraFiles = ["import/john/bogartbank/savings/2017-opening.journal"]+import Hledger.Flow.Internals (versionInfo, systemInfo)+import Hledger.Flow.PathHelpers (RelFile) -hiddenFiles :: [TurtlePath]-hiddenFiles = [".hiddenfile", "checking/.DS_Store", "import/john/bogartbank/savings/1-in/.anotherhiddenfile", "import/john/bogartbank/checking/1-in/2018/.hidden"]+import qualified Hledger.Flow.Types as FlowTypes+import Hledger.Flow.RuntimeOptions  defaultHlInfo :: FlowTypes.HledgerInfo-defaultHlInfo = FlowTypes.HledgerInfo "/path/to/hledger" "1.2.3"+defaultHlInfo = FlowTypes.HledgerInfo [absfile|/path/to/hledger|] "1.2.3"  defaultOpts :: FlowTypes.BaseDir -> RuntimeOptions defaultOpts bd = RuntimeOptions {@@ -66,7 +22,7 @@   , importRunDir = [reldir|./|]   , importStartYear = Nothing   , onlyNewFiles = False-  , hfVersion = versionInfo'+  , hfVersion = versionInfo   , hledgerInfo = defaultHlInfo   , sysInfo = systemInfo   , verbose = False@@ -74,13 +30,141 @@   , sequential = False } -toJournals :: [TurtlePath] -> [TurtlePath]-toJournals = map (changePathAndExtension "3-journal" "journal")+toJournal :: RelFile -> RelFile+toJournal inFile = do+  let journalFile = fromMaybe [relfile|oops.err|] $ replaceExtension ".journal" inFile+  let journalName = filename journalFile+  let yearDir = dirname . parent $ inFile+  (parent . parent . parent) inFile </> [reldir|3-journal|] </> yearDir </> journalName -touchAll :: [TurtlePath] -> Shell ()-touchAll = foldl (\acc file -> acc <> superTouch file) (return ())+inputJohnSavings2017 :: [RelFile]+inputJohnSavings2017 = [+  [relfile|import/john/bogartbank/savings/1-in/2017/2017-11-30.csv|],+  [relfile|import/john/bogartbank/savings/1-in/2017/2017-12-30.csv|]+  ] -superTouch :: TurtlePath -> Shell ()-superTouch file = do-  mktree $ directory file-  touch file+johnSavingsJournals2017 :: [RelFile]+johnSavingsJournals2017 = map toJournal inputJohnSavings2017++inputJohnSavings2018 :: [RelFile]+inputJohnSavings2018 = [+  [relfile|import/john/bogartbank/savings/1-in/2018/2018-02-30.csv|],+  [relfile|import/john/bogartbank/savings/1-in/2018/2018-01-30.csv|]+  ]++johnSavingsJournals2018 :: [RelFile]+johnSavingsJournals2018 = map toJournal inputJohnSavings2018++inputJohnChecking2018 :: [RelFile]+inputJohnChecking2018 = [+  [relfile|import/john/bogartbank/checking/1-in/2018/2018-11-30.csv|],+  [relfile|import/john/bogartbank/checking/1-in/2018/2018-10-30.csv|],+  [relfile|import/john/bogartbank/checking/1-in/2018/2018-12-30.csv|]+  ]++johnCheckingJournals2018 :: [RelFile]+johnCheckingJournals2018 = map toJournal inputJohnChecking2018++inputJohnChecking2019 :: [RelFile]+inputJohnChecking2019 = [+  [relfile|import/john/bogartbank/checking/1-in/2019/2019-01-30.csv|],+  [relfile|import/john/bogartbank/checking/1-in/2019/2019-02-30.csv|]+  ]++johnCheckingJournals2019 :: [RelFile]+johnCheckingJournals2019 = map toJournal inputJohnChecking2019++inputJohnBogart :: [RelFile]+inputJohnBogart = inputJohnSavings2017 <> inputJohnSavings2018 <> inputJohnChecking2018 <> inputJohnChecking2019++johnCC2017 :: RelFile+johnCC2017 = [relfile|import/john/otherbank/creditcard/1-in/2017/2017-12-30.csv|]++johnCCJournal2017 :: RelFile+johnCCJournal2017 = toJournal johnCC2017++johnCC2018 :: RelFile+johnCC2018 = [relfile|import/john/otherbank/creditcard/1-in/2018/2018-01-30.csv|]++johnCCJournal2018 :: RelFile+johnCCJournal2018 = toJournal johnCC2018++johnInvest2018 :: RelFile+johnInvest2018 = [relfile|import/john/otherbank/investments/1-in/2018/2018-12-30.csv|]++johnInvestJournal2018 :: RelFile+johnInvestJournal2018 = toJournal johnInvest2018++johnInvest2019 :: RelFile+johnInvest2019 = [relfile|import/john/otherbank/investments/1-in/2019/2019-01-30.csv|]++johnInvestJournal2019 :: RelFile+johnInvestJournal2019 = toJournal johnInvest2019++inputJohnOther :: [RelFile]+inputJohnOther = [johnCC2017, johnCC2018, johnInvest2018, johnInvest2019]++janeSavings2017 :: RelFile+janeSavings2017 = [relfile|import/jane/bogartbank/savings/1-in/2017/2017-12-30.csv|]++janeSavings2018 :: [RelFile]+janeSavings2018 = [+    [relfile|import/jane/bogartbank/savings/1-in/2018/2018-01-30.csv|]+  , [relfile|import/jane/bogartbank/savings/1-in/2018/2018-12-30.csv|]+  ]++janeSavings2019 :: RelFile+janeSavings2019 = [relfile|import/jane/bogartbank/savings/1-in/2019/2019-01-30.csv|]++inputJaneBogart :: [RelFile]+inputJaneBogart = [+  janeSavings2017+  , [relfile|import/jane/bogartbank/savings/3-journals/2018/2018-01-30.journal|]+  , [relfile|import/jane/bogartbank/savings/3-journals/2018/2018-12-30.journal|]+  , janeSavings2019+  ]++janeSavingsJournal2017 :: RelFile+janeSavingsJournal2017 = toJournal janeSavings2017++janeSavingsJournals2018 :: [RelFile]+janeSavingsJournals2018 = map toJournal janeSavings2018++janeSavingsJournal2019 :: RelFile+janeSavingsJournal2019 = toJournal janeSavings2019++janeSavingsJournals :: [RelFile]+janeSavingsJournals = [janeSavingsJournal2017] ++ janeSavingsJournals2018 ++ [janeSavingsJournal2019]++janeCC2017 :: RelFile+janeCC2017 = [relfile|import/jane/otherbank/creditcard/1-in/2017/2017-12-30.csv|]++janeCCJournal2017 :: RelFile+janeCCJournal2017 = toJournal janeCC2017++janeCC2018 :: RelFile+janeCC2018 = [relfile|import/jane/otherbank/creditcard/1-in/2018/2018-01-30.csv|]++janeCCJournal2018 :: RelFile+janeCCJournal2018 = toJournal janeCC2018++janeInvest2018 :: RelFile+janeInvest2018 = [relfile|import/jane/otherbank/investments/1-in/2018/2018-12-30.csv|]++janeInvestJournal2018 :: RelFile+janeInvestJournal2018 = toJournal janeInvest2018++janeInvest2019 :: RelFile+janeInvest2019 = [relfile|import/jane/otherbank/investments/1-in/2019/2019-01-30.csv|]++janeInvestJournal2019 :: RelFile+janeInvestJournal2019 = toJournal janeInvest2019++inputJaneOther :: [RelFile]+inputJaneOther = [janeCC2017, janeCC2018, janeInvest2018, janeInvest2019]++inputFiles :: [RelFile]+inputFiles = inputJohnBogart <> inputJohnOther <> inputJaneBogart <> inputJaneOther++journalFiles :: [RelFile]+journalFiles = map toJournal inputFiles

@@ -0,0 +1,71 @@+{-# LANGUAGE OverloadedStrings #-}++module TestHelpersTurtle where++import Turtle++import Hledger.Flow.PathHelpers (TurtlePath)+import Hledger.Flow.Common (changePathAndExtension)++inputJohnBogart :: [TurtlePath]+inputJohnBogart = [+  "import/john/bogartbank/savings/1-in/2017/2017-11-30.csv",+  "import/john/bogartbank/savings/1-in/2017/2017-12-30.csv",+  "import/john/bogartbank/savings/1-in/2018/2018-02-30.csv",+  "import/john/bogartbank/savings/1-in/2018/2018-01-30.csv",+  "import/john/bogartbank/checking/1-in/2018/2018-11-30.csv",+  "import/john/bogartbank/checking/1-in/2018/2018-10-30.csv",+  "import/john/bogartbank/checking/1-in/2018/2018-12-30.csv",+  "import/john/bogartbank/checking/1-in/2019/2019-01-30.csv",+  "import/john/bogartbank/checking/1-in/2019/2019-02-30.csv"]++inputJohnOther :: [TurtlePath]+inputJohnOther = [+  "import/john/otherbank/creditcard/1-in/2017/2017-12-30.csv",+  "import/john/otherbank/creditcard/1-in/2018/2018-01-30.csv",+  "import/john/otherbank/investments/1-in/2018/2018-12-30.csv",+  "import/john/otherbank/investments/1-in/2019/2019-01-30.csv"]++inputJaneBogart :: [TurtlePath]+inputJaneBogart = [+  "import/jane/bogartbank/savings/1-in/2017/2017-12-30.csv",+  "import/jane/bogartbank/savings/1-in/2018/2018-01-30.csv",+  "import/jane/bogartbank/checking/1-in/2018/2018-12-30.csv",+  "import/jane/bogartbank/checking/1-in/2019/2019-01-30.csv"]++inputJaneOther :: [TurtlePath]+inputJaneOther = [+  "import/jane/otherbank/creditcard/1-in/2017/2017-12-30.csv",+  "import/jane/otherbank/creditcard/1-in/2018/2018-01-30.csv",+  "import/jane/otherbank/investments/1-in/2018/2018-12-30.csv",+  "import/jane/otherbank/investments/1-in/2019/2019-01-30.csv"]++inputFiles :: [TurtlePath]+inputFiles = inputJohnBogart <> inputJohnOther <> inputJaneBogart <> inputJaneOther++journalFiles :: [TurtlePath]+journalFiles = toJournals inputFiles ++ ignoredJournalFiles++ignoredJournalFiles :: [TurtlePath]+ignoredJournalFiles = ["import/john/bogartbank/checking/3-journal/2018/not-a-journal.pdf"]++extraFiles :: [TurtlePath]+extraFiles = ["import/john/bogartbank/savings/2017-opening.journal"]++hiddenFiles :: [TurtlePath]+hiddenFiles = [".hiddenfile",+               "checking/.DS_Store",+               "import/john/bogartbank/savings/1-in/.anotherhiddenfile",+               "import/john/bogartbank/checking/1-in/2018/.hidden",+               "import/john/bogartbank/checking/3-journal/2018/.hidden"]++toJournals :: [TurtlePath] -> [TurtlePath]+toJournals = map (changePathAndExtension "3-journal" "journal")++touchAll :: [TurtlePath] -> Shell ()+touchAll = foldl (\acc file -> acc <> superTouch file) (return ())++superTouch :: TurtlePath -> Shell ()+superTouch file = do+  mktree $ directory file+  touch file