hledger-flow-0.21.0: src/Hledger/Flow/StatementScan.hs
{-# LANGUAGE OverloadedStrings #-}
-- | Delimited statement records and the date evidence they contain.
module Hledger.Flow.StatementScan
( DelimitedStatement (..),
parseDelimitedStatement,
parseRuleSample,
readRuleSample,
readDelimitedStatement,
splitHeader,
looksLikeHeader,
isDelimitedFile,
DateCell,
parseDateCell,
dateCellYear,
inferDateFormat,
isSignedDecimal,
StatementYear (..),
inferStatementYear,
)
where
import Data.Char (isDigit, isSpace, toLower)
import Data.List (intersect, nub, sort, transpose)
import Data.Maybe (catMaybes, isJust)
import qualified Data.Text as T
import qualified Data.Text.IO as T
import qualified Data.Text.Read as T
import Data.Time (fromGregorianValid)
import Hledger.Flow.PathHelpers (AbsFile)
import Path (toFilePath)
import qualified System.FilePath as F
import System.IO (IOMode (ReadMode), Handle, hIsEOF, hSetEncoding, mkTextEncoding, withFile)
data DelimitedStatement = DelimitedStatement
{ statementSeparator :: Char,
statementHeader :: Maybe [T.Text],
statementRows :: [[T.Text]]
}
deriving (Eq, Show)
isDelimitedFile :: FilePath -> Bool
isDelimitedFile path = map toLower (F.takeExtension path) `elem` [".csv", ".tsv", ".ssv"]
parseDelimitedStatement :: FilePath -> T.Text -> Either T.Text DelimitedStatement
parseDelimitedStatement = parseStatementSample (const Nothing)
parseRuleSample :: FilePath -> T.Text -> Either T.Text DelimitedStatement
parseRuleSample = parseStatementSample (Just . ruleRows)
parseStatementSample :: (DelimitedStatement -> Maybe Int) -> FilePath -> T.Text -> Either T.Text DelimitedStatement
parseStatementSample limit file contents =
case logicalRecords (T.lines (T.dropWhile (== '\xFEFF') contents)) of
[] -> Left noColumns
first : rest -> do
initial <- first >>= startStatement file
rows <- traverse (>>= parseRow initial) (maybe rest (\n -> take (remaining n initial) rest) (limit initial))
return initial {statementRows = statementRows initial ++ rows}
readDelimitedStatement :: Maybe Int -> AbsFile -> IO (Either T.Text DelimitedStatement)
readDelimitedStatement = readSample . const
readRuleSample :: AbsFile -> IO (Either T.Text DelimitedStatement)
readRuleSample = readSample (Just . ruleRows)
ruleRows :: DelimitedStatement -> Int
ruleRows initial = case statementHeader initial of
Just _ -> 0
Nothing -> 50
readSample :: (DelimitedStatement -> Maybe Int) -> AbsFile -> IO (Either T.Text DelimitedStatement)
readSample limit path = withFile (toFilePath path) ReadMode $ \handle -> do
hSetEncoding handle =<< mkTextEncoding "UTF-8//ROUNDTRIP"
first <- readRecord handle
case first >>= traverse (startStatement (toFilePath path) . T.dropWhile (== '\xFEFF')) of
Left err -> return (Left err)
Right Nothing -> return (Left noColumns)
Right (Just initial) -> do
rows <- readRows handle initial (fmap (\n -> remaining n initial) (limit initial)) []
return $ fmap (\xs -> initial {statementRows = statementRows initial ++ xs}) rows
remaining :: Int -> DelimitedStatement -> Int
remaining n initial = n - length (statementRows initial)
readRows :: Handle -> DelimitedStatement -> Maybe Int -> [[T.Text]] -> IO (Either T.Text [[T.Text]])
readRows _ _ (Just n) rows | n <= 0 = return (Right (reverse rows))
readRows handle initial limit rows = do
next <- readRecord handle
case next >>= traverse (parseRow initial) of
Left err -> return (Left err)
Right Nothing -> return (Right (reverse rows))
Right (Just row) -> readRows handle initial (subtract 1 <$> limit) (row : rows)
readRecord :: Handle -> IO (Either T.Text (Maybe T.Text))
readRecord handle = do
eof <- hIsEOF handle
if eof then return (Right Nothing) else do
line <- stripCR <$> T.hGetLine handle
if T.null (T.strip line) then readRecord handle else finish (openQuote False line) [line]
where
finish False chunks = return (Right (Just (T.intercalate "\n" (reverse chunks))))
finish True chunks = do
eof <- hIsEOF handle
if eof then return (Left "Unclosed quoted field.") else do
line <- stripCR <$> T.hGetLine handle
finish (openQuote True line) (line : chunks)
logicalRecords :: [T.Text] -> [Either T.Text T.Text]
logicalRecords = next . map stripCR
where
next [] = []
next (line : rest)
| T.null (T.strip line) = next rest
| otherwise = collect (openQuote False line) [line] rest
collect False chunks rest = Right (T.intercalate "\n" (reverse chunks)) : next rest
collect True _ [] = [Left "Unclosed quoted field."]
collect True chunks (line : rest) = collect (openQuote True line) (line : chunks) rest
stripCR :: T.Text -> T.Text
stripCR = T.dropWhileEnd (== '\r')
openQuote :: Bool -> T.Text -> Bool
openQuote = T.foldl' (\inside c -> if c == '"' then not inside else inside)
noColumns :: T.Text
noColumns = "Could not determine the statement's columns."
startStatement :: FilePath -> T.Text -> Either T.Text DelimitedStatement
startStatement file record = do
let sep = chooseSeparator (map toLower (F.takeExtension file)) record
fields <- splitFields sep record
case fields of
_ : _ : _ | looksLikeHeader fields -> return (DelimitedStatement sep (Just fields) [])
| otherwise -> return (DelimitedStatement sep Nothing [fields])
_ -> Left noColumns
parseRow :: DelimitedStatement -> T.Text -> Either T.Text [T.Text]
parseRow initial record = do
row <- splitFields (statementSeparator initial) record
let width = case (statementHeader initial, statementRows initial) of
(Just header, _) -> length header
(_, first : _) -> length first
_ -> 0
if length row == width then Right row else Left "Statement rows have different numbers of columns."
splitHeader :: T.Text -> (Char, [T.Text])
splitHeader line =
let record = stripCR (T.dropWhile (== '\xFEFF') line)
sep = chooseSeparator ".csv" record
in (sep, either (const []) id (splitFields sep record))
chooseSeparator :: String -> T.Text -> Char
chooseSeparator ".tsv" _ = '\t'
chooseSeparator ".ssv" _ = ';'
chooseSeparator _ record = foldl' pick ',' [';', '\t']
where
pick best other | count other > count best = other
| otherwise = best
count sep = snd (T.foldl' (step sep) (False, 0 :: Int) record)
step _ (quoted, n) '"' = (not quoted, n)
step sep (quoted, n) c = (quoted, if not quoted && c == sep then n + 1 else n)
splitFields :: Char -> T.Text -> Either T.Text [T.Text]
splitFields sep = field [] . T.unpack
where
field fields rest
| '"' : contents <- dropWhile (\c -> isSpace c && c /= sep) rest = quoted fields [] contents
| otherwise =
let (value, tail') = break (== sep) rest
in if '"' `elem` value then Left "Unexpected quote in a delimited field."
else end fields (T.strip (T.pack value)) tail'
quoted _ _ [] = Left "Unclosed quoted field."
quoted fields value ('"' : '"' : rest) = quoted fields ('"' : value) rest
quoted fields value ('"' : rest) =
end fields (T.strip (T.pack (reverse value))) (dropWhile (\c -> isSpace c && c /= sep) rest)
quoted fields value (c : rest) = quoted fields (c : value) rest
end fields value [] = Right (reverse (value : fields))
end fields value (c : rest)
| c == sep = field (value : fields) rest
| otherwise = Left "Unexpected text after a quoted field."
looksLikeHeader :: [T.Text] -> Bool
looksLikeHeader = not . any dataish
where
dataish f = isJust (parseDateCell f) ||
(not (T.null f) && T.all (`elem` ("0123456789-/.:,+$€£ " :: String)) f)
isSignedDecimal :: T.Text -> Bool
isSignedDecimal raw = case T.splitOn "." (dropSign raw) of
[whole] -> digits whole
[whole, fraction] -> digits whole && digits fraction
_ -> False
where
dropSign t = case T.uncons t of
Just (sign, rest) | sign `elem` ['+', '-'] -> rest
_ -> t
digits :: T.Text -> Bool
digits t = not (T.null t) && T.all (\c -> c >= '0' && c <= '9') t
decimal :: T.Text -> Maybe Int
decimal t
| digits t, Right (n, "") <- T.decimal t = Just n
| otherwise = Nothing
data DateCell = DateCell {dateCellYear :: Integer, dateFormats :: [T.Text]}
deriving (Eq, Show)
parseDateCell :: T.Text -> Maybe DateCell
parseDateCell value = do
let (date, suffix) = T.break (\c -> isSpace c || c == 'T') (T.strip value)
(sep, _) <- T.uncons (T.dropWhile isDigit date)
if sep `elem` ['-', '/', '.'] then Just () else Nothing
-- hledger accepts both padded and unpadded components with %-m and %-d.
case T.splitOn (T.singleton sep) date of
[a,b,c]
| T.length a == 4 && short b && short c -> do
year <- parseYear a
m <- decimal b
d <- decimal c
-- Year-first inputs use conventional ISO order wherever it fits.
let formats = if valid year m d then [fmt sep ["%Y","%-m","%-d"]]
else [fmt sep ["%Y","%-d","%-m"] | valid year d m]
cell year formats suffix
| short a && short b && T.length c == 4 -> do
year <- parseYear c
lo <- decimal a
hi <- decimal b
cell year ([fmt sep ["%-m","%-d","%Y"] | valid year lo hi] ++
[fmt sep ["%-d","%-m","%Y"] | valid year hi lo]) suffix
_ -> Nothing
where
short t = T.length t <= 2 && digits t
valid y m d = isJust (fromGregorianValid y m d)
parseYear t = do
n <- decimal t
if n >= 1900 && n <= 2099 then Just (toInteger n) else Nothing
fmt sep = T.intercalate (T.singleton sep)
cell _ [] _ = Nothing
cell year formats suffix = case timeSuffix suffix of
Nothing -> Nothing
Just layout -> Just (DateCell year (maybe [] (\timeFormat -> map (<> timeFormat) formats) layout))
-- | An unfamiliar but time-like suffix supplies a year without a format guess.
timeSuffix :: T.Text -> Maybe (Maybe T.Text)
timeSuffix "" = Just (Just "")
timeSuffix suffix = do
let (joiner, time) = case T.stripPrefix "T" suffix of
Just t -> ("T", t)
Nothing -> (" ", T.stripStart suffix)
(hour, tail') <- T.uncons time
if isDigit hour && ":" `T.isInfixOf` tail' then Just () else Nothing
return $ case T.splitOn ":" time of
[h,m] | within 23 h && within 59 m -> Just (joiner <> "%H:%M")
[h,m,s] | within 23 h && within 59 m && within 59 s -> Just (joiner <> "%H:%M:%S")
_ -> Nothing
where
within bound t = T.length t == 2 && maybe False (<= bound) (decimal t)
inferDateFormat :: [DateCell] -> Maybe T.Text
inferDateFormat cells = case map dateFormats cells of
first : rest | [one] <- foldl' intersect first rest -> Just one
_ -> Nothing
data StatementYear = UnknownYear | SingleYear Integer | SpanningYears [Integer]
deriving (Eq, Show)
inferStatementYear :: DelimitedStatement -> StatementYear
inferStatementYear statement = case traverse agreed (transpose dateColumns) of
Just years -> case sort (nub years) of
[year] -> SingleYear year
[] -> UnknownYear
many -> SpanningYears many
Nothing -> UnknownYear
where
dateColumns =
[ dates
| column <- transpose (statementRows statement),
let dates = map parseDateCell column,
any isJust dates,
and (zipWith (\cell date -> T.null cell || isJust date) column dates)
]
agreed cells = case nub (map dateCellYear (catMaybes cells)) of
[year] -> Just year
_ -> Nothing