hledger-flow-0.20.1: src/Hledger/Flow/Import/Dependencies.hs
{-# LANGUAGE CPP #-}
{-# LANGUAGE DeriveFunctor #-}
{-# LANGUAGE OverloadedStrings #-}
module Hledger.Flow.Import.Dependencies
( Release (..), FallbackReason (..), ReuseDecision (..), Converter (..),
ruleIncludes, nestedRuleIncludes, resolveRuleInclude, discoverRules, selectRules, isSeparatedValues,
checkedIO, atPath, checkArguments, discoverConfig, discoverDirectives, checkConfig, checkDirectives,
newConverterProbe, converterProbeWith
) where
import Control.Concurrent.MVar (modifyMVar, newMVar)
import Control.Monad (ap, foldM)
import qualified Data.ByteString as B
import Data.Char (isAlpha, isAlphaNum, isAscii, isControl, isDigit, isSpace, toLower, GeneralCategory(CurrencySymbol), generalCategory)
import Data.List (isPrefixOf)
import Data.Maybe (mapMaybe)
import qualified Data.Set as Set
import qualified Data.Text as T
import qualified Data.Text.Encoding as TE
import qualified Data.Text.Read as TR
import Data.Time (getCurrentTimeZone, timeZoneMinutes)
import GHC.IO.Encoding (getLocaleEncoding, getFileSystemEncoding, getForeignEncoding)
import Hledger.Flow.Common (firstConfirmedFile, lookupHledgerConf, probeHledgerVersion, hledgerVersionNumber)
import Hledger.Flow.Import.Fingerprint
import Hledger.Flow.Import.ImportHelpers (rulesFileCandidates)
import Hledger.Flow.Import.Types (ImportDirs)
import Hledger.Flow.PathHelpers (AbsDir, AbsFile)
import qualified Hledger.Flow.RuntimeOptions as RT
import Hledger.Flow.Types (HledgerConf(..), hledgerConfPath)
import Path (Abs, parseAbsFile, toFilePath)
import qualified System.Directory as D
import qualified System.FilePath as F
import System.Exit (ExitCode(..))
import System.IO (IOMode(ReadMode), withBinaryFile)
import System.IO.Error (tryIOError, ioeGetFileName, ioeGetErrorString)
import qualified System.Info as Sys
#ifdef mingw32_HOST_OS
import System.Win32.Console (getConsoleCP, getConsoleOutputCP)
import System.Win32.NLS (getACP)
#endif
-- Exact releases have known reader quirks. Other eligible versions use syntax
-- shared by both readers, leaving ambiguous inputs to the ordinary converter.
data Release = V1524 | V1996 | OtherSupported deriving (Eq, Show)
data FallbackReason
= FallbackReason {reasonScope :: T.Text, reasonDetail :: T.Text}
| FileReason {reasonScope :: T.Text, reasonFile :: FilePath, reasonDetail :: T.Text}
| OrdinaryInputError {reasonScope :: T.Text, reasonDetail :: T.Text}
| CyclicRules {reasonScope :: T.Text, reasonDetail :: T.Text}
deriving (Eq, Ord, Show)
data ReuseDecision a = Reusable a | Fallback FallbackReason deriving (Functor, Eq, Show)
instance Applicative ReuseDecision where
pure = Reusable
(<*>) = ap
instance Monad ReuseDecision where
Reusable a >>= f = f a
Fallback r >>= _ = Fallback r
atPath :: T.Text -> FilePath -> T.Text -> ReuseDecision a
atPath reason path detail = Fallback (FileReason reason path detail)
checkedIO :: T.Text -> FilePath -> IO (ReuseDecision a) -> IO (ReuseDecision a)
checkedIO reason path action = do
result <- tryIOError action
pure $ case result of
Right a -> a
Left e -> atPath reason (maybe path id (ioeGetFileName e)) (T.pack (ioeGetErrorString e))
physicalLines :: AbsFile -> B.ByteString -> ReuseDecision [T.Text]
physicalLines path bytes = case TE.decodeUtf8' bytes of
Left _ -> atPath "unrecognised text encoding" (toFilePath path) "expected UTF-8"
Right s -> pure (T.lines (T.replace "\r\n" "\n" s))
ruleIncludes :: Release -> AbsFile -> B.ByteString -> ReuseDecision [FilePath]
ruleIncludes release path bytes = do
ls <- physicalLines path (dropUtf8Bom bytes)
sequence (mapMaybe include (zip [1 :: Int ..] ls))
where
include (n,line)
| Just raw <- T.stripPrefix "include " line = Just (checkPath n raw)
| release == OtherSupported && looksLikeInclude line = Just (uncertain n)
| otherwise = Nothing
looksLikeInclude line = case T.words line of
word:_ -> T.toCaseFold word == "include"
[] -> False
checkPath n raw
| T.null name || T.any isControl name = uncertain n
| release == OtherSupported && name /= T.stripEnd name = uncertain n
| otherwise = pure (T.unpack name)
where name = if release == V1996 then T.strip raw else T.stripStart raw
uncertain n = atPath "uncertain rules include" (toFilePath path) ("line " <> T.pack (show n))
-- Older readers open nested rules through the native text handle. On POSIX a
-- CR at the end of an include line may be part of the filename.
nestedRuleIncludes :: Release -> AbsFile -> B.ByteString -> ReuseDecision [FilePath]
nestedRuleIncludes release path bytes
| release /= V1996, any ambiguous (B.split 10 bytes) =
atPath "uncertain nested rules include" (toFilePath path) "native line ending or byte-order mark"
| otherwise = ruleIncludes release path bytes
where
ambiguous line = "include " `B.isPrefixOf` dropUtf8Bom line &&
(B.elem 13 line || dropUtf8Bom line /= line)
dropUtf8Bom :: B.ByteString -> B.ByteString
dropUtf8Bom b = maybe b id (B.stripPrefix (B.pack [0xef,0xbb,0xbf]) b)
-- Walk leading .. components relative to the including file's spelled path.
-- Verify the result against the filesystem's resolution, which can differ when a
-- symlink precedes .. . An internal symlink is mirrored under its original name.
resolveRuleInclude :: AbsDir -> AbsFile -> FilePath -> IO (ReuseDecision AbsFile)
resolveRuleInclude base parent child = checkedIO "uncertain rules include" raw $
if F.hasDrive child || not (F.isRelative child) || any F.isPathSeparator (take 1 child)
then pure (atPath "absolute rules include" child (T.pack (toFilePath parent)))
else do
let (ups, rest) = span (== "..") (filter (/= ".") (F.splitDirectories child))
lexical = foldr (const F.takeDirectory) (F.takeDirectory (toFilePath parent)) ups F.</> F.joinPath rest
if ".." `elem` rest || null rest || not (inside (toFilePath base) lexical)
then pure outside
else do
actual <- D.canonicalizePath raw
mirror <- D.canonicalizePath lexical
root <- D.canonicalizePath (toFilePath base)
if F.equalFilePath actual mirror && inside root actual
then pure (maybe outside Reusable (parseAbsFile lexical))
else pure outside
where
raw = F.takeDirectory (toFilePath parent) F.</> child
outside = atPath "rules include escapes the ledger or cannot be mirrored" raw (T.pack (toFilePath parent))
inside root path = F.splitDirectories (F.dropTrailingPathSeparator root) `isPrefixOf` F.splitDirectories path
discoverRules :: AbsDir -> Release -> AbsFile -> IO (ReuseDecision [(AbsFile,Digest)])
discoverRules base release = walk Set.empty
where
walk ancestors path = checkedIO "could not read rules file" (toFilePath path) $ do
canonical <- D.canonicalizePath (toFilePath path)
directory <- D.canonicalizePath (F.takeDirectory (toFilePath path))
-- The same rules bytes can resolve includes in different directories.
-- In 1.52.4 the first read also decodes line endings differently.
let node = (canonical, directory, release /= V1996 && Set.null ancestors)
if Set.member node ancestors && release /= V1996
then pure $ Fallback (CyclicRules "cyclic rules include" (T.pack (toFilePath path)))
else if Set.member node ancestors || Set.size ancestors >= 256
then pure (atPath "cyclic or excessively deep rules include" (toFilePath path) "")
else do
bytes <- B.readFile (toFilePath path)
let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes
case scan release path bytes of
Fallback r -> pure (Fallback r)
Reusable children -> do
nested <- descend (Set.insert node ancestors) path children
pure (((path,digestBytes bytes):) <$> nested)
descend _ _ [] = pure (pure [])
descend ancestors path (c:cs) = do
selected <- resolveRuleInclude base path c
tree <- case selected of
Fallback r -> pure (Fallback r)
Reusable file -> walk ancestors file
case tree of
Fallback r -> pure (Fallback r)
Reusable files -> fmap (files ++) <$> descend ancestors path cs
selectRules :: AbsFile -> ImportDirs Abs -> IO (ReuseDecision (Maybe AbsFile))
selectRules csv dirs = checkedIO "could not select rules file" (toFilePath csv) $ do
candidates <- rulesFileCandidates csv dirs
selected <- firstConfirmedFile (map toFilePath candidates)
Reusable <$> traverse parseAbsFile selected
isSeparatedValues :: AbsFile -> Bool
isSeparatedValues path = map toLower (F.takeExtension (toFilePath path)) `elem` [".csv",".tsv",".ssv"]
checkArguments :: [T.Text] -> ReuseDecision ()
checkArguments args = case filter ("@" `T.isPrefixOf`) args of
a:_ -> Fallback (FallbackReason ("indirect hledger argument " <> a) "")
[] -> pure ()
discoverConfig :: Release -> Maybe HledgerConf -> IO (ReuseDecision (Maybe (AbsFile,Digest)))
discoverConfig release configured = checkedIO "could not discover hledger config" "hledger.conf" $ do
let given = case configured of
Just (GivenConf p) -> Just p
_ -> Nothing
case checkArguments (maybe [] (pure . T.pack) given) of
Fallback r -> pure (Fallback r)
Reusable () -> do
selected <- lookupHledgerConf given
case hledgerConfPath <$> selected of
Nothing -> pure (pure Nothing)
Just path | "~" `isPrefixOf` path || ".." `elem` F.splitDirectories path ->
pure (atPath "uncertain config path" path "")
Just path -> do
absolute <- D.makeAbsolute path
case parseAbsFile absolute of
Nothing -> pure (atPath "uncertain config path" path "")
Just absPath -> fmap Just <$> readDependency (checkConfig release) absPath
discoverDirectives :: AbsFile -> IO (ReuseDecision (Maybe (AbsFile,Digest)))
discoverDirectives file = checkedIO "could not discover directives" (toFilePath file) $ do
exists <- firstConfirmedFile [toFilePath file]
case exists of
Nothing -> pure (pure Nothing)
Just _ -> fmap Just <$> readDependency (checkDirectives V1996) file
readDependency :: (AbsFile -> B.ByteString -> ReuseDecision ()) -> AbsFile -> IO (ReuseDecision (AbsFile,Digest))
readDependency check path = checkedIO "could not read dependency" (toFilePath path) $ do
bytes <- B.readFile (toFilePath path)
pure ((path,digestBytes bytes) <$ check path bytes)
checkConfig :: Release -> AbsFile -> B.ByteString -> ReuseDecision ()
checkConfig release path bytes = do
ls <- physicalLines path bytes
() <$ foldM line "general" (zip [1 :: Int ..] ls)
where
-- Only canonical built-ins other than print have an inert section. Addons,
-- aliases, abbreviations and names introduced in another release can have
-- different scope and must be passed through to the ordinary converter.
inactive = T.words $ "accounts activity add aregister balance balancesheet balancesheetequity cashflow check close codes commodities descriptions diff files help import incomestatement notes payees prices register repl rewrite roi run setup stats tags " <>
case release of
V1524 -> "commands demo test"
V1996 -> "get holdings transactions"
OtherSupported -> ""
line section (n,raw)
| T.null body || "#" `T.isPrefixOf` body = pure section
| release /= V1996, "[" `T.isPrefixOf` body, T.stripStart raw /= raw = unsupported n body
| Just start <- T.stripPrefix "[" body = case T.breakOn "]" start of
(s,end) | not (T.null end), not ("#" `T.isInfixOf` s), let name = T.strip s,
name `elem` ("general":"print":inactive) -> option n name (T.drop 1 end)
_ -> unsupported n body
| otherwise = option n section body
where body = T.strip raw
option n section arg
| section `elem` inactive = pure section
| T.null setting = pure section
| section == "print" && release /= V1524 && setting == "--layout=hledger1" = pure section
| otherwise = unsupported n setting
where setting = T.strip (T.takeWhile (/= '#') arg)
unsupported n value = atPath "unsupported hledger config" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)
checkDirectives :: Release -> AbsFile -> B.ByteString -> ReuseDecision ()
checkDirectives _ path bytes = do
ls <- physicalLines path (dropUtf8Bom bytes)
case [(n,T.strip l) | (n,l) <- zip [1 :: Int ..] ls, not (allowed l)] of
[] -> pure ()
(n,value):_ -> atPath "unsupported journal directive" (toFilePath path) ("line " <> T.pack (show n) <> ": " <> value)
where
allowed raw
| T.null stripped || T.isPrefixOf ";" stripped || T.isPrefixOf "#" stripped || T.isPrefixOf "*" stripped = True
| Just (c,_) <- T.uncons raw, isSpace c = False
| otherwise = case T.words (T.takeWhile (/= ';') raw) of
"account":xs -> not (null xs) && all (T.all accountChar) xs
"commodity":xs -> amount xs
"P":date:symbol:xs -> fullDate date && commodity symbol && amount xs
_ -> False
where stripped = T.strip raw
accountChar c = isAlphaNum c || c `elem` (":_-/.'’" :: String)
commodity s = not (T.null s) && T.all (\c -> isAlpha c || generalCategory c == CurrencySymbol) s
amount [a,b] = (number a && commodity b) || (commodity a && number b)
amount [a] =
let (symbol,rest) = T.span (\c -> isAlpha c || generalCategory c == CurrencySymbol) a
(value,suffix) = T.span (\c -> isDigit c || c `elem` (".,+-" :: String)) a
in (commodity symbol && number rest) || (number value && commodity suffix)
amount _ = False
number s = case T.uncons (T.dropWhile (\c -> c == '+' || c == '-') s) of
Just (c,rest) -> digit c && T.all (\x -> digit x || x == ',' || x == '.') rest
_ -> False
digit c = isAscii c && isDigit c
fullDate s = case T.split (\c -> c == '-' || c == '/') s of
[y,m,d] -> map T.length [y,m,d] == [4,2,2] && all (T.all digit) [y,m,d]
_ -> False
data Converter = Converter
{ converterRelease :: Release
, converterExecutable :: AbsFile
, converterStamp :: FileStamp
, converterKey :: [T.Text]
} deriving (Eq, Show)
newConverterProbe :: RT.RuntimeOptions -> IO (IO (ReuseDecision Converter))
newConverterProbe opts = converterProbeWith (RT.hledgerExecutable opts) probeHledgerVersion
-- The expensive executable hash and version probe are shared by workers only
-- while the resolved executable and its handle-derived status still agree.
converterProbeWith :: AbsFile -> (AbsFile -> IO (ExitCode,T.Text)) -> IO (IO (ReuseDecision Converter))
converterProbeWith executable getVersion = do
context <- checkedIO "could not identify the conversion context" original conversionContext
cache <- newMVar Nothing
pure $ case context of
Fallback r -> pure (Fallback r)
Reusable environment -> checkedIO "could not identify hledger executable" original $ do
resolved <- D.canonicalizePath original >>= parseAbsFile
let target = toFilePath resolved
wrapped <- launcher original target
if wrapped
then pure (atPath "hledger launcher" original "select the real executable with --hledger-path")
else modifyMVar cache $ \old -> do
stamp <- statFile resolved
case old of
Just (p,s,c) | p == resolved && s == stamp -> pure (old, c)
_ -> do
observed <- observeFile resolved
(status,version) <- getVersion resolved
after <- statFile resolved
live <- D.canonicalizePath original
let key = [T.pack original,T.pack target,digestHex (observationDigest observed),version] ++ environment
result | after /= stamp || fileStamp observed /= stamp || not (F.equalFilePath live target) =
atPath "hledger executable changed" original ""
| status /= ExitSuccess = atPath "could not identify hledger version" original ""
| otherwise = case supportedRelease version of
Just release -> Reusable (Converter release resolved stamp key)
Nothing -> Fallback (FallbackReason ("unsupported hledger version " <> version) (T.pack original))
pure (Just (resolved,stamp,result),result)
where original = toFilePath executable
conversionContext :: IO (ReuseDecision [T.Text])
conversionContext = do
offset <- timeZoneMinutes <$> getCurrentTimeZone
locale <- show <$> getLocaleEncoding
fs <- show <$> getFileSystemEncoding
foreignEnc <- show <$> getForeignEncoding
#ifdef mingw32_HOST_OS
pages <- sequence [show <$> getConsoleCP, show <$> getConsoleOutputCP, show <$> getACP]
#else
let pages = []
#endif
-- Rules and directives are inspected from their exact UTF-8 snapshot bytes.
-- A different reader encoding could interpret a byte sequence as another path.
pure $ if map toLower locale `elem` ["utf-8","utf8","cp65001"]
then Reusable (map T.pack ([Sys.os,Sys.arch,show offset,locale,fs,foreignEnc] ++ pages))
else Fallback (FallbackReason ("unsupported conversion encoding " <> T.pack locale) "")
supportedRelease :: T.Text -> Maybe Release
supportedRelease text = do
let number = T.takeWhile (/= '-') (hledgerVersionNumber text)
component part = case TR.decimal part of
Right (n, rest) | T.null rest -> Just (n :: Integer)
_ -> Nothing
version <- traverse component (T.splitOn "." number)
case version of
[1,52,4] -> Just V1524
[1,99,6] -> Just V1996
_ | version >= [1,52,4] -> Just OtherSupported
| otherwise -> Nothing
launcher :: FilePath -> FilePath -> IO Bool
launcher original target
| any (\p -> map toLower (F.takeExtension p) `elem` [".cmd",".bat"]) [original,target] = pure True
| otherwise = do
shim <- firstConfirmedFile [F.replaceExtension p "shim" | p <- [original,target], map toLower (F.takeExtension p) == ".exe"]
case shim of
Just _ -> pure True
Nothing -> do
header <- withBinaryFile target ReadMode (flip B.hGet 4)
pure $ not (B.take 2 header == "MZ" || header `elem` nativeMagic)
where
-- ELF, Mach-O (both byte orders) and universal Mach-O (32- and 64-bit).
nativeMagic = map B.pack [[127,69,76,70],[0xfe,0xed,0xfa,0xce],[0xce,0xfa,0xed,0xfe],
[0xfe,0xed,0xfa,0xcf],[0xcf,0xfa,0xed,0xfe],[0xca,0xfe,0xba,0xbe],[0xbe,0xba,0xfe,0xca],
[0xca,0xfe,0xba,0xbf],[0xbf,0xba,0xfe,0xca]]