packages feed

hledger-flow-0.20.1: src/Hledger/Flow/Import/Snapshot.hs

{-# LANGUAGE CPP #-}
{-# LANGUAGE OverloadedStrings #-}
{-# LANGUAGE QuasiQuotes #-}
{-# LANGUAGE ScopedTypeVariables #-}

module Hledger.Flow.Import.Snapshot
  ( SnapshotRun, SnapshotInput (..), PreparedSnapshot, snapshotInputs, snapshotManifest,
    withSnapshotRun, withSnapshotRunUsing, withSnapshot,
    translateDiagnostic, windowsSnapshotPathFits
  ) where

import Control.Concurrent (threadDelay)
import Control.Concurrent.MVar
import Control.Exception hiding (handle)
import Control.Monad (filterM, forM_, unless, void, when)
import qualified Data.ByteString as BS
import Data.Char (isAlphaNum, isAsciiLower, isDigit)
import Data.IORef
import qualified Data.Map.Strict as Map
import qualified Data.Set as Set
import Data.Text (Text)
import qualified Data.Text as T
import Data.Word (Word32)
import GHC.IO.Handle.Lock (LockMode (ExclusiveLock), hLock, hTryLock)
import Hledger.Flow.Import.Dependencies
import Hledger.Flow.Import.Fingerprint
import Hledger.Flow.Import.PrivateDirectory
import Hledger.Flow.PathHelpers (AbsDir, AbsFile)
import Numeric (showHex)
import Path
import qualified Path.IO as P
import qualified System.Directory as D
import qualified System.FilePath as FP
import System.IO
#ifndef mingw32_HOST_OS
import qualified System.Posix.IO as Posix
#endif
import Text.Read (readMaybe)
import System.IO.Error

data SnapshotInput
  = RulesInput AbsFile AbsFile (Maybe AbsFile) (Maybe AbsFile) -- data, rules, directives, config
  | ConstructInput (Maybe AbsFile)
  deriving (Eq, Show)

data PreparedSnapshot = PreparedSnapshot
  { snapshotInputs :: SnapshotInput,
    snapshotManifest :: Map.Map AbsFile Digest,
    originalPaths :: Map.Map AbsFile AbsFile
  }

data SnapshotRun = SnapshotRun
  { ledger :: AbsDir,
    registration :: MVar (Maybe (AbsDir, Handle)),
    nextTree :: IORef Word32,
    makePrivate :: AbsDir -> IO ()
  }

withSnapshotRun :: AbsDir -> (SnapshotRun -> IO a) -> IO a
withSnapshotRun = withSnapshotRunUsing createPrivateDirectory

withSnapshotRunUsing :: (AbsDir -> IO ()) -> AbsDir -> (SnapshotRun -> IO a) -> IO a
withSnapshotRunUsing create base use = do
  run <- SnapshotRun base <$> newMVar Nothing <*> newIORef 0 <*> pure create
  let area = snapshotArea run
  -- Failure to claim an abandoned tree leaves it for a later invocation.
  void $ tryIOError $ do
    exists <- P.doesDirExist area
    when exists $ withRegistration area $ reap area
  use run `finally` (readMVar (registration run) >>= mapM_ (releaseRun area))

snapshotArea :: SnapshotRun -> AbsDir
snapshotArea run = ledger run </> [reldir|.hledger-flow/cache/snapshots|]

-- GHC prevents two writable opens of the same file within one process. Wait
-- for that case too (concurrent imports in one process). The OS lock also
-- serializes independent processes, including while an initializer is paused.
withRegistration :: AbsDir -> IO a -> IO a
withRegistration area action = bracket open hClose $ \handle -> hLock handle ExclusiveLock >> action
  where
    file = toFilePath (area </> [relfile|registration.lock|])
    open = catchIOError (openLockFile file) $ \e ->
      if isAlreadyInUseError e then threadDelay 1000 >> open else ioError e

-- A surviving hook must not retain its killed parent's lifetime lock.
openLockFile :: FilePath -> IO Handle
#ifdef mingw32_HOST_OS
openLockFile file = openBinaryFile file ReadWriteMode
#else
openLockFile file = bracketOnError (Posix.openFd file Posix.ReadWrite flags) Posix.closeFd Posix.fdToHandle
  where flags = Posix.defaultFileFlags {Posix.creat = Just 0o666, Posix.cloexec = True}
#endif

shortName :: Char -> Word32 -> String
shortName prefix n = prefix : replicate (8 - length digits) '0' ++ digits
  where digits = showHex n ""

lockFile :: AbsDir -> FilePath
lockFile dir = toFilePath (dir </> [relfile|run.lock|])

acquireTree :: SnapshotRun -> IO AbsDir
acquireTree run = do
  dir <- modifyMVarMasked (registration run) $ \current -> case current of
    Just registered -> pure (current, fst registered)
    Nothing -> do
      registered <- catchIOError register $ \e -> reject "private staging unavailable" (T.pack (displayException e))
      pure (Just registered, fst registered)
  n <- atomicModifyIORef' (nextTree run) $ \i -> (i + 1, i)
  tree <- (dir </>) <$> parseRelDir (shortName 't' n)
  P.createDir tree
  pure tree
  where
    area = snapshotArea run
    register = do
      P.createDirIfMissing True area
      withRegistration area (createRun 0)
    createRun n = do
      dir <- (area </>) <$> parseRelDir (shortName 'r' n)
      result <- tryIOError (makePrivate run dir)
      case result of
        Left e | isAlreadyExistsError e && n < maxBound -> createRun (n + 1)
        Left e -> ioError e
        Right () -> flip onException (P.removeDirRecur dir) $
          bracketOnError (openLockFile (lockFile dir)) hClose $ \handle -> do
            hLock handle ExclusiveLock
            pure (dir, handle)

releaseRun :: AbsDir -> (AbsDir, Handle) -> IO ()
releaseRun area (dir, handle) = withRegistration area $
  (clearRun dir `finally` hClose handle) >> removeEmptyRun dir

-- Keep the ownership marker until every byte of statement data is gone. A
-- process killed during reclamation leaves a lockable tree for the next run.
clearRun :: AbsDir -> IO ()
clearRun dir = do
  entries <- D.listDirectory (toFilePath dir)
  forM_ (filter (/= "run.lock") entries) $ \entry ->
    D.removePathForcibly (toFilePath dir FP.</> entry)

removeEmptyRun :: AbsDir -> IO ()
removeEmptyRun dir = D.removeFile (lockFile dir) >> P.removeDir dir

reap :: AbsDir -> IO ()
reap area = do
  names <- D.listDirectory (toFilePath area)
  forM_ (filter isRunName names) $ \name -> void $ tryIOError $ do
    dir <- (area </>) <$> parseRelDir name
    let path = FP.dropTrailingPathSeparator (toFilePath dir)
    linked <- D.pathIsSymbolicLink path
    unless linked $ do
      let marker = lockFile dir
      exists <- D.doesFileExist marker
      when exists $ do
        link <- D.pathIsSymbolicLink marker
        unless link $ do
          claimed <- bracket (openLockFile marker) hClose $ \handle -> do
            mine <- hTryLock handle ExclusiveLock
            when mine (clearRun dir)
            pure mine
          when claimed (removeEmptyRun dir)
  where
    isRunName ('r':digits) = length digits == 8 && all hex digits
    isRunName _ = False
    hex c = c >= '0' && c <= '9' || isAsciiLower c && c <= 'f'

newtype PreparationFailure = PreparationFailure FallbackReason deriving Show
instance Exception PreparationFailure

reject :: Text -> Text -> IO a
reject reason detail = throwIO (PreparationFailure (FallbackReason reason detail))

fromDecision :: ReuseDecision a -> IO a
fromDecision (Reusable x) = pure x
fromDecision (Fallback reason) = throwIO (PreparationFailure reason)

prepare :: IO a -> IO (ReuseDecision a)
prepare io = (Reusable <$> io) `catches`
  [ Handler $ \(PreparationFailure reason) -> pure (Fallback reason),
    Handler $ \(e :: IOException) -> pure (Fallback (FallbackReason "snapshot creation failed" (T.pack (displayException e))))
  ]

withSnapshot :: SnapshotRun -> Release -> SnapshotInput -> (PreparedSnapshot -> IO a) -> IO (ReuseDecision a)
withSnapshot run version input convert = mask $ \restore -> do
  allocation <- prepare (acquireTree run)
  case allocation of
    Fallback reason -> pure (Fallback reason)
    Reusable tree -> flip finally (P.removeDirRecur tree) $ restore $ do
      result <- prepare (copyInputs (ledger run) tree version input)
      case result of
        Fallback reason -> pure (Fallback reason)
        Reusable snapshot -> Reusable <$> convert snapshot

windowsSnapshotPathFits :: FilePath -> Bool
windowsSnapshotPathFits = (< 260) . sum . map units
  where units c = if fromEnum c > 0xffff then 2 :: Int else 1

copyInputs :: AbsDir -> AbsDir -> Release -> SnapshotInput -> IO PreparedSnapshot
copyInputs base tree version input = do
  manifest <- newIORef Map.empty
  paths <- newIORef Map.empty
  completed <- newIORef Set.empty
  let mirror file = case stripProperPrefix base file of
        Nothing -> reject "snapshot input outside ledger" (T.pack (toFilePath file))
        Just rel -> pure (tree </> [reldir|l|] </> rel)
      destination source target = do
#ifdef mingw32_HOST_OS
        unless (windowsSnapshotPathFits (toFilePath target)) $
          reject "overlong Windows snapshot path" (T.pack (toFilePath target))
#endif
        P.createDirIfMissing True (parent target)
        -- Source and cache directories can have different case or Unicode
        -- equivalence rules. Never overwrite a copy made for another spelling.
        previous <- Map.lookup target <$> readIORef paths
        collision <- case previous of
          Just original -> pure (toFilePath original /= toFilePath source)
          Nothing -> D.doesPathExist (toFilePath target)
        when collision $ reject "snapshot filenames collide" (T.pack (toFilePath source))
      record source target digest = do
        previous <- Map.lookup source <$> readIORef manifest
        when (maybe False (/= digest) previous) $
          reject "dependency changed during snapshot" (T.pack (toFilePath source))
        modifyIORef' manifest (Map.insert source digest)
        modifyIORef' paths (Map.insert target source)
      copyBytes source target = do
        destination source target
        bytes <- BS.readFile (toFilePath source)
        BS.writeFile (toFilePath target) bytes
        record source target (digestBytes bytes)
        pure bytes
      validated check source target = do
        bytes <- copyBytes source target
        () <- fromDecision (check version source bytes)
        pure target
      copyConfig source = validated checkConfig source (tree </> [reldir|c|] </> filename source)
      copyRule ancestors source = do
        target <- mirror source
        identity <- D.canonicalizePath (toFilePath source)
        when (Set.member identity ancestors || Set.size ancestors >= 256) $ reject "cyclic or excessively deep rules include" (T.pack (toFilePath source))
        done <- Set.member (toFilePath source) <$> readIORef completed
        unless done $ do
          bytes <- copyBytes source target
          let scan = if Set.null ancestors then ruleIncludes else nestedRuleIncludes
          includes <- fromDecision (scan version source bytes)
          forM_ includes $ \spelling -> do
            child <- resolveRuleInclude base source spelling >>= fromDecision
            void $ copyRule (Set.insert identity ancestors) child
          modifyIORef' completed (Set.insert (toFilePath source))
        pure target
  copied <- case input of
    ConstructInput conf -> ConstructInput <$> traverse copyConfig conf
    RulesInput csv rules directives conf -> do
      dataCopy <- mirror csv
      destination csv dataCopy
      copyDigest csv dataCopy >>= record csv dataCopy
      rulesCopy <- copyRule Set.empty rules
      directivesCopy <- traverse (\source -> mirror source >>= validated checkDirectives source) directives
      RulesInput dataCopy rulesCopy directivesCopy <$> traverse copyConfig conf
  PreparedSnapshot copied <$> readIORef manifest <*> readIORef paths

-- hledger emits locations as "FILE:line:column:" (also "FILE:line:" and
-- "FILE: explanation"). Restrict replacement to its location fields.
-- Source excerpts following a Megaparsec gutter are user data.
translateDiagnostic :: PreparedSnapshot -> Text -> IO Text
translateDiagnostic snap diagnostic = T.intercalate "\n" <$> mapM translate (T.splitOn "\n" diagnostic)
  where
    translate line = do
      let (prefix, body) = splitHeader line
          candidates value = [ (T.take i value, T.drop i value)
                       | (i, ':') <- zip [0..] (T.unpack value)
                       , let next = T.take 1 (T.drop (i+1) value)
                       , T.null next || next == " " || T.all isDigit next ]
      if FP.isAbsolute (T.unpack body)
        then firstKnown prefix line (candidates body)
        else case T.stripPrefix "\"" body of
          Just quoted -> quotedPath prefix line (candidates quoted)
          Nothing -> translateRulePosition line
    -- Some config errors quote the entire location with Haskell string escaping.
    quotedPath _ line [] = pure line
    quotedPath prefix line ((token,suffix):rest) =
      case readMaybe (T.unpack ("\"" <> token <> "\"")) of
        Just name | FP.isAbsolute name -> do
          mapped <- lookupPath (T.pack name)
          case mapped of
            Just p -> pure (prefix <> T.dropEnd 1 (T.pack (show (toFilePath p))) <> suffix)
            Nothing -> quotedPath prefix line rest
        _ -> quotedPath prefix line rest
    -- 1.99.6 appends a second location to a field-assignment diagnostic.
    -- Keep arbitrary template/record text ahead of the final location intact.
    translateRulePosition line
      | any (`T.isPrefixOf` line) ["the ", "rule ", "    (overrides: "] || fieldLabel =
          let (before, body) = T.breakOnEnd "  (" line
              (fileColon, lineEnd) = T.breakOnEnd ":" body
              (digits, close) = T.span isDigit lineEnd
          in if T.null before || T.null fileColon || T.null digits || close `notElem` [")", "))"]
             then pure line
             else firstKnown before line [(T.dropEnd 1 fileColon, ":" <> lineEnd)]
      | otherwise = pure line
      where
        fieldLabel = case T.stripPrefix "  " line of
          Just field -> let (name, rest) = T.span isAlphaNum field
                        in not (T.null name) && ":" `T.isPrefixOf` rest
          Nothing -> False
    splitHeader line =
      let (spaces, rest) = T.span (== ' ') line
          (program, message) = splitPrefix ["hledger: ", "hledger.exe: "] rest
          (label, body) = splitPrefix ["Warning: ", "Error: ", "error: ", "in "] message
      in (spaces <> program <> label, body)
    splitPrefix prefixes value = case [(p,s) | p <- prefixes, Just s <- [T.stripPrefix p value]] of
      (parts:_) -> parts
      [] -> ("",value)
    firstKnown _ original [] = pure original
    firstKnown prefix original ((token, suffix):more) = do
      mapped <- lookupPath token
      case mapped of
        Just path -> pure (prefix <> T.pack (toFilePath path) <> suffix)
        Nothing -> firstKnown prefix original more
    lookupPath token = case parseAbsFile (T.unpack token) >>= (\p -> Map.lookup p (originalPaths snap)) of
      Just path -> pure (Just path)
      Nothing -> catchIOError (byIdentity (T.unpack token)) (const (pure Nothing))
    byIdentity name = do
      actual <- D.canonicalizePath name
      -- canonicalizePath resolves symlinked ancestors and native spellings.
      -- File IDs also identify a Windows short path when its spelling differs.
      source <- parseAbsFile actual
      targetId <- stampIdentity <$> statFile source
      matches <- filterM (\p -> (== targetId) . stampIdentity <$> statFile p) (Map.keys (originalPaths snap))
      pure $ case matches of
        [p] -> Map.lookup p (originalPaths snap)
        _ -> Nothing