packages feed

hledger-flow-0.21.0: src/Hledger/Flow/Import/ShellSources.hs

{-# LANGUAGE CPP #-}
{-# LANGUAGE OverloadedStrings #-}

module Hledger.Flow.Import.ShellSources
  ( SourceCache, newSourceCache, SourceInput, sourceKey, hookSources ) where

import Control.Monad (foldM, unless, when)
import Data.Char (isAlpha, isAlphaNum, isAscii, isControl, isSpace)
import qualified Data.ByteString as BS
import qualified Data.ByteString.Char8 as B8
import Data.Foldable (toList)
import Data.Functor.Identity (runIdentity)
import Data.IORef
import Data.List (isPrefixOf)
import Data.Maybe (isJust)
import qualified Data.Map.Strict as Map
import qualified Data.Set as Set
import qualified Data.Text as T
import qualified Data.Text.Encoding as TE
import Hledger.Flow.Import.Fingerprint
import Hledger.Flow.PathHelpers (AbsFile)
import qualified Path as P
import ShellCheck.AST
import ShellCheck.ASTLib (getLiteralString)
import ShellCheck.Interface
import ShellCheck.Parser (parseScript)
import qualified System.Directory as D
import System.Environment (lookupEnv)
import qualified System.FilePath as F
import System.IO (IOMode(ReadMode), withBinaryFile)
#ifndef mingw32_HOST_OS
import qualified System.Posix.Files as Posix
#endif

-- Cache only syntax, never path resolution or filesystem observations.
type SourceCache = IORef (Map.Map Digest [Step])
newSourceCache :: IO SourceCache
newSourceCache = newIORef Map.empty

data SourceInput = SourceInput FilePath FileObservation [(String, Digest)] deriving (Eq, Show)

sourceKey :: SourceInput -> [(T.Text, T.Text)]
sourceKey (SourceInput target obs variables) =
  (T.pack target, digestHex (observationDigest obs)) :
  [("source variable " <> T.pack name, digestHex digest) | (name,digest) <- variables]

data Piece = Literal String | Variable Bool String
data Step = Include [Piece] | ShellStateChanged | Uncertain

hookSources :: SourceCache -> AbsFile -> FileObservation -> IO [SourceInput]
hookSources cache hook observed = do
  let file = P.toFilePath hook
  header <- withBinaryFile file ReadMode (fmap (B8.takeWhile (/= '\n')) . flip BS.hGet 256)
  case interpreter file header of
    Other -> pure []
    UnrecognisedShell -> uncertain file
    ShellScript -> do
      startup <- lookupEnv "BASH_ENV"
      when (maybe False (not . null) startup) (uncertain file)
      files <- newIORef []
      visits <- newIORef (0 :: Int)
      let visit parents path observation = do
            canonical <- D.canonicalizePath path
            when (Set.member canonical parents) (uncertain path)
            n <- atomicModifyIORef' visits (\i -> (i+1,i))
            when (n >= 128) (uncertain path)
            plan <- scriptSteps cache path (observationDigest observation)
            foldM (step (Set.insert canonical parents) path) True plan
          step _ _ _ ShellStateChanged = pure False
          step _ path _ Uncertain = uncertain path
          step parents path pristine (Include parts) = do
            -- Earlier commands can change working directory, variable bindings
            -- or the meaning of the source builtin.
            unless pristine (uncertain path)
            expanded <- mapM (expand path) parts
            let name = concatMap fst expanded
                variables = concatMap snd expanded
            unless ('/' `elem` name && not ("-" `isPrefixOf` name) &&
                    not (any isControl name)) (uncertain path)
            -- Test the original spelling before canonicalizing missing/../
            -- and symlink/../ components.
#ifdef mingw32_HOST_OS
            regular <- D.doesFileExist name
#else
            regular <- Posix.isRegularFile <$> Posix.getFileStatus name
#endif
            unless regular (uncertain path)
            target <- D.canonicalizePath name
            dep <- P.parseAbsFile target >>= observeFile
            modifyIORef' files (SourceInput target dep variables :)
            visit parents target dep
      _ <- visit Set.empty file observed
      reverse <$> readIORef files
  where
    uncertain f = ioError (userError ("cannot resolve shell source safely in " ++ f))
    expand _ (Literal s) = pure (s, [])
    expand f (Variable quoted name) = do
      value <- lookupEnv name
      case value of
        Just s | not (null s), quoted || not (any (\c -> isSpace c || c `elem` ("*?[" :: String)) s) ->
          pure (s, [(name, digestBytes (TE.encodeUtf8 (T.pack s)))])
        _ -> uncertain f

data Interpreter = ShellScript | Other | UnrecognisedShell

interpreter :: FilePath -> BS.ByteString -> Interpreter
#ifdef mingw32_HOST_OS
interpreter file _ | F.takeExtension file == ".ps1" = Other
#endif
interpreter _ header = case B8.stripPrefix "#!" header of
  Nothing -> if BS.all (\c -> c == 9 || c == 13 || c >= 32) header then UnrecognisedShell else Other
  Just line -> case words (B8.unpack line) of
    exe:args | F.takeFileName exe == "env" -> case args of
                 [name] | shell name -> ShellScript
                 _ | any shell args -> UnrecognisedShell
                 _ -> Other
             | shell exe -> if null args then ShellScript else UnrecognisedShell
    _ -> Other
  where shell p = F.takeFileName p `elem` ["bash","sh","dash"]

scriptSteps :: SourceCache -> FilePath -> Digest -> IO [Step]
scriptSteps cache path digest = do
  existing <- Map.lookup digest <$> readIORef cache
  case existing of
    Just plan -> pure plan
    Nothing -> do
      bytes <- BS.readFile path
      unless (digestBytes bytes == digest) (ioError (userError ("script changed while reading " ++ path)))
      let plan = case TE.decodeUtf8' bytes of
            Left _ -> [Uncertain]
            Right body ->
              let result = runIdentity $ parseScript newSystemInterface newParseSpec
                    { psScript = T.unpack body, psFilename = path, psShellTypeOverride = Just Bash, psIgnoreRC = True }
              in case prRoot result of
                Just root | all ((/= ErrorC) . cSeverity . pcComment) (prComments result) -> steps root
                _ -> [Uncertain]
      atomicModifyIORef' cache (\m -> (Map.insert digest plan m, ()))
      pure plan

-- A function or loop can execute later or repeatedly with different bindings.
steps :: Token -> [Step]
steps token = case token of
  T_Annotation _ _ t -> steps t
  T_Script _ _ ts -> concatMap steps ts
  T_Pipeline _ [] [t] -> steps t
  T_Redirecting _ [] t -> steps t
  T_AndIf _ a b | not (containsSource b) -> steps a ++ steps b
  T_OrIf _ a b | not (containsSource b) -> steps a ++ steps b
  T_SimpleCommand _ [] (cmd:args)
    | isSource cmd -> case args of
        path:rest | all sourceArgument rest, Just parts <- pieces False path -> [Include parts]
        _ -> [Uncertain]
    | Just name <- getLiteralString cmd, name `elem` ["exit","true","false",":"],
      all (isJust . getLiteralString) args -> []
    | getLiteralString cmd == Just "set",
      all (\a -> getLiteralString a `elem` map Just ["-e","-u","-eu","-euo","-o","pipefail"]) args -> []
  T_Function _ _ _ name body | name `notElem` ["source",".","set","exit","true","false",":","[","test"], not (containsSource body) -> []
  T_Condition _ _ (TC_Unary _ _ op arg) | op `elem` ["-z","-n"], isJust (pieces True arg) || positionalArgument arg -> []
  _ | containsSource token -> [Uncertain]
    | otherwise -> [ShellStateChanged]
  where
    -- Forwarded hook arguments are already captured by the hook key.
    sourceArgument t = positionalArgument t || maybe False (all literal) (pieces False t)
    literal (Literal _) = True
    literal _ = False
    positionalArgument (T_NormalWord _ [T_DoubleQuoted _ [T_DollarBraced _ _ name]]) =
      case getLiteralString name of
        Just "@" -> True
        Just n -> not (null n) && all (\c -> c >= '0' && c <= '9') n
        Nothing -> False
    positionalArgument _ = False

isSource :: Token -> Bool
isSource t = getLiteralString t `elem` [Just ".", Just "source"]

containsSource :: Token -> Bool
containsSource t@(OuterToken _ children) = case t of
  T_SimpleCommand _ _ (cmd:args)
    | isSource cmd || getLiteralString cmd == Just "eval" -> True
    | getLiteralString cmd `elem` [Just "builtin", Just "command"], any isSource args -> True
  _ -> any containsSource (toList children)

-- Parameter operators, command substitution, globs and tilde expansion need Bash.
pieces :: Bool -> Token -> Maybe [Piece]
pieces quoted token = case token of
  T_NormalWord _ ts -> concat <$> mapM (pieces quoted) ts
  T_DoubleQuoted _ ts -> concat <$> mapM (pieces True) ts
  T_SingleQuoted _ s -> Just [Literal s]
  T_Literal _ s | quoted || not (any (\c -> c `elem` ("~*?[]{}" :: String)) s) -> Just [Literal s]
  T_DollarBraced _ _ t | Just name@(c:_) <- getLiteralString t,
    isAscii c && (isAlpha c || c == '_'), all (\x -> isAscii x && (isAlphaNum x || x == '_')) name,
    not ("BASH" `isPrefixOf` name || name `elem` ["_", "PWD", "OLDPWD", "IFS", "LINENO", "RANDOM", "SRANDOM",
          "SECONDS", "SHLVL", "UID", "EUID", "PPID", "OPTIND", "OPTARG", "SHELLOPTS", "FUNCNAME",
          "DIRSTACK", "GROUPS", "PIPESTATUS", "EPOCHSECONDS", "EPOCHREALTIME"]) ->
        Just [Variable quoted name]
  _ -> Nothing