clafer-0.5.8: src/Language/Clafer/Import.hs
{-# LANGUAGE ScopedTypeVariables #-}
{-
Copyright (C) 2026 Michal Antkiewicz <http://gsd.uwaterloo.ca>
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
of the Software, and to permit persons to whom the Software is furnished to do
so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-}
{- | Models made of several files: a model imports other files with the option
@--import=FILE@, in its @//# OPTIONS@ directive or on the command line. The imported
files are compiled with the model as one model, with one namespace, before it and
each only once; their options are combined with the model's ('combineOptions').
A model and its imports are local files or URLs (@http://@, @https://@). An import is
relative to the importing file: to its folder, or to its URL. A model read from a URL
may only import URLs.
An import must be a @.cfr@ file: a URL's path must end with @.cfr@, also after a redirect, and
a URL served as HTML, JSON or XML is refused. Then a model, which may come from anyone (e.g.,
in a web tool), cannot read other files or web pages, whose text the errors would show.
-}
module Language.Clafer.Import
( ModelFile (..)
, Location (..)
, readModel
, location
, importLocation
, fileOptions
, combineOptions
, modelText
, modelPosition
, fileStarts
, isPublicAddress
, isModelPath
, isModelContentType
) where
import Control.Exception (IOException, bracketOnError, catch, displayException, fromException, try)
import Control.Monad
import Control.Monad.Except
import Control.Monad.State
import qualified Data.ByteString as B
import qualified Data.ByteString.Char8 as BC
import Data.IORef
import Data.Char (isSpace, toLower)
import Data.List (dropWhileEnd, find, intercalate, isPrefixOf, isSuffixOf, nub)
import qualified Data.Text as T
import qualified Data.Text.Encoding as TE
import Data.Text.Encoding.Error (lenientDecode)
import Data.Default (def)
import qualified Network.Connection as NC
import Network.HTTP.Client (BodyReader, HttpException (..), HttpExceptionContent (..), Manager, ManagerSettings (..), brRead, getOriginalRequest, getUri, managerSetProxy, newManager, noProxy, parseUrlThrow, responseBody, responseHeaders, responseStatus, responseTimeoutMicro, withResponse)
import qualified Network.HTTP.Client as HC (path)
import Network.HTTP.Client.Internal (Connection, makeConnection)
import Network.HTTP.Client.TLS (newTlsManager, tlsManagerSettings)
import Network.Socket (AddrInfo (..), Family (AF_INET), HostAddress, SockAddr (..), SocketType (Stream), close, connect, defaultHints, getAddrInfo, hostAddressToTuple, socket)
import Network.HTTP.Types.Header (hContentType)
import Network.HTTP.Types.Status (statusCode, statusMessage)
import Network.URI (URI (..), escapeURIString, isUnescapedInURI, parseAbsoluteURI, parseRelativeReference, relativeTo, unEscapeString, uriToString)
import System.Console.CmdArgs (cmdArgsValue)
import System.Console.CmdArgs.Explicit (process)
import System.Directory (canonicalizePath, doesFileExist)
import System.FilePath (isAbsolute, normalise, splitDirectories, takeDirectory, takeFileName, (</>))
import qualified System.IO as IO
import System.IO.Error (ioeGetErrorString)
import Language.Clafer.ClaferArgs
import Language.Clafer.SplitJoin (splitArgs)
-- | A file of a model and its text, which ends with a newline.
data ModelFile = ModelFile
{ modelFilePath :: FilePath -- ^ its path or URL, as shown in messages
, modelFileText :: String
} deriving (Eq, Show)
-- | Where a file of a model is: a local path, or a URL (http or https).
data Location = Local FilePath | Remote URI
deriving (Eq, Show)
-- | The location of a model file name: a URL with @http://@ or @https://@, otherwise a local
-- path (a @file://@ URL is its path).
location :: String -> Location
location name = case parseAbsoluteURI name of
Just uri | uriScheme uri `elem` ["http:", "https:"] -> Remote uri
_ | "file://" `isPrefixOf` name -> Local $ unEscapeString $ drop 7 name
| otherwise -> Local name
-- | The location of an import, relative to the importing file: to its folder, or to its URL.
-- A file read from a URL may only import URLs.
importLocation :: Location -> String -> Either String Location
importLocation importer name = case (location name, importer) of
(Remote uri, _) -> Right $ Remote uri
(Local path, Local importerPath) -> Right $ Local $ normalise $ takeDirectory importerPath </> path
(Local path, Remote base)
| "file://" `isPrefixOf` name || isAbsolute path -> Left $ "a model read from a URL cannot import the local file " ++ name
| otherwise -> case parseRelativeReference (escapeURIString isUnescapedInURI path) of
Just reference -> Right $ Remote $ reference `relativeTo` base
Nothing -> Left $ "the import " ++ name ++ " is not a valid relative URL"
-- | How a location is shown in messages.
showLocation :: Location -> String
showLocation (Local path) = path
showLocation (Remote uri) = uriToString id uri ""
-- | Reads the model given by the arguments (the file or URL 'file', or the standard input if it
-- is empty) and the files it imports, transitively. Returns the arguments combined with the
-- options of all files (the given arguments take precedence) and the files: imported files
-- before the files importing them, each once, and the model last. For a model read from a
-- URL, 'file' becomes the last segment of the URL's path, which names the outputs.
readModel :: ClaferArgs -> IO (Either String (ClaferArgs, [ModelFile]))
readModel args' = do
manager <- newIORef Nothing -- created at the first URL
roots <- mapM canonicalizePath (import_root args')
runExceptT $ do
let root = file args'
rootLocation = location root
-- the name of the model in messages
rootName = if null root then "<stdin>" else root
-- the location of the model for its imports: where its text is, or --import-base
baseLocation = if null (import_base args') then rootLocation else location (import_base args')
fetch = readLocation (public_imports args') manager
text <- withNewline <$> if null root then liftIO getContents else fetch False "" rootLocation
rootOptions <- liftEither $ either (Left . inFile rootName) Right $ fileOptions text
rootImports <- liftEither $ either (Left . inFile rootName) Right $ mapM (importLocation baseLocation) (imports rootOptions)
rootKey <- if null root then return [] else (:[]) <$> key baseLocation
imported <- flip execStateT ([], []) $
forM_ (map location (imports args') ++ rootImports) $ visit fetch roots [ (k, rootName) | k <- rootKey ]
options <- liftEither $ combineOptions (snd imported) (rootName, rootOptions)
let localName = case rootLocation of
Local path -> path
Remote uri -> case takeFileName (unEscapeString (uriPath uri)) of { "" -> "model.cfr"; name -> name }
return ( (withDefaultMode $ mergeArgs args' options){file = localName}
, map fst (snd imported) ++ [ModelFile (if null root then rootName else showLocation rootLocation) text] )
where
-- the stack of the importing files (key and name), to report cycles
visit :: (Bool -> String -> Location -> ExceptT String IO String) -> [FilePath] -> [(String, String)] -> Location -> StateT ([String], [(ModelFile, ClaferArgs)]) (ExceptT String IO) ()
visit fetch roots stack loc = do
let importer = maybe "the command line" snd $ safeHead stack
name = showLocation loc
-- before anything is known about the file, e.g., whether it exists
unless (isModelPath $ locationPath loc) $
throwError $ inFile importer $ "the import " ++ name ++ " is not a .cfr file"
lift $ insideRoots importer roots loc
k <- lift $ either (const $ return name) return =<< liftIO (runExceptT (key loc))
when (k `elem` map fst stack) $
throwError $ "Import cycle: " ++ intercalate " imports " (reverse (name : map snd (takeWhile ((/= k) . fst) stack) ++ [name]))
done <- gets fst
unless (k `elem` done) $ do
text <- withNewline <$> lift (fetch True importer loc)
options <- either (throwError . inFile name) return $ fileOptions text
locations <- either (throwError . inFile name) return $ mapM (importLocation loc) (imports options)
forM_ locations $ visit fetch roots ((k, name) : stack)
modify $ \(done', files) -> (k : done', files ++ [(ModelFile name text, options)])
-- identifies a file: its real path, or its URL
key :: Location -> ExceptT String IO String
key (Local path) = do
exists <- liftIO $ doesFileExist path
if exists then liftIO (canonicalizePath path) else return path
key loc@(Remote _) = return $ showLocation loc
safeHead xs = case xs of { x : _ -> Just x; [] -> Nothing }
locationPath (Local path) = path
locationPath (Remote uri) = unEscapeString $ uriPath uri
withDefaultMode a = if null (mode a) then a{mode = [Alloy]} else a
-- | Whether a path names a Clafer model: it ends with @.cfr@.
isModelPath :: String -> Bool
isModelPath = (".cfr" `isSuffixOf`)
-- | Whether a URL's Content-Type may be a Clafer model: not HTML, JSON or XML, which web pages
-- and services are served as, whatever their URLs.
isModelContentType :: Maybe B.ByteString -> Bool
isModelContentType Nothing = True
isModelContentType (Just value) = not $
mediaType `elem` ["text/html", "application/xhtml+xml", "application/json", "text/json", "application/xml", "text/xml"]
|| any (`isSuffixOf` mediaType) ["+json", "+xml"]
where mediaType = map toLower $ dropWhileEnd isSpace $ dropWhile isSpace $ takeWhile (/= ';') $ BC.unpack value
-- | With import roots (--import-root), a local file must be inside one of them, after resolving
-- @..@ and symbolic links.
insideRoots :: String -> [FilePath] -> Location -> ExceptT String IO ()
insideRoots importer roots@(_ : _) (Local path) = do
real <- liftIO $ canonicalizePath path
unless (any (\r -> splitDirectories r `isPrefixOf` splitDirectories real) roots) $
throwError $ inFile importer $ "the import " ++ path ++ " is outside the folders from which files can be imported"
insideRoots _ _ _ = return ()
-- | The largest file that is downloaded.
maxDownload :: Int
maxDownload = 16 * 1024 * 1024
-- | Reads a file or downloads a URL, from a public address only if @public@; the importer names
-- the file importing it, for messages. An import, downloaded after redirects, must still be a
-- @.cfr@ file and not be served as a web page ('isModelContentType').
readLocation :: Bool -> IORef (Maybe Manager) -> Bool -> String -> Location -> ExceptT String IO String
readLocation _ _ _ importer (Local path) = do
exists <- liftIO $ doesFileExist path
unless exists $ throwError $ inFile importer $ "the file " ++ path ++ " does not exist"
liftIO $ IO.withFile path IO.ReadMode $ \h -> do
s <- IO.hGetContents h
length s `seq` return s
readLocation public managerRef isImport importer loc@(Remote uri) = do
manager <- liftIO $ readIORef managerRef >>= maybe (newManager' >>= \m -> writeIORef managerRef (Just m) >> return m) return
result <- liftIO $ try $ do
request <- parseUrlThrow $ showLocation loc
withResponse request manager $ \response -> do
-- the request of the response, after redirects
let final = getOriginalRequest response
contentType = lookup hContentType $ responseHeaders response
if isImport && not (isModelPath $ unEscapeString $ BC.unpack $ HC.path final)
then return $ Left $ "redirected to " ++ uriToString id (getUri final) "" ++ ", which is not a .cfr file"
else if isImport && not (isModelContentType contentType)
then return $ Left $ "it is served as " ++ maybe "" BC.unpack contentType ++ ", not as a Clafer model"
else maybe (Left $ "larger than " ++ show (maxDownload `div` (1024 * 1024)) ++ " MB") Right
<$> readBody 0 [] (responseBody response)
case result of
Right (Left problem) -> throwError $ inFile importer $ "cannot read " ++ showLocation loc ++ ": " ++ problem
Right (Right body) -> return $ T.unpack $ TE.decodeUtf8With lenientDecode body
Left (HttpExceptionRequest _ (StatusCodeException response _)) ->
throwError $ inFile importer $ "cannot read " ++ showLocation loc ++ ": HTTP "
++ show (statusCode $ responseStatus response) ++ " " ++ T.unpack (TE.decodeUtf8With lenientDecode $ statusMessage $ responseStatus response)
Left (HttpExceptionRequest _ (ConnectionFailure e)) -> throwError $ inFile importer $ "cannot read " ++ showLocation loc ++ ": "
++ maybe (displayException e) ioeGetErrorString (fromException e)
Left (HttpExceptionRequest _ content) -> throwError $ inFile importer $ "cannot read " ++ showLocation loc ++ ": " ++ show content
Left (InvalidUrlException _ reason) -> throwError $ inFile importer $ "invalid URL " ++ uriToString id uri "" ++ ": " ++ reason
where
newManager' = if public then newManager publicManagerSettings else newTlsManager
-- the body, unless it is larger than maxDownload
readBody :: Int -> [B.ByteString] -> BodyReader -> IO (Maybe B.ByteString)
readBody size chunks reader = do
chunk <- brRead reader
let size' = size + B.length chunk
if B.null chunk then return $ Just $ B.concat $ reverse chunks
else if size' > maxDownload then return Nothing
else readBody size' (chunk : chunks) reader
-- | Connections to public addresses only: the address is checked when connecting (also after a
-- redirect), and the connection is made to that address; no proxy.
publicManagerSettings :: ManagerSettings
publicManagerSettings = managerSetProxy noProxy tlsManagerSettings
{ managerRawConnection = do
raw <- managerRawConnection tlsManagerSettings
return $ \_ host port -> do
address <- publicAddress host
raw (Just address) host port
, managerTlsConnection = do
context <- NC.initConnectionContext
return $ \_ host port -> do
address <- publicAddress host
tlsConnection context address host port
, managerResponseTimeout = responseTimeoutMicro 30000000
}
-- | A public IPv4 address of the host, or an error.
publicAddress :: String -> IO HostAddress
publicAddress host = do
infos <- getAddrInfo (Just defaultHints{ addrFamily = AF_INET, addrSocketType = Stream }) (Just host) Nothing
case find isPublicAddress [ a | SockAddrInet _ a <- map addrAddress infos ] of
Just address -> return address
Nothing -> ioError $ userError $ "the host " ++ host ++ " has no public address"
-- | A TLS connection to the address, for the host.
tlsConnection :: NC.ConnectionContext -> HostAddress -> String -> Int -> IO Connection
tlsConnection context address host port =
bracketOnError (socket AF_INET Stream 0) close $ \sock -> do
connect sock (SockAddrInet (fromIntegral port) address)
conn <- NC.connectFromSocket context sock NC.ConnectionParams
{ NC.connectionHostname = host
, NC.connectionPort = fromIntegral port
, NC.connectionUseSecure = Just def -- validates the certificate for the host
, NC.connectionUseSocks = Nothing
}
makeConnection (NC.connectionGetChunk conn) (NC.connectionPut conn)
(NC.connectionClose conn `catch` \(_ :: IOException) -> return ())
-- | Whether an IPv4 address is public: not unspecified, loopback, private, shared, link-local,
-- reserved, benchmarking, multicast or broadcast.
isPublicAddress :: HostAddress -> Bool
isPublicAddress address = not $ case hostAddressToTuple address of
(0, _, _, _) -> True
(10, _, _, _) -> True
(100, b, _, _) | b >= 64 && b < 128 -> True
(127, _, _, _) -> True
(169, 254, _, _) -> True
(172, b, _, _) | b >= 16 && b < 32 -> True
(192, 0, 0, _) -> True
(192, 0, 2, _) -> True
(192, 168, _, _) -> True
(198, b, _, _) | b == 18 || b == 19 -> True
(198, 51, 100, _) -> True
(203, 0, 113, _) -> True
(a, _, _, _) -> a >= 224
inFile :: FilePath -> String -> String
inFile path message = (if null path then "" else path ++ ": ") ++ message
withNewline :: String -> String
withNewline s = if null s || last s == '\n' then s else s ++ "\n"
-- | The options of a model's @//# OPTIONS@ directive, its first line; the defaults without one.
fileOptions :: String -> Either String ClaferArgs
fileOptions model
| "//# OPTIONS " `isPrefixOf` model =
either (Left . ("//# OPTIONS: " ++)) (Right . withExplicit . cmdArgsValue) $
process claferMode $ splitArgs $ filter (/= '\r') $ drop 12 $ takeWhile (/= '\n') model
| otherwise = Right claferDefaults
{- | Combines the options of the imported files and of the model:
* the largest @--maxint@, @--decimal-digits@, @--tl@ and @--timeout-analysis@, the smallest @--minint@;
* all modes (@-m@) and imports;
* @-k@, @-c@, @--check-afm@ and @-i@ if any file has them, @-f@ only if all files have it;
* the layout (@-l@, @--nl@) and the scope strategy (@--ss@) must be the same in all files,
since the files are parsed together;
* the options of the compilation rather than of the model (e.g., @-s@, @-o@) are only allowed
in the model, not in an imported file;
* the options that run tools or restrict the imports (@-v@, @--tooldir@, @--import-root@,
@--public-imports@, @--import-base@) are only allowed on the command line: a model, which may
come from anyone, cannot set them for itself.
-}
combineOptions :: [(ModelFile, ClaferArgs)] -> (FilePath, ClaferArgs) -> Either String ClaferArgs
combineOptions imported (root, rootOptions) = do
forM_ ((root, rootOptions) : files) $ \(path, options) ->
case [ name' | (name', isChanged) <- commandLineOptions, isChanged options ] of
[] -> return ()
names' -> Left $ inFile path $ theOptions names' ++ " can only be given on the command line, not in //# OPTIONS"
forM_ imported $ \(ModelFile path _, options) ->
case [ name' | (name', isChanged) <- runOptions, isChanged options ] of
[] -> return ()
names' -> Left $ inFile path $ theOptions names'
++ " can only be given in the compiled model or on the command line, not in an imported file"
forM_ sameOptions $ \(name', isDifferent) ->
case [ path | (path, options) <- files, isDifferent options rootOptions ] of
[] -> return ()
paths -> Left $ "The option " ++ name' ++ " must be the same in all files of the model: it differs in "
++ intercalate ", " paths ++ " from " ++ root
return rootOptions
{ mode = nub $ concatMap mode all'
, imports = nub $ concatMap imports all'
, maxint = maximumOf $ map maxint all'
, minint = minimumOf $ map minint all'
, decimal_digits = maximumOf $ map decimal_digits all'
, trace_len = maximum $ map trace_len all'
, timeout_analysis = maximum $ map timeout_analysis all'
, keep_unused = any keep_unused all'
, check_duplicates = any check_duplicates all'
, afm = any afm all'
, flatten_inheritance = any flatten_inheritance all'
, skip_resolver = all skip_resolver all'
}
where
files = [ (path, options) | (ModelFile path _, options) <- imported ]
all' = rootOptions : map snd files
maximumOf xs = case [ x | Just x <- xs ] of { [] -> Nothing; ys -> Just (maximum ys) }
minimumOf xs = case [ x | Just x <- xs ] of { [] -> Nothing; ys -> Just (minimum ys) }
-- the options of the compilation, which differ from the defaults
runOptions =
[ ("-o", changed console_output), ("-s", changed no_stats)
, ("-a", changed alloy_mapping), ("--self-contained", changed self_contained)
, ("--add-graph", changed add_graph), ("--sr", changed show_references), ("--add-comments", changed add_comments)
, ("--meta-data", changed meta_data), ("--ecore2clafer", changed ecore2clafer), ("a model file", changed file) ]
theOptions [name'] = "the option " ++ name'
theOptions names' = "the options " ++ intercalate ", " names'
-- the options that run tools or restrict the imports, which a model cannot set for itself
commandLineOptions =
[ ("-v", changed validate), ("--tooldir", changed tooldir), ("--import-root", changed import_root)
, ("--public-imports", changed public_imports), ("--import-base", changed import_base) ]
changed :: Eq a => (ClaferArgs -> a) -> ClaferArgs -> Bool
changed f options = f options /= f claferDefaults
sameOptions =
[ ("-l", differs no_layout), ("--nl", differs new_layout), ("--ss", differs scope_strategy) ]
differs :: Eq a => (ClaferArgs -> a) -> ClaferArgs -> ClaferArgs -> Bool
differs f a b = f a /= f b
-- | The text of the complete model: the files in order.
modelText :: [ModelFile] -> String
modelText = concatMap modelFileText
-- | The files and the lines of the complete model where they start (lines from 1).
fileStarts :: [ModelFile] -> [(Integer, FilePath)]
fileStarts files = zip (scanl (+) 1 [ fromIntegral $ length $ filter (== '\n') text | ModelFile _ text <- files ]) (map modelFilePath files)
-- | The file and the line in it of a line of the complete model (lines from 1).
modelPosition :: [ModelFile] -> Integer -> (FilePath, Integer)
modelPosition files line = go 1 files
where
go _ [] = ("", line)
go start (ModelFile path text : rest)
| null rest || line < start + size = (path, line - start + 1)
| otherwise = go (start + size) rest
where size = fromIntegral $ length $ filter (== '\n') text