diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,11 @@ # ChangeLog hie-bios +## 2020-12-16 - 0.7.2 + +* Faster Bios protocol [#271](https://github.com/mpickering/hie-bios/pull/271) +* Modify unreachable cabal website links [#259](https://github.com/mpickering/hie-bios/pull/259) +* Only take the last line of output in getRuntimeGhcX [#256](https://github.com/mpickering/hie-bios/pull/256) + ## 2020-09-01 - 0.7.1 * Add explicit type for stack.yaml location [#243](https://github.com/mpickering/hie-bios/pull/243) diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -209,7 +209,7 @@ This configuration should work in (almost) every standard project setup, since `cabal` finds the component associated to a given source file. However, due to an unfortunate [bug](https://github.com/haskell/cabal/issues/6622), this fails on some files with `cabal` versions older than `3.4`. So, to make your project loadable by older `cabal` versions, you can specify a component to load. -A [component](https://cabal.readthedocs.io/en/stable/nix-local-build.html?highlight=component#cabal-v2-build) is roughly speaking a library, executable, test-suite or benchmark in `cabal`. +A [component](https://cabal.readthedocs.io/en/latest/nix-local-build.html?highlight=component#cabal-v2-build) is roughly speaking a library, executable, test-suite or benchmark in `cabal`. The `hie.yaml` file looks like this: ```yaml @@ -228,7 +228,7 @@ component: "lib:hie-bios" ``` -The component syntax `"lib:hie-bios"` refers to the library of the package `hie-bios`. For a complete reference of the component syntax, we refer to the [documentation](https://cabal.readthedocs.io/en/stable/nix-local-build.html?highlight=component#cabal-v2-build). +The component syntax `"lib:hie-bios"` refers to the library of the package `hie-bios`. For a complete reference of the component syntax, we refer to the [documentation](https://cabal.readthedocs.io/en/latest/nix-local-build.html?highlight=component#cabal-v2-build). Note that `cabal` and `stack` have different ways of specifying their components. @@ -560,29 +560,23 @@ - default.nix ``` -For the `Bios` cradle type, there is an optional field for specifying a program -or shell command to obtain cradle dependencies from: +For the `Bios` cradle type, the newline-separated cradle dependencies must be written out +to the file specified by the `HIE_BIOS_DEPS` environment variable. +Previous versions implemented a different mechanism for collecting cradle dependencies +by means of a second program/shell field. This is still supported for backwards +compatibility: + ```yaml cradle: bios: - program: ./flags.sh dependency-program: ./dependency.sh ``` ```yaml cradle: bios: - shell: build-tool flags $HIE_BIOS_ARG > $HIE_BIOS_OUTPUT dependency-shell: build-tool dependencies $HIE_BIOS_ARG > $HIE_BIOS_OUTPUT ``` - -The dependency program or command is executed in a similar fashion to the bios -program, using the environment variables HIE_BIOS_ARG and HIE_BIOS_OUTPUT to -communicate respectively the path of the source file being loaded, and the path -where the set of dependency paths should be written, one per line, -and relative to the root of the cradle, not to the location of the program. - -Programs and shell commands for flags and dependencies can be mixed and matched. ## Configuration specification diff --git a/hie-bios.cabal b/hie-bios.cabal --- a/hie-bios.cabal +++ b/hie-bios.cabal @@ -1,6 +1,6 @@ Cabal-Version: 2.2 Name: hie-bios -Version: 0.7.1 +Version: 0.7.2 Author: Matthew Pickering Maintainer: Matthew Pickering License: BSD-3-Clause @@ -19,6 +19,10 @@ tests/configs/*.yaml tests/projects/symlink-test/a/A.hs tests/projects/symlink-test/hie.yaml + tests/projects/deps-bios-new/A.hs + tests/projects/deps-bios-new/B.hs + tests/projects/deps-bios-new/hie-bios.sh + tests/projects/deps-bios-new/hie.yaml tests/projects/multi-direct/A.hs tests/projects/multi-direct/B.hs tests/projects/multi-direct/hie.yaml @@ -142,8 +146,8 @@ autogen-modules: Paths_hie_bios Build-Depends: base >= 4.8 && < 5, - aeson >= 1.4.1 && < 2, - base16-bytestring >= 0.1.1 && < 0.2, + aeson >= 1.4.5 && < 2, + base16-bytestring >= 0.1.1 && < 1.1, bytestring >= 0.10.8 && < 0.11, deepseq >= 1.4.3 && < 1.5, containers >= 0.5.10 && < 0.7, @@ -160,7 +164,7 @@ unix-compat >= 0.5.1 && < 0.6, unordered-containers >= 0.2.9 && < 0.3, vector >= 0.12.0 && < 0.13, - yaml >= 0.8.32 && < 0.12, + yaml >= 0.10.0 && < 0.12, hslogger >= 1.2 && < 1.4, file-embed >= 0.0.11 && < 1, conduit >= 1.3 && < 2, diff --git a/src/HIE/Bios/Cradle.hs b/src/HIE/Bios/Cradle.hs --- a/src/HIE/Bios/Cradle.hs +++ b/src/HIE/Bios/Cradle.hs @@ -18,7 +18,7 @@ , isDefaultCradle , isOtherCradle , getCradle - , readProcessWithOutputFile + , readProcessWithOutputs , readProcessWithCwd , makeCradleResult ) where @@ -33,6 +33,7 @@ import HIE.Bios.Config import HIE.Bios.Environment (getCacheDir) import System.Directory hiding (findFile) +import Control.Monad.Trans.Cont import Control.Monad.Trans.Maybe import System.FilePath import Control.Monad @@ -59,6 +60,8 @@ import Data.Maybe (fromMaybe, maybeToList) import GHC.Fingerprint (fingerprintString) +hie_bios_output :: String +hie_bios_output = "HIE_BIOS_OUTPUT" ---------------------------------------------------------------- -- | Given root\/foo\/bar.hs, return root\/hie.yaml, or wherever the yaml file was found. @@ -357,10 +360,10 @@ biosDepsAction :: LoggingFunction -> FilePath -> Maybe Callable -> FilePath -> IO [FilePath] biosDepsAction l wdir (Just biosDepsCall) fp = do biosDeps' <- callableToProcess biosDepsCall (Just fp) - (ex, sout, serr, args) <- readProcessWithOutputFile l wdir biosDeps' + (ex, sout, serr, [(_, args)]) <- readProcessWithOutputs [hie_bios_output] l wdir biosDeps' case ex of ExitFailure _ -> error $ show (ex, sout, serr) - ExitSuccess -> return args + ExitSuccess -> return $ fromMaybe [] args biosDepsAction _ _ Nothing _ = return [] biosAction :: FilePath @@ -371,13 +374,17 @@ -> IO (CradleLoadResult ComponentOptions) biosAction wdir bios bios_deps l fp = do bios' <- callableToProcess bios (Just fp) - (ex, _stdo, std, res) <- readProcessWithOutputFile l wdir bios' - deps <- biosDepsAction l wdir bios_deps fp + (ex, _stdo, std, [(_, res),(_, mb_deps)]) <- + readProcessWithOutputs [hie_bios_output, "HIE_BIOS_DEPS"] l wdir bios' + + deps <- case mb_deps of + Just x -> return x + Nothing -> biosDepsAction l wdir bios_deps fp -- Output from the program should be written to the output file and -- delimited by newlines. -- Execute the bios action and add dependencies of the cradle. -- Removes all duplicates. - return $ makeCradleResult (ex, std, wdir, res) deps + return $ makeCradleResult (ex, std, wdir, fromMaybe [] res) deps callableToProcess :: Callable -> Maybe String -> IO CreateProcess callableToProcess (Command shellCommand) file = do @@ -491,8 +498,9 @@ cabalAction work_dir mc l fp = do withCabalWrapperTool ("ghc", []) work_dir $ \wrapper_fp -> do let cab_args = ["v2-repl", "--with-compiler", wrapper_fp, fromMaybe (fixTargetPath fp) mc] - (ex, output, stde, args) <- - readProcessWithOutputFile l work_dir (proc "cabal" cab_args) + (ex, output, stde, [(_,mb_args)]) <- + readProcessWithOutputs [hie_bios_output] l work_dir (proc "cabal" cab_args) + let args = fromMaybe [] mb_args case processCabalWrapperArgs args of Nothing -> do -- Best effort. Assume the working directory is the @@ -502,7 +510,7 @@ ["Failed to parse result of calling cabal" , unlines output , unlines stde - , unlines args]) + , unlines $ args]) Just (componentDir, final_args) -> do deps <- cabalCradleDependencies work_dir componentDir pure $ makeCradleResult (ex, stde, componentDir, final_args) deps @@ -615,16 +623,17 @@ let ghcProcArgs = ("stack", stackYamlProcessArgs syaml <> ["exec", "ghc", "--"]) -- Same wrapper works as with cabal withCabalWrapperTool ghcProcArgs work_dir $ \wrapper_fp -> do - (ex1, _stdo, stde, args) <- - readProcessWithOutputFile l work_dir $ + (ex1, _stdo, stde, [(_, mb_args)]) <- + readProcessWithOutputs [hie_bios_output] l work_dir $ stackProcess syaml $ ["repl", "--no-nix-pure", "--with-ghc", wrapper_fp] <> [ comp | Just comp <- [mc] ] (ex2, pkg_args, stdr, _) <- - readProcessWithOutputFile l work_dir $ + readProcessWithOutputs [hie_bios_output] l work_dir $ stackProcess syaml ["path", "--ghc-package-path"] let split_pkgs = concatMap splitSearchPath pkg_args pkg_ghc_args = concatMap (\p -> ["-package-db", p] ) split_pkgs + args = fromMaybe [] mb_args case processCabalWrapperArgs args of Nothing -> do -- Best effort. Assume the working directory is the @@ -776,49 +785,65 @@ e <- getEnvironment return $ Map.toList $ Map.delete "GHC_PACKAGE_PATH" $ Map.fromList e --- | Call a given process. --- * A special file is created for the process to write to, the process can discover the name of --- the file by reading the @HIE_BIOS_OUTPUT@ environment variable. The contents of this file is --- returned by the function. +type Outputs = [OutputName] +type OutputName = String + +-- | Call a given process with temp files for the process to write to. +-- * The process can discover the temp files paths by reading the environment. +-- * The contents of the temp files are returned by this function, if any. -- * The logging function is called every time the process emits anything to stdout or stderr. -- it can be used to report progress of the process to a user. -- * The process is executed in the given directory. -readProcessWithOutputFile - :: LoggingFunction -- ^ Output of the process is streamed into this function. +readProcessWithOutputs + :: Outputs -- ^ Names of the outputs produced by this process + -> LoggingFunction -- ^ Output of the process is streamed into this function. -> FilePath -- ^ Working directory. Process is executed in this directory. -> CreateProcess -- ^ Parameters for the process to be executed. - -> IO (ExitCode, [String], [String], [String]) -readProcessWithOutputFile l work_dir cp = do - old_env <- getCleanEnvironment + -> IO (ExitCode, [String], [String], [(OutputName, Maybe [String])]) +readProcessWithOutputs outputNames l work_dir cp = flip runContT return $ do + old_env <- liftIO getCleanEnvironment + output_files <- traverse (withOutput old_env) outputNames - withHieBiosOutput old_env $ \output_file -> do - -- Pipe stdout directly into the logger - let process = cp { env = Just - $ (hieBiosOutput, output_file) - : (fromMaybe old_env (env cp)), - cwd = Just work_dir - } + let process = cp { env = Just $ output_files ++ fromMaybe old_env (env cp), + cwd = Just work_dir + } -- Windows line endings are not converted so you have to filter out `'r` characters - let loggingConduit = (C.decodeUtf8 C..| C.lines C..| C.filterE (/= '\r') C..| C.map T.unpack C..| C.iterM l C..| C.sinkList) - (ex, stdo, stde) <- sourceProcessWithStreams process mempty loggingConduit loggingConduit - res <- withFile output_file ReadMode $ \handle -> do - hSetBuffering handle LineBuffering - !res <- force <$> hGetContents handle - return res + let loggingConduit = C.decodeUtf8 C..| C.lines C..| C.filterE (/= '\r') C..| C.map T.unpack C..| C.iterM l C..| C.sinkList + (ex, stdo, stde) <- liftIO $ sourceProcessWithStreams process mempty loggingConduit loggingConduit - return (ex, stdo, stde, lines (filter (/= '\r') res)) + res <- forM output_files $ \(name,path) -> + liftIO $ (name,) <$> readOutput path + return (ex, stdo, stde, res) + where - withHieBiosOutput :: [(String,String)] -> (FilePath -> IO a) -> IO a - withHieBiosOutput env' action = do - let mbHieBiosOut = lookup hieBiosOutput env' - case mbHieBiosOut of - Just file@(_:_) -> action file - _ -> withSystemTempFile "hie-bios" $ - \ file h -> hClose h >> action file + readOutput :: FilePath -> IO (Maybe [String]) + readOutput path = do + haveFile <- doesFileExist path + if haveFile + then withFile path ReadMode $ \handle -> do + hSetBuffering handle LineBuffering + !res <- force <$> hGetContents handle + return $ Just $ lines $ filter (/= '\r') res + else + return Nothing - hieBiosOutput = "HIE_BIOS_OUTPUT" + withOutput :: [(String,String)] -> OutputName -> ContT a IO (OutputName, String) + withOutput env' name = + case lookup name env' of + Just file@(_:_) -> ContT $ \action -> do + removeFileIfExists file + action (name, file) + _ -> ContT $ \action -> withSystemTempFile name $ \ file h -> do + hClose h + removeFileIfExists file + action (name, file) + +removeFileIfExists :: FilePath -> IO () +removeFileIfExists f = do + yes <- doesFileExist f + when yes (removeFile f) makeCradleResult :: (ExitCode, [String], FilePath, [String]) -> [FilePath] -> CradleLoadResult ComponentOptions makeCradleResult (ex, err, componentDir, gopts) deps = diff --git a/src/HIE/Bios/Environment.hs b/src/HIE/Bios/Environment.hs --- a/src/HIE/Bios/Environment.hs +++ b/src/HIE/Bios/Environment.hs @@ -309,5 +309,9 @@ anyToken = satisfy $ const True -- Used for clipping the trailing newlines on GHC output +-- Also only take the last line of output +-- (Stack's ghc output has a lot of preceding noise from 7zip etc) trim :: String -> String -trim = dropWhileEnd isSpace +trim s = case lines s of + [] -> s + ls -> dropWhileEnd isSpace $ last ls diff --git a/tests/BiosTests.hs b/tests/BiosTests.hs --- a/tests/BiosTests.hs +++ b/tests/BiosTests.hs @@ -183,6 +183,7 @@ = [ testCaseSteps "simple-bios" $ testDirectory isBiosCradle "./tests/projects/simple-bios" "B.hs" , testCaseSteps "simple-bios-ghc" $ testDirectory isBiosCradle "./tests/projects/simple-bios-ghc" "B.hs" , testCaseSteps "simple-bios-deps" $ testLoadCradleDependencies isBiosCradle "./tests/projects/simple-bios" "B.hs" (assertEqual "dependencies" ["hie-bios.sh", "hie.yaml"]) + , testCaseSteps "simple-bios-deps-new" $ testLoadCradleDependencies isBiosCradle "./tests/projects/deps-bios-new" "B.hs" (assertEqual "dependencies" ["hie-bios.sh", "hie.yaml"]) ] | otherwise = [] diff --git a/tests/projects/deps-bios-new/A.hs b/tests/projects/deps-bios-new/A.hs new file mode 100644 --- /dev/null +++ b/tests/projects/deps-bios-new/A.hs @@ -0,0 +1,1 @@ +module A where diff --git a/tests/projects/deps-bios-new/B.hs b/tests/projects/deps-bios-new/B.hs new file mode 100644 --- /dev/null +++ b/tests/projects/deps-bios-new/B.hs @@ -0,0 +1,3 @@ +module B where + +import A diff --git a/tests/projects/deps-bios-new/hie-bios.sh b/tests/projects/deps-bios-new/hie-bios.sh new file mode 100644 --- /dev/null +++ b/tests/projects/deps-bios-new/hie-bios.sh @@ -0,0 +1,5 @@ +echo "-Wall" >> $HIE_BIOS_OUTPUT +echo "A" >> $HIE_BIOS_OUTPUT +echo "B" >> $HIE_BIOS_OUTPUT +echo "hie-bios.sh" >> $HIE_BIOS_DEPS +echo "hie.yaml" >> $HIE_BIOS_DEPS diff --git a/tests/projects/deps-bios-new/hie.yaml b/tests/projects/deps-bios-new/hie.yaml new file mode 100644 --- /dev/null +++ b/tests/projects/deps-bios-new/hie.yaml @@ -0,0 +1,3 @@ +cradle: + bios: + program: ./hie-bios.sh