diff --git a/CHANGELOG.md b/CHANGELOG.md --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,3 +2,9 @@ - Initial Hackage release - Add CHANGELOG.md + +# 0.4.0 +- Add `--resolver` option to generate full stackage packages set +- Add Nix wrapper, see `nix/README.md` +- Upd support cabal2nix >2.5 +- Fix honor `--hackage-db` flag diff --git a/README.md b/README.md --- a/README.md +++ b/README.md @@ -5,7 +5,47 @@ `stackage2nix` converts a Stack file into a Nix Haskell packages set. It creates LTS Stackage packages set, and applies appropriate overrides on top of it. +## Install + +You can install `stackage2nix` from the Nix expression: + ``` +nix-env -i -f ./nix/stackage2nix +``` + +It provides pre-configured wrapper around the raw executable with runtime +`PATH` and all auxiliary flags set up. + +## Build project + +Generate derivations from `stack.yaml` config using Nix wrapper: + +``` bash +stackage2nix ./stack.yaml +``` + +if you're using the raw executable, you should supply additional flags. See +section 'Flags' below for details. + +This command will result in a Haskell packages set, similar to +`pkgs.haskell.packages.`, containing only packages that are required +to build targets listed in `stackage.yaml`. To build a package run: + +``` bash +nix-build -A my-package +``` + +## Build stackage + +Generate complete Stackage packages set from resolver: + +``` bash +stackage2nix --resolver lts-9.0 +``` + +## Flags + +``` stackage2nix \ --lts-haskell "$LTS_HASKELL_REPO" \ --all-cabal-hashes "$ALL_CABAL_HASHES_REPO" \ @@ -28,12 +68,15 @@ nix-build -A ``` +See also the [blog post](https://blog.typeable.io/posts/2017-08-24-stackage2nix.html) +about history and motivation behind the project. + ## Runtime dependencies - `nix-env` is required to be on PATH by the [distribution-nixpkgs](https://hackage.haskell.org/package/distribution-nixpkgs) dependency -- `nix-prefetch-git` is required on PATH if you have git dependencies in +- `nix-prefetch-scripts` is required on PATH if you have git dependencies in `stack.yaml` ## Override result derivation @@ -59,8 +102,13 @@ For more complex overrides and detailed information on how to work with Haskell packages in Nix, see Nixpkgs manual [User’s Guide to the Haskell Infrastructure](http://nixos.org/nixpkgs/manual/#users-guide-to-the-haskell-infrastructure) +## Tests -## Examples +Integration tests that build stackage2nix form different stack configs: -For other examples of `stackage2nix` usage, see [4e6/stackage2nix-examples](https://github.com/4e6/stackage2nix-examples) repository. -It verifies `stackage2nix` by running it on different public projects. +``` +STACKAGE_REPO= \ +ALL_CABAL_HASHES= \ +STACK_FILE=stack-ghc-7103.yaml \ +./ci-stackage2nix +``` diff --git a/src/Distribution/Nixpkgs/Haskell/FromStack.hs b/src/Distribution/Nixpkgs/Haskell/FromStack.hs --- a/src/Distribution/Nixpkgs/Haskell/FromStack.hs +++ b/src/Distribution/Nixpkgs/Haskell/FromStack.hs @@ -26,19 +26,22 @@ , targetCompiler :: CompilerInfo } data PackageConfig = PackageConfig - { enableCheck :: Bool - , enableHaddock :: Bool } + { enableCheck :: Bool + , enableHaddock :: Bool } removeTests :: GenericPackageDescription -> GenericPackageDescription removeTests gd = gd { condTestSuites = [] } +removeBenches :: GenericPackageDescription -> GenericPackageDescription +removeBenches gd = gd { condBenchmarks = [] } + planDependencies :: PackagePlan -> [Dependency] planDependencies = map makeDependency . Map.toList . sdPackages . ppDesc where makeDependency (name, depInfo) = Dependency name (diRange depInfo) -buildNodeM :: PackageSetConfig -> PackageConfig -> PackageName -> PackagePlan -> IO Node -buildNodeM conf pconf name plan = do +buildNode :: PackageSetConfig -> PackageConfig -> PackageName -> PackagePlan -> IO Node +buildNode conf pconf name plan = do let cabalHashes = maybe mempty cfiHashes $ ppCabalFileInfo plan mGitSha1 = Map.lookup "GitSHA1" cabalHashes @@ -50,15 +53,16 @@ let constraints = ppConstraints plan testsEnabled = - enableCheck pconf && pcTests constraints == ExpectSuccess haddocksEnabled = - enableHaddock pconf && pcHaddocks constraints == ExpectSuccess && not (Set.null (sdModules (ppDesc plan))) configureTests | pcTests constraints == Don'tBuild = removeTests | otherwise = id + configureBenches + | pcBenches constraints == Don'tBuild = removeBenches + | otherwise = id flags = Map.toList (pcFlagOverrides constraints) (descr, missingDeps) = finalizeGenericPackageDescription (haskellResolver conf) @@ -66,7 +70,7 @@ (targetCompiler conf) flags (planDependencies plan) - (configureTests (pkgCabal pkg)) + (configureBenches . configureTests $ pkgCabal pkg) genericDrv = fromPackageDescription (haskellResolver conf) (nixpkgsResolver conf) @@ -83,11 +87,14 @@ testsEnabled && not (null missingDeps) && setOf (folded . to depName) missingDeps `hasIntersection` testDeps descr - in - genericDrv - & src .~ pkgSource pkg + in finalizePackage pkg pconf + $ genericDrv & doCheck .~ testsEnabled & runHaddock .~ haddocksEnabled & metaSection . Nix.broken .~ brokenEnabled - -- TODO: remove after Nixos/Nixpkgs #27196 released - & enableSeparateDataOutput .~ False + +finalizePackage :: Package -> PackageConfig -> Derivation -> Derivation +finalizePackage pkg pconf drv = drv + & src .~ pkgSource pkg + & doCheck &&~ enableCheck pconf + & runHaddock &&~ enableHaddock pconf diff --git a/src/Distribution/Nixpkgs/Haskell/FromStack/Package.hs b/src/Distribution/Nixpkgs/Haskell/FromStack/Package.hs --- a/src/Distribution/Nixpkgs/Haskell/FromStack/Package.hs +++ b/src/Distribution/Nixpkgs/Haskell/FromStack/Package.hs @@ -1,8 +1,9 @@ module Distribution.Nixpkgs.Haskell.FromStack.Package where import Control.Lens -import Data.Graph (Graph, Vertex) import Data.Foldable as F +import Data.Function (on) +import Data.Graph (Graph, Vertex) import Data.Maybe import Data.Monoid import Data.Ord as O @@ -21,12 +22,21 @@ import qualified Data.Set as Set data Node = Node - { _nodeDerivation :: Derivation - , _nodeTestDepends :: Set.Set String - , _nodeOtherDepends :: Set.Set String } + { _nodeDerivation :: Derivation + , _nodeTestDepends :: Set.Set String + , _nodeBenchmarkDepends :: Set.Set String + , _nodeExecutableDepends :: Set.Set String + , _nodeSetupDepends :: Set.Set String + , _nodeOtherDepends :: Set.Set String } makeLenses ''Node +instance Eq Node where + (==) = (==) `on` nodeName + +instance Ord Node where + compare = compare `on` nodeName + mkNode :: Derivation -> Node mkNode _nodeDerivation = Node{..} where @@ -34,17 +44,27 @@ . Set.filter isFromHackage $ view (s . (haskell <> tool)) _nodeDerivation _nodeTestDepends = haskellDependencies testDepends + _nodeBenchmarkDepends = haskellDependencies benchmarkDepends + _nodeExecutableDepends = haskellDependencies executableDepends + _nodeSetupDepends = haskellDependencies setupDepends _nodeOtherDepends = haskellDependencies (executableDepends <> libraryDepends) nodeName :: Node -> String nodeName = unPackageName . packageName . view (nodeDerivation . pkgid) +nodeCycleDepends :: Node -> Set.Set String +nodeCycleDepends = _nodeTestDepends <> _nodeOtherDepends + nodeDepends :: Node -> Set.Set String -nodeDepends = _nodeTestDepends <> _nodeOtherDepends +nodeDepends = _nodeTestDepends + <> _nodeOtherDepends + <> _nodeBenchmarkDepends + <> _nodeExecutableDepends + <> _nodeSetupDepends findCycles :: [Node] -> [[Node]] findCycles nodes = mapMaybe cyclic $ - Graph.stronglyConnComp [(node, nodeName node, Set.toList $ nodeDepends node) | node <- nodes] + Graph.stronglyConnComp [(node, nodeName node, Set.toList $ nodeCycleDepends node) | node <- nodes] where cyclic (Graph.AcyclicSCC _) = Nothing cyclic (Graph.CyclicSCC c) = Just c @@ -68,12 +88,13 @@ buildNodeGraph nodes = Graph.graphFromEdges [(node, nodeName node, Set.toList $ nodeDepends node) | node <- nodes] -reachableDependencies :: [Node] -> [Node] -> [Node] -reachableDependencies keyNodes nodes = view _1 . fromVertex <$> reachableVertices +reachableDependencies :: [Node] -> [Node] -> Set.Set Node +reachableDependencies keyNodes nodes = Set.map (view _1 . fromVertex) reachableVerticesS where (graph, fromVertex, fromKey) = buildNodeGraph nodes keys = mapMaybe (fromKey . nodeName) keyNodes - reachableVertices = concatMap F.toList $ Graph.dfs graph keys + reachableVerticesS = F.foldr1 Set.union + $ (Set.fromList . F.toList) <$> Graph.dfs graph keys -- pretty printing @@ -85,9 +106,9 @@ pPrintOutConfig :: SystemInfo -> [Node] -> Doc pPrintOutConfig systemInfo nodes = vcat - [ "{ pkgs }:" + [ "{ pkgs, haskellLib }:" , "" - , "with pkgs.haskell.lib; self: super: {" + , "with haskellLib; self: super: {" , "" , " # core packages" , nest 2 $ vcat $ diff --git a/src/Distribution/Nixpkgs/Haskell/Stack.hs b/src/Distribution/Nixpkgs/Haskell/Stack.hs --- a/src/Distribution/Nixpkgs/Haskell/Stack.hs +++ b/src/Distribution/Nixpkgs/Haskell/Stack.hs @@ -1,11 +1,14 @@ +{-# LANGUAGE CPP #-} module Distribution.Nixpkgs.Haskell.Stack where import Control.Lens +import Data.Maybe (fromMaybe) import Data.Text as T import Distribution.Compiler as Compiler import Distribution.Nixpkgs.Fetch import Distribution.Nixpkgs.Haskell.Derivation import Distribution.Nixpkgs.Haskell.FromCabal as FromCabal +import Distribution.Nixpkgs.Haskell.FromStack as FromStack import Distribution.Nixpkgs.Haskell.PackageSourceSpec as PackageSourceSpec import Distribution.PackageDescription as PackageDescription import Distribution.System as System @@ -40,32 +43,46 @@ -> StackPackage -> IO Package getStackPackageFromDb optHackageDb stackPackage = +#if MIN_VERSION_cabal2nix(2,6,0) PackageSourceSpec.getPackage + False (unHackageDb <$> optHackageDb) - (stackLocationToSource $ stackPackage ^. spLocation) + Nothing + (stackLocationToSource (stackPackage ^. spLocation) (stackPackage ^. spDir)) +#else + PackageSourceSpec.getPackage + (unHackageDb <$> optHackageDb) + (stackLocationToSource (stackPackage ^. spLocation) (stackPackage ^. spDir)) +#endif -stackLocationToSource :: PackageLocation -> Source -stackLocationToSource = \case +stackLocationToSource + :: PackageLocation + -- ^ Subdirectory in the package containing cabal file. + -> Maybe FilePath + -> Source +stackLocationToSource pl mCabalDir = case pl of HackagePackage p -> Source { sourceUrl = "cabal://" ++ T.unpack p , sourceRevision = mempty , sourceHash = UnknownHash - , sourceCabalDir = mempty } + , sourceCabalDir = cabalDir } StackFilePath p -> Source { sourceUrl = p , sourceRevision = mempty , sourceHash = UnknownHash - , sourceCabalDir = mempty } + , sourceCabalDir = cabalDir } StackUri uri -> Source { sourceUrl = URI.uriToString id uri mempty , sourceRevision = mempty , sourceHash = UnknownHash - , sourceCabalDir = mempty } + , sourceCabalDir = cabalDir } StackRepo r -> Source { sourceUrl = T.unpack $ r ^. rUri , sourceRevision = T.unpack $ r ^. rCommit , sourceHash = UnknownHash - , sourceCabalDir = mempty } + , sourceCabalDir = cabalDir } + where + cabalDir = fromMaybe mempty mCabalDir packageDerivation :: StackPackagesConfig @@ -76,16 +93,12 @@ pkg <- getStackPackageFromDb optHackageDb stackPackage let drv = genericPackageDerivation conf pkg - & src .~ pkgSource pkg - -- TODO: remove after Nixos/Nixpkgs #27196 released - & enableSeparateDataOutput .~ False - return $ if stackPackage ^. spExtraDep - then drv - & doCheck &&~ conf ^. spcDoCheckStackage - & runHaddock &&~ conf ^. spcDoHaddockStackage - else drv - & doCheck &&~ conf ^. spcDoCheckPackages - & runHaddock &&~ conf ^. spcDoHaddockPackages + & subpath %~ flip fromMaybe (stackPackage ^. spDir) + isExtraDep = stackPackage ^. spExtraDep + pconf = PackageConfig + { enableCheck = if isExtraDep then conf ^. spcDoCheckStackage else conf ^. spcDoCheckPackages + , enableHaddock = if isExtraDep then conf ^. spcDoHaddockStackage else conf ^. spcDoHaddockPackages } + return $ finalizePackage pkg pconf drv genericPackageDerivation :: StackPackagesConfig diff --git a/src/Distribution/Nixpkgs/Haskell/Stack/PrettyPrinting.hs b/src/Distribution/Nixpkgs/Haskell/Stack/PrettyPrinting.hs --- a/src/Distribution/Nixpkgs/Haskell/Stack/PrettyPrinting.hs +++ b/src/Distribution/Nixpkgs/Haskell/Stack/PrettyPrinting.hs @@ -42,15 +42,11 @@ importStackagePackages :: FilePath -> Doc importStackagePackages path = hsep - [ funarg "self", "import", disp (fromString path :: Nix.FilePath), "{" - , "inherit pkgs stdenv;" - , "inherit (self) callPackage;" - , "}" - ] + ["import", disp (fromString path :: Nix.FilePath)] -callStackageConfig :: FilePath -> Doc -callStackageConfig path = hsep - [ "callPackage", disp (fromString path :: Nix.FilePath), "{}"] +importStackageConfig :: FilePath -> Doc +importStackageConfig path = hsep + ["import ", disp (fromString path :: Nix.FilePath), "{ inherit pkgs haskellLib; }"] overrideHaskellPackages :: OverrideConfig -> NonEmpty Derivation -> Doc overrideHaskellPackages oc packages = @@ -67,8 +63,9 @@ , "let" , nest 2 "inherit (stdenv.lib) extends;" , nest 2 $ vcat - [ attr "stackagePackages" . importStackagePackages $ oc ^. ocStackagePackages - , attr "stackageConfig" . callStackageConfig $ oc ^. ocStackageConfig ] + [ attr "haskellLib" "callPackage (nixpkgs.path + \"/pkgs/development/haskell-modules/lib.nix\") {}" + , attr "stackagePackages" . importStackagePackages $ oc ^. ocStackagePackages + , attr "stackageConfig" . importStackageConfig $ oc ^. ocStackageConfig ] , nest 2 $ vcat [ "stackPackages =" , nest 2 $ overridePackages packages <> semi @@ -82,8 +79,36 @@ , "in callPackage (nixpkgs.path + \"/pkgs/development/haskell-modules\") {" , nest 2 $ vcat [ attr "ghc" ("pkgs.haskell.compiler." <> toNixGhcVersion (oc ^. ocGhc)) - , attr "compilerConfig" "self: extends pkgOverrides (extends stackageConfig (stackagePackages self))" - , attr "haskellLib" "callPackage (nixpkgs.path + \"/pkgs/development/haskell-modules/lib.nix\") {}" + , attr "compilerConfig" "self: extends pkgOverrides (stackageConfig self)" + , attr "initialPackages" "stackagePackages" + , attr "configurationCommon" "args: self: super: {}" + , "inherit haskellLib;" + ] + , "}" + ] + +pPrintHaskellPackages :: OverrideConfig -> Doc +pPrintHaskellPackages oc = + let + nixpkgs = if oc ^. ocNixpkgs . to fromString == systemNixpkgs + then systemNixpkgs + else (disp . (fromString :: FilePath -> Nix.FilePath)) (oc ^. ocNixpkgs) + in vcat + [ funargs + [ "nixpkgs ? import " <> nixpkgs <> " {}" + ] + , "" + , "with nixpkgs; let" + , nest 2 $ vcat + [ attr "haskellLib" "callPackage (nixpkgs.path + /pkgs/development/haskell-modules/lib.nix) {}" + ] + , "in callPackage (nixpkgs.path + /pkgs/development/haskell-modules) {" + , nest 2 $ vcat + [ attr "ghc" ("pkgs.haskell.compiler." <> toNixGhcVersion (oc ^. ocGhc)) + , attr "compilerConfig" . importStackageConfig $ oc ^. ocStackageConfig + , attr "initialPackages" . importStackagePackages $ oc ^. ocStackagePackages + , attr "configurationCommon" "if builtins.pathExists ./configuration-common.nix then import ./configuration-common.nix else args: self: super: {}" + , "inherit haskellLib;" ] , "}" ] diff --git a/src/Language/Nix/FilePath.hs b/src/Language/Nix/FilePath.hs --- a/src/Language/Nix/FilePath.hs +++ b/src/Language/Nix/FilePath.hs @@ -11,7 +11,7 @@ import Distribution.Text import GHC.Generics (Generic) import Prelude hiding (FilePath) -import qualified System.FilePath.Posix as Posix +import qualified System.FilePath as FP import Test.QuickCheck as QC import Text.PrettyPrint as PP @@ -25,7 +25,7 @@ rnf = rnf . unFilePath instance Arbitrary FilePath where - arbitrary = FilePath <$> QC.suchThat arbitrary Posix.isValid + arbitrary = FilePath <$> QC.suchThat arbitrary FP.isValid shrink = fmap FilePath . shrink . unFilePath instance Text FilePath where @@ -38,12 +38,12 @@ parseFilePath :: ReadP r FilePath parseFilePath = do path <- ReadP.munch (const True) - if Posix.isValid path + if FP.isValid path then pure (FilePath path) else pfail renderFilePath :: FilePath -> String -renderFilePath = Posix.combine "." - . Posix.normalise - . Posix.dropTrailingPathSeparator +renderFilePath = FP.combine "." + . FP.normalise + . FP.dropTrailingPathSeparator . unFilePath diff --git a/src/LtsHaskell.hs b/src/LtsHaskell.hs --- a/src/LtsHaskell.hs +++ b/src/LtsHaskell.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE CPP #-} module LtsHaskell where import AllCabalHashes @@ -46,17 +47,21 @@ (error (display pkgId ++ ": meta data has no SHA256 hash for the tarball")) (view (mHashes . at "SHA256") meta) source = DerivationSource "url" ("mirror://hackage/" ++ display pkgId ++ ".tar.gz") "" tarballSHA256 +#if MIN_VERSION_cabal2nix(2,6,0) + return $ Package source False pkgDesc +#else return $ Package source pkgDesc +#endif -getPackageFromDb :: PackageIdentifier -> IO Package -getPackageFromDb pkgId = - getStackPackageFromDb Nothing - $ StackPackage (HackagePackage (T.pack $ Text.display pkgId)) True +getPackageFromDb :: Maybe HackageDb -> PackageIdentifier -> IO Package +getPackageFromDb mHackageDb pkgId = + getStackPackageFromDb mHackageDb + $ StackPackage (HackagePackage (T.pack $ Text.display pkgId)) True Nothing -loadPackage :: FilePath -> Maybe SHA1Hash -> PackageIdentifier -> IO Package -loadPackage allCabalHashesPath mSha1Hash pkgId = +loadPackage :: Maybe HackageDb -> FilePath -> Maybe SHA1Hash -> PackageIdentifier -> IO Package +loadPackage mHackageDb allCabalHashesPath mSha1Hash pkgId = getPackageFromRepo allCabalHashesPath mSha1Hash pkgId - `catchIOError` const (getPackageFromDb pkgId) + `catchIOError` const (getPackageFromDb mHackageDb pkgId) ghcCompilerInfo :: Version -> CompilerInfo ghcCompilerInfo v = CompilerInfo @@ -72,17 +77,18 @@ maybe False (`withinRange` versionRange) $ Map.lookup depName packageVersions buildPackageSetConfig - :: FilePath + :: Maybe HackageDb -> FilePath + -> FilePath -> BuildPlan -> IO PackageSetConfig -buildPackageSetConfig optAllCabalHashes optNixpkgsRepository buildPlan = do +buildPackageSetConfig mHackageDb optAllCabalHashes optNixpkgsRepository buildPlan = do nixpkgs <- readNixpkgPackageMap optNixpkgsRepository Nothing let systemInfo = bpSystemInfo buildPlan packageVersions = fmap ppVersion (bpPackages buildPlan) `Map.union` siCorePackages systemInfo return PackageSetConfig - { packageLoader = loadPackage optAllCabalHashes + { packageLoader = loadPackage mHackageDb optAllCabalHashes , targetPlatform = Platform (siArch systemInfo) (siOS systemInfo) , targetCompiler = ghcCompilerInfo (siGhcVersion systemInfo) , nixpkgsResolver = resolve (Map.map (Set.map (over path ("pkgs":))) nixpkgs) diff --git a/src/Runner.hs b/src/Runner.hs --- a/src/Runner.hs +++ b/src/Runner.hs @@ -2,7 +2,7 @@ import Control.Lens import Data.Foldable as F -import Data.Functor +import Data.List as L import Distribution.Nixpkgs.Haskell.FromStack import Distribution.Nixpkgs.Haskell.FromStack.Package import Distribution.Nixpkgs.Haskell.Stack @@ -18,8 +18,8 @@ import Stack.Config import Stack.Types import Stackage.Types -import System.IO (withFile, IOMode(..), hPutStrLn, hPrint) -import Text.PrettyPrint.HughesPJClass (render) +import System.IO (withFile, IOMode(..), hPutStrLn) +import Text.PrettyPrint.HughesPJClass (Doc, render) import qualified Data.Set as Set import qualified Data.Map as Map @@ -29,60 +29,92 @@ run :: IO () run = do opts <- execParser pinfo - stackYaml <- envStackYaml >>= \case - Just p -> putStrLn "Getting project config file from STACK_YAML environment" $> p - Nothing -> pure $ opts ^. optStackYaml - stackConf <- either fail pure =<< readStackConfig stackYaml - let buildPlanFile = LH.buildPlanFilePath (opts ^. optLtsHaskellRepo) (stackConf ^. scResolver) - buildPlan <- LH.loadBuildPlan buildPlanFile - packageSetConfig <- LH.buildPackageSetConfig - (opts ^. optAllCabalHashesRepo) - (opts ^. optNixpkgsRepository) - buildPlan - -- generate haskell packages override - let - overrideConfig = mkOverrideConfig opts (siGhcVersion $ bpSystemInfo buildPlan) - stackPackagesConfig = mkStackPackagesConfig opts - packages <- traverse (packageDerivation stackPackagesConfig (opts ^. optHackageDb)) - $ stackConf ^. scPackages - let out = PP.overrideHaskellPackages overrideConfig packages - withFile (opts ^. optOutDerivation) WriteMode $ \h -> do - hPutStrLn h ("# Generated by stackage2nix " ++ Text.display version ++ " from " ++ stackYaml ^. syFilePath) - hPrint h out - let - reachable = Set.map mkPackageName - $ F.foldr1 Set.union - $ nodeDepends . mkNode <$> packages - s2nLoader mHash pkgId = - if pkgName pkgId `Set.member` reachable - then packageLoader packageSetConfig Nothing pkgId - else packageLoader packageSetConfig mHash pkgId - s2nPackageSetConfig = packageSetConfig { packageLoader = s2nLoader } - s2nPackageConfig = PackageConfig - { enableCheck = opts ^. optDoCheckStackage - , enableHaddock = opts ^. optDoHaddockStackage } - allNodes <- traverse (uncurry (buildNodeM s2nPackageSetConfig s2nPackageConfig)) - $ Map.toList (bpPackages buildPlan) + case opts ^. optConfigOrigin of + -- Generate build derivation from stack.yaml file + OriginStackYaml stackYaml -> do + stackConf <- either fail pure =<< readStackConfig stackYaml + let buildPlanFile = LH.buildPlanFilePath (opts ^. optLtsHaskellRepo) (stackConf ^. scResolver) + buildPlan <- LH.loadBuildPlan buildPlanFile + packageSetConfig <- LH.buildPackageSetConfig + (opts ^. optHackageDb) + (opts ^. optAllCabalHashesRepo) + (opts ^. optNixpkgsRepository) + buildPlan + let + overrideConfig = mkOverrideConfig opts (siGhcVersion $ bpSystemInfo buildPlan) + stackPackagesConfig = mkStackPackagesConfig opts + stackConfPackages <- traverse (packageDerivation stackPackagesConfig (opts ^. optHackageDb)) + $ stackConf ^. scPackages + let + reachable = Set.map mkPackageName + $ F.foldr1 Set.union + $ nodeDepends . mkNode <$> stackConfPackages + s2nLoader mHash pkgId = + if pkgName pkgId `Set.member` reachable + then packageLoader packageSetConfig Nothing pkgId + else packageLoader packageSetConfig mHash pkgId + s2nPackageSetConfig = packageSetConfig { packageLoader = s2nLoader } + s2nPackageConfig = PackageConfig + { enableCheck = opts ^. optDoCheckStackage + , enableHaddock = opts ^. optDoHaddockStackage } + stackagePackages <- traverse (uncurry (buildNode s2nPackageSetConfig s2nPackageConfig)) + $ Map.toAscList (bpPackages buildPlan) - -- Find all reachable dependencies in stackage set to stick into - -- stackage packages file. This is performed on the full stackage - -- set rather than pruning stackage packages beforehand because - -- stackage does not concern itself with build tools while cabal2nix - -- does: pruning only after generating full set of packages allows - -- us to make sure all those extra dependencies are explicitly - -- listed as well. - let nodes = case opts ^. optOutPackagesClosure of - True -> flip reachableDependencies allNodes - -- Originally reachable nodes are root nodes - $ filter (\n -> mkPackageName (nodeName n) `Set.member` reachable) allNodes - False -> allNodes + let + -- Nixpkgs generic-builder puts hscolour on path for all libraries + withHscolour pkgs = + let hscolour = F.find ((== "hscolour") . nodeName) stackagePackages + in maybe pkgs (`Set.insert` pkgs) hscolour + -- Find all reachable dependencies in stackage set to stick into + -- stackage packages file. This is performed on the full stackage + -- set rather than pruning stackage packages beforehand because + -- stackage does not concern itself with build tools while cabal2nix + -- does: pruning only after generating full set of packages allows + -- us to make sure all those extra dependencies are explicitly + -- listed as well. + nodes = case opts ^. optOutPackagesClosure of + True -> Set.toAscList + $ withHscolour + $ flip reachableDependencies stackagePackages + -- Originally reachable nodes are root nodes + $ L.filter (\n -> mkPackageName (nodeName n) `Set.member` reachable) stackagePackages + False -> stackagePackages + writeOutFile buildPlanFile (opts ^. optOutStackagePackages) + $ pPrintOutPackages (view nodeDerivation <$> nodes) + writeOutFile buildPlanFile (opts ^. optOutStackageConfig) + $ pPrintOutConfig (bpSystemInfo buildPlan) nodes + writeOutFile (stackYaml ^. syFilePath) (opts ^. optOutDerivation) + $ PP.overrideHaskellPackages overrideConfig stackConfPackages - withFile (opts ^. optOutStackagePackages) WriteMode $ \h -> do - hPutStrLn h ("# Generated by stackage2nix " ++ Text.display version ++ " from " ++ buildPlanFile) - hPutStrLn h $ render $ pPrintOutPackages (view nodeDerivation <$> nodes) - withFile (opts ^. optOutStackageConfig) WriteMode $ \h -> do - hPutStrLn h ("# Generated by stackage2nix " ++ Text.display version ++ " from " ++ buildPlanFile) - hPutStrLn h $ render $ pPrintOutConfig (bpSystemInfo buildPlan) nodes + -- Generate Stackage packages from resolver + OriginResolver stackResolver -> do + let + buildPlanFile = LH.buildPlanFilePath (opts ^. optLtsHaskellRepo) stackResolver + packageConfig = PackageConfig + { enableCheck = True + , enableHaddock = True } + buildPlan <- LH.loadBuildPlan buildPlanFile + packageSetConfig <- LH.buildPackageSetConfig + (opts ^. optHackageDb) + (opts ^. optAllCabalHashesRepo) + (opts ^. optNixpkgsRepository) + buildPlan + nodes <- traverse (uncurry (buildNode packageSetConfig packageConfig)) + $ Map.toAscList (bpPackages buildPlan) + let overrideConfig = mkOverrideConfig opts (siGhcVersion $ bpSystemInfo buildPlan) + + writeOutFile buildPlanFile (opts ^. optOutStackagePackages) + $ pPrintOutPackages (view nodeDerivation <$> nodes) + writeOutFile buildPlanFile (opts ^. optOutStackageConfig) + $ pPrintOutConfig (bpSystemInfo buildPlan) nodes + writeOutFile buildPlanFile (opts ^. optOutDerivation) + $ PP.pPrintHaskellPackages overrideConfig + +writeOutFile :: Show source => source -> FilePath -> Doc -> IO () +writeOutFile source filePath contents = + withFile filePath WriteMode $ \h -> do + hPutStrLn h ("# Generated by stackage2nix " ++ Text.display version ++ " from " ++ show source) + hPutStrLn h $ render contents mkOverrideConfig :: Options -> Version -> OverrideConfig mkOverrideConfig opts ghcVersion = OverrideConfig diff --git a/src/Runner/Cli.hs b/src/Runner/Cli.hs --- a/src/Runner/Cli.hs +++ b/src/Runner/Cli.hs @@ -10,27 +10,37 @@ import qualified Distribution.Text as Text import Options.Applicative as Opts import Paths_stackage2nix ( version ) +import Stack.Config import Stack.Types import System.Environment import System.FilePath +-- | Source that would be used to produce target derivation +data ConfigOrigin + = OriginResolver StackResolver + -- ^ Resolver to generate Stackage LTS + | OriginStackYaml StackYaml + -- ^ Stack config to generate build derivation + deriving (Show) +makePrisms ''ConfigOrigin + data Options = Options - { _optAllCabalHashesRepo :: !FilePath - , _optLtsHaskellRepo :: !FilePath - , _optOutStackagePackages :: !FilePath - , _optOutStackageConfig :: !FilePath - , _optOutPackagesClosure :: !Bool - , _optOutDerivation :: !FilePath - , _optDoCheckPackages :: !Bool - , _optDoHaddockPackages :: !Bool - , _optDoCheckStackage :: !Bool - , _optDoHaddockStackage :: !Bool - , _optHackageDb :: !(Maybe HackageDb) - , _optNixpkgsRepository :: !FilePath - , _optCompilerId :: !CompilerId - , _optPlatform :: !Platform - , _optStackYamlArg :: !FilePath + { _optAllCabalHashesRepo :: FilePath + , _optLtsHaskellRepo :: FilePath + , _optOutStackagePackages :: FilePath + , _optOutStackageConfig :: FilePath + , _optOutPackagesClosure :: Bool + , _optOutDerivation :: FilePath + , _optDoCheckPackages :: Bool + , _optDoHaddockPackages :: Bool + , _optDoCheckStackage :: Bool + , _optDoHaddockStackage :: Bool + , _optHackageDb :: (Maybe HackageDb) + , _optNixpkgsRepository :: FilePath + , _optCompilerId :: CompilerId + , _optPlatform :: Platform + , _optConfigOrigin :: ConfigOrigin } deriving (Show) makeLenses ''Options @@ -38,9 +48,6 @@ envStackYaml :: IO (Maybe StackYaml) envStackYaml = fmap mkStackYaml <$> lookupEnv "STACK_YAML" -optStackYaml :: Getter Options StackYaml -optStackYaml = optStackYamlArg . to mkStackYaml - mkStackYaml :: FilePath -> StackYaml mkStackYaml p = case splitFileName p of (dir, "") -> StackYaml dir "stack.yaml" @@ -64,7 +71,7 @@ <*> nixpkgsRepository <*> compilerId <*> platform - <*> stackYamlArg + <*> configOrigin pinfo :: ParserInfo Options pinfo = info @@ -143,10 +150,19 @@ ( long "do-haddock-stackage" <> help "enable haddock for Stackage packages" ) -stackYamlArg :: Parser FilePath -stackYamlArg = Opts.argument str +resolver :: Parser StackResolver +resolver = StackResolver <$> option text + ( long "resolver" + <> help "stackage LTS resolver" ) + +stackYamlArg :: Parser StackYaml +stackYamlArg = mkStackYaml <$> Opts.argument str ( metavar "STACK_YAML" <> help "path to stack.yaml file or directory" ) + +configOrigin :: Parser ConfigOrigin +configOrigin = OriginResolver <$> resolver + <|> OriginStackYaml <$> stackYamlArg -- inherited from cabal2nix diff --git a/src/Stack/Config.hs b/src/Stack/Config.hs --- a/src/Stack/Config.hs +++ b/src/Stack/Config.hs @@ -9,6 +9,7 @@ import Data.Foldable as F import Data.List.NonEmpty as NE import Data.Maybe +import Data.Semigroup (sconcat) import Data.Text as T import Data.Yaml import Network.URI @@ -48,6 +49,8 @@ data StackPackage = StackPackage { _spLocation :: !PackageLocation , _spExtraDep :: !Bool + -- | Subdirectory containing the cabal file if any specified. + , _spDir :: !(Maybe FilePath) } deriving (Eq, Ord, Show) makeLenses ''StackPackage @@ -62,28 +65,39 @@ fromYamlConfig :: Yaml.Config -> StackConfig fromYamlConfig c = StackConfig{..} where - _scResolver = coerce $ c ^. cResolver - _scPackages = F.foldr (NE.<|) neYamlPackages yamlExtraDeps - neYamlPackages = fromMaybe (pure defaultPackage) $ NE.nonEmpty yamlPackages - yamlPackages = fromYamlPackage <$> fromMaybe mempty (c ^. cPackages) + _scResolver = coerce $ c ^. cResolver + _scPackages = F.foldr (NE.<|) neYamlPackages yamlExtraDeps + neYamlPackages = fromMaybe (pure defaultPackage) + . fmap sconcat $ NE.nonEmpty yamlPackages + yamlPackages = fromYamlPackage <$> (fromMaybe mempty (c ^. cPackages)) yamlExtraDeps = fromYamlExtraDep <$> fromMaybe mempty (c ^. cExtraDeps) - defaultPackage = StackPackage (StackFilePath ".") False + defaultPackage = StackPackage (StackFilePath ".") False Nothing -fromYamlPackage :: Yaml.Package -> StackPackage + +fromYamlPackage + :: Yaml.Package + -- ^ A single 'Yaml.Package' can result in multiple actual + -- packages if it has multiple subdirs specified. + -> NonEmpty StackPackage fromYamlPackage = \case Yaml.Simple p -> - StackPackage (parseSimplePath p) False - Yaml.LocationSimple (Yaml.Location p extraDep) -> - StackPackage (parseSimplePath p) (fromMaybe False extraDep) - Yaml.LocationGit (Location git extraDep) -> - StackPackage (StackRepo $ fromYamlGit git) (fromMaybe False extraDep) - Yaml.LocationHg (Location hg extraDep) -> - StackPackage (StackRepo $ fromYamlHg hg) (fromMaybe False extraDep) + unroll Nothing $ StackPackage (parseSimplePath p) False + Yaml.LocationSimple (Location p extraDep ms) -> + unroll ms $ StackPackage (parseSimplePath p) (fromMaybe False extraDep) + Yaml.LocationGit (Location git extraDep ms) -> + unroll ms $ StackPackage (StackRepo $ fromYamlGit git) (fromMaybe False extraDep) + Yaml.LocationHg (Location hg extraDep ms) -> + unroll ms $ StackPackage (StackRepo $ fromYamlHg hg) (fromMaybe False extraDep) where parseSimplePath (T.unpack -> p) = maybe (StackFilePath p) StackUri $ parseURI p + -- Each package gets a single directory with cabal file in it. If + -- it's not specified, path is empty. + unroll subs p = case subs of + Just (x : xs) -> NE.map (p . Just) (x :| xs) + _ -> p Nothing :| [] fromYamlExtraDep :: Text -> StackPackage -fromYamlExtraDep = flip StackPackage True . HackagePackage +fromYamlExtraDep t = StackPackage (HackagePackage t) True Nothing fromYamlGit :: Yaml.Git -> Repo fromYamlGit yg = Repo{..} diff --git a/src/Stack/Config/Yaml.hs b/src/Stack/Config/Yaml.hs --- a/src/Stack/Config/Yaml.hs +++ b/src/Stack/Config/Yaml.hs @@ -10,6 +10,7 @@ data Location a = Location { _lLocation :: !a , _lExtraDep :: !(Maybe Bool) + , _lSubdirs :: !(Maybe [FilePath]) } deriving (Eq, Show) makeLenses ''Location diff --git a/src/Stack/Types.hs b/src/Stack/Types.hs --- a/src/Stack/Types.hs +++ b/src/Stack/Types.hs @@ -10,6 +10,7 @@ data StackYaml = StackYaml { _syDirName :: !FilePath , _syFileName :: !FilePath } + deriving (Ord, Eq, Show) makeLenses ''StackYaml diff --git a/stackage2nix.cabal b/stackage2nix.cabal --- a/stackage2nix.cabal +++ b/stackage2nix.cabal @@ -1,7 +1,7 @@ name: stackage2nix -version: 0.3.0 +version: 0.4.0 synopsis: Convert Stack files into Nix build instructions. -homepage: https://github.com/4e6/stackage2nix#readme +homepage: https://github.com/typeable/stackage2nix#readme license: BSD3 license-file: LICENSE author: Dmitry Bushev @@ -44,7 +44,7 @@ , gitlib > 3 , gitlib-libgit2 > 3 , hopenssl > 2.2 - , inflections + , inflections >= 0.3 , language-nix , lens , network-uri @@ -91,4 +91,4 @@ source-repository head type: git - location: https://github.com/4e6/stackage2nix + location: https://github.com/typeable/stackage2nix diff --git a/test/Stack/Config/YamlSpec.hs b/test/Stack/Config/YamlSpec.hs --- a/test/Stack/Config/YamlSpec.hs +++ b/test/Stack/Config/YamlSpec.hs @@ -28,6 +28,12 @@ - location: git: git@github.com:commercialhaskell/stack.git commit: 6a86ee32e5b869a877151f74064572225e1a0398 + - location: + git: git@github.com:example/mega-repo + commit: 6a86ee32e5b869a877151f74064572225e1a0000 + subdirs: + - subdir1 + - subdir2 # Comment extra-deps: - acme-missiles-0.3 @@ -45,13 +51,19 @@ { _cResolver = "lts-3.7" , _cPackages = Just [ Simple "." - , LocationSimple (Location "dir1/dir2" Nothing) - , LocationSimple (Location "https://example.com/foo/bar/baz-0.0.2.tar.gz" (Just True)) + , LocationSimple (Location "dir1/dir2" Nothing Nothing) + , LocationSimple (Location "https://example.com/foo/bar/baz-0.0.2.tar.gz" (Just True) Nothing) , LocationGit $ Location (Git "git@github.com:commercialhaskell/stack.git" "6a86ee32e5b869a877151f74064572225e1a0398") - Nothing] + Nothing Nothing + , LocationGit (Location + (Git + "git@github.com:example/mega-repo" + "6a86ee32e5b869a877151f74064572225e1a0000") + Nothing (Just ["subdir1", "subdir2"])) + ] , _cExtraDeps = Just ["acme-missiles-0.3"] }