diff --git a/Main.hs b/Main.hs --- a/Main.hs +++ b/Main.hs @@ -7,6 +7,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-} +{-# LANGUAGE ScopedTypeVariables #-} {-| @dhall-to-nixpkgs@ is essentially the Dhall analog of @cabal2nix@. @@ -76,8 +77,10 @@ import Control.Monad.Trans.State.Strict (StateT) import Data.Aeson (FromJSON) import Data.List.NonEmpty (NonEmpty (..)) +import Data.Maybe (mapMaybe) import Data.Text (Text) import Data.Void (Void) +import Dhall.Crypto (SHA256Digest (..)) import Dhall.Import (Status (..), stack) import Dhall.Parser (Src) import GHC.Generics (Generic) @@ -101,30 +104,33 @@ , URL (..) ) -import qualified Control.Foldl as Foldl -import qualified Control.Monad.Trans.State.Strict as State -import qualified Data.Aeson as Aeson -import qualified Data.Foldable as Foldable -import qualified Data.List.NonEmpty as NonEmpty -import qualified Data.Text as Text -import qualified Data.Text.Encoding as Text.Encoding -import qualified Data.Text.IO as Text.IO -import qualified Data.Text.Prettyprint.Doc.Render.Text as Prettyprint.Text +import qualified Control.Foldl as Foldl +import qualified Control.Monad.Trans.State.Strict as State +import qualified Data.Aeson as Aeson +import qualified Data.ByteString.Base16 as Base16 +import qualified Data.ByteString.Base64 as Base64 +import qualified Data.ByteString.Char8 as ByteString.Char8 +import qualified Data.Foldable as Foldable +import qualified Data.List.NonEmpty as NonEmpty +import qualified Data.Text as Text +import qualified Data.Text.Encoding as Text.Encoding +import qualified Data.Text.IO as Text.IO import qualified Dhall.Core import qualified Dhall.Import import qualified Dhall.Optics import qualified Dhall.Parser import qualified GHC.IO.Encoding import qualified NeatInterpolation -import qualified Network.URI as URI -import qualified Nix.Expr.Shorthands as Nix +import qualified Network.URI as URI +import qualified Nix.Expr.Shorthands as Nix import qualified Nix.Pretty -import qualified Options.Applicative as Options +import qualified Options.Applicative as Options +import qualified Prettyprinter.Render.Text as Prettyprint.Text import qualified System.Exit import qualified System.IO -import qualified Text.Megaparsec as Megaparsec -import qualified Text.Megaparsec.Char as Megaparsec.Char -import qualified Text.Megaparsec.Char.Lexer as Megaparsec.Char.Lexer +import qualified Text.Megaparsec as Megaparsec +import qualified Text.Megaparsec.Char as Megaparsec.Char +import qualified Text.Megaparsec.Char.Lexer as Megaparsec.Char.Lexer import qualified Turtle data Options @@ -149,6 +155,7 @@ , file :: FilePath , source :: Bool , document :: Bool + , fixedOutputDerivations :: Bool } data NixPrefetchGit = NixPrefetchGit @@ -206,6 +213,13 @@ <> Options.help "Generate documentation for the Nix package" ) +parseFixedOutputDerivations :: Parser Bool +parseFixedOutputDerivations = + Options.switch + ( Options.long "fixed-output-derivations" + <> Options.help "Translate Dhall remote imports to Nix fixed-output derivations" + ) + parseName :: Parser (Maybe Text) parseName = optional @@ -271,6 +285,8 @@ document <- parseDocument + fixedOutputDerivations <- parseFixedOutputDerivations + return Directory{..} parserInfoOptions :: ParserInfo Options @@ -306,11 +322,12 @@ This function finds all remote imports that are transitive dependencies of the given expression, failing if any of them are missing integrity checks. -} -findExternalDependencies :: Expr Src Import -> StateT Status Shell URL +findExternalDependencies :: Expr Src Import -> StateT Status Shell (URL, SHA256Digest) findExternalDependencies expression = do -- This is a best-effort attempt to pick an import alternative if there is -- more than one - let pickAlt (ImportAlt e0 e1) + let pickAlt :: Expr Src Import -> Maybe (Expr Src Import) + pickAlt (ImportAlt e0 e1) -- If only the latter import has an integrity check, then select -- that | Embed Import{ importHashed = ImportHashed{ hash = Nothing } } <- Dhall.Core.shallowDenote e0 @@ -322,7 +339,8 @@ pickAlt _ = Nothing - let rewrittenExpression = + let rewrittenExpression :: Expr Src Import + rewrittenExpression = Dhall.Optics.rewriteOf Dhall.Core.subExpressions pickAlt expression import_ <- lift (Turtle.select (Foldable.toList rewrittenExpression)) @@ -349,8 +367,8 @@ Remote url -> case hash of - Just _ -> - return url + Just sha256 -> + return (url, sha256) Nothing -> die (MissingSemanticIntegrityCheck url) @@ -366,17 +384,70 @@ findExternalDependencies parsedExpression data Dependency = Dependency - { functionParameter :: (Text, Maybe NExpr) + { functionParameter :: Maybe (Text, Maybe NExpr) -- ^ Function parameter used to bring the dependency into scope for the - -- Nix package. The @`Maybe` `NExpr`@ is always `Nothing`, but we + -- Nix package. + -- + -- This is 'Nothing' when 'fixedOutputDerivations' is enabled, since these + -- dependencies don't need to passed in as arguments. This is 'Just' + -- when 'fixedOutputDerivations' is not enabled. + -- + -- The @'Maybe' 'NExpr'@ is always 'Nothing', but we -- include it here for convenience , dependencyExpression :: NExpr - -- ^ The dependency expression to include in the dependency list. This - -- will be an expression of the form: + -- ^ The dependency expression to include in the dependency list. -- + -- 'dependencyToNix' will create an expression of the following form. + -- This is called when 'fixedOutputDerivations' is 'False': + -- -- > someDependency.override { file = "./someFile.dhall" } + -- + -- 'dependencyToNixAsFOD' will create an expression of the following form. + -- This is called when 'fixedOutputDerivations' is 'True': + -- + -- > buildDhallUrl { + -- > url = "https://some.url.to/a/dhall/file.dhall"; + -- > hash = "sha256-ZTSiQUXpPbPfPvS8OeK6dDQE6j6NbP27ho1cg9YfENI="; + -- > dhallHash = + -- > "sha256:6534a24145e93db3df3ef4bc39e2ba743404ea3e8d6cfdbb868d5c83d61f10d2"; + -- > } } + deriving stock Show +-- | Convert a 'URL' and integrity check to a Nix 'Dependency' that uses the +-- Nix function @buildDhallUrl@ to build. +-- +-- This function will create a Nix dependency of the form: +-- +-- > buildDhallUrl { +-- > url = "https://some.url.to/a/dhall/file.dhall"; +-- > hash = "sha256-ZTSiQUXpPbPfPvS8OeK6dDQE6j6NbP27ho1cg9YfENI="; +-- > dhallHash = +-- > "sha256:6534a24145e93db3df3ef4bc39e2ba743404ea3e8d6cfdbb868d5c83d61f10d2"; +-- > } +-- +-- The @hash@ argument is an SRI hash that Nix understands. The @dhallHash@ +-- argument is a base-16-encoded hash that Dhall understands. +dependencyToNixAsFOD :: URL -> SHA256Digest -> IO Dependency +dependencyToNixAsFOD url (SHA256Digest shaBytes) = do + let functionParameter = Nothing + + let dhallHash = + "sha256:" <> ByteString.Char8.unpack (Base16.encode shaBytes) + + let nixSRIHash = + "sha256-" <> ByteString.Char8.unpack (Base64.encode shaBytes) + + let dependencyExpression = + "buildDhallUrl" + @@ Nix.attrsE + [ ("url", Nix.mkStr $ Dhall.Core.pretty url) + , ("hash", Nix.mkStr $ Text.pack nixSRIHash) + , ("dhallHash", Nix.mkStr $ Text.pack dhallHash) + ] + + return Dependency{..} + {-| The Nixpkgs support for Dhall implements two conventions that @dhall-to-nixpkgs@ depends on: @@ -409,7 +480,7 @@ "dhall-lang" : "dhall-lang" : _rev : "Prelude" : rest -> do let fileArgument = Text.intercalate "/" rest - let functionParameter = (prelude, Nothing) + let functionParameter = Just (prelude, Nothing) let dependencyExpression = (Nix.mkSym prelude @. "overridePackage") @@ -421,7 +492,7 @@ _owner : repo : _rev : rest -> do let fileArgument = Text.intercalate "/" rest - let functionParameter = (repo, Nothing) + let functionParameter = Just (repo, Nothing) let dependencyExpression = (Nix.mkSym repo @. "overridePackage") @@ -431,6 +502,29 @@ return Dependency{..} _ -> do + die (NotAValidGistRepositoryURL url) + + "gist.githubusercontent.com" -> do + let File{ directory, file } = path + + let Dhall.Core.Directory{ components } = directory + + case reverse (file : components) of + owner : hash : "raw" : _rev : rest -> do + let fileArgument = Text.intercalate "/" rest + + let package = owner <> "_" <> hash + + let functionParameter = Just (package, Nothing) + + let dependencyExpression = + (Nix.mkSym package @. "overridePackage") + @@ Nix.attrsE + [ ("file", Nix.mkStr fileArgument ) ] + + return Dependency{..} + + _ -> do die (NotAValidGitHubRepositoryURL url) "prelude.dhall-lang.org" -> do @@ -460,10 +554,10 @@ rest rest -> rest - + let fileArgument = Text.intercalate "/" pathComponents - let functionParameter = (prelude, Nothing) + let functionParameter = Just (prelude, Nothing) let dependencyExpression = (Nix.mkSym prelude @. "overridePackage") @@ -475,6 +569,45 @@ _ -> do die (UnsupportedDomainDependency url authority) +-- | Turn a list of 'Dependency's into an argument list for the generated Nix +-- function. +-- +-- The following 'makeNixFunctionParams' call: +-- +-- @@ +-- 'makeNixFunctionParams' +-- \"buildDhallDirectoryPackage\" +-- [ 'Dependency' ('Just' (\"Prelude\", 'Nothing')) ... +-- , 'Dependency' ('Just' (\"Prelude\", 'Nothing')) ... +-- , 'Dependency' 'Nothing' ... +-- , 'Dependency' ('Just' (\"example-repo\", 'Nothing')) ... +-- ] +-- @@ +-- +-- will generate an argument list like the following: +-- +-- > { buildDhallDirectoryPackage, buildDhallUrl, Prelude, example-repo }: +-- +-- Note that identical 'functionParameter's will be collapsed into a single +-- parameter (like @Prelude@ above). +-- +-- @buildDhallUrl@ will be added as an argument only if there is a 'Dependency' +-- with a 'Nothing' value for 'functionalParameter'. +makeNixFunctionParams :: Text -> [Dependency] -> [(Text, Maybe NExpr)] +makeNixFunctionParams buildDhallFuncName nixDependencies = + let containsBuildDhallUrlDependency = + any (\dep -> functionParameter dep == Nothing) nixDependencies + + buildDhallUrlParam = + if containsBuildDhallUrlDependency + then [ ("buildDhallUrl", Nothing) ] + else [ ] + + in ( [ (buildDhallFuncName, Nothing) ] + <> buildDhallUrlParam + <> nub (mapMaybe functionParameter nixDependencies) + ) + githubToNixpkgs :: GitHub -> IO () githubToNixpkgs GitHub{ name, uri, rev = maybeRev, hash, fetchSubmodules, directory, file, source, document } = do URI{ uriScheme, uriAuthority = Just URIAuth{ uriUserInfo, uriRegName, uriPort }, uriPath, uriQuery, uriFragment } <- do @@ -487,8 +620,9 @@ _ -> die (UnsupportedURIScheme uri uriScheme) case uriRegName of - "github.com" -> return () - _ -> die (UnsupportedDomain uri uriRegName) + "github.com" -> return () + "gist.github.com" -> return () + _ -> die (UnsupportedDomain uri uriRegName) case uriPort of "" -> return () @@ -600,18 +734,16 @@ dependencies <- Turtle.reduce Foldl.nub (State.evalStateT (findExternalDependencies expression) status) - nixDependencies <- traverse dependencyToNix dependencies + nixDependencies <- traverse (\(url, _sha256) -> dependencyToNix url) dependencies let buildDhallGitHubPackage = "buildDhallGitHubPackage" + let functionParams = + makeNixFunctionParams buildDhallGitHubPackage nixDependencies + let nixExpression = Nix.mkFunction - (Nix.mkParamset - ( [ (buildDhallGitHubPackage, Nothing) ] - <> nub (fmap functionParameter nixDependencies) - ) - False - ) + (Nix.mkParamset functionParams False) ( Nix.mkSym buildDhallGitHubPackage @@ Nix.attrsE [ ("name", Nix.mkStr finalName) @@ -633,7 +765,7 @@ Prettyprint.Text.putDoc ((Nix.Pretty.prettyNix nixExpression) <> "\n") directoryToNixpkgs :: Directory -> IO () -directoryToNixpkgs Directory{ name, directory, file, source, document } = do +directoryToNixpkgs Directory{ name, directory, file, source, document, fixedOutputDerivations } = do let finalName = case name of Nothing -> Turtle.format fp (Turtle.dirname directory) @@ -661,8 +793,14 @@ dependencies <- Turtle.reduce Foldl.nub (State.evalStateT (findExternalDependencies expression) status) - nixDependencies <- traverse dependencyToNix dependencies + let depToNix :: (URL, SHA256Digest) -> IO Dependency + depToNix (url, sha256) = + if fixedOutputDerivations + then dependencyToNixAsFOD url sha256 + else dependencyToNix url + nixDependencies <- traverse depToNix dependencies + let buildDhallDirectoryPackage = "buildDhallDirectoryPackage" let src | null directoryString = directoryString @@ -671,14 +809,12 @@ where directoryString = Turtle.encodeString directory + let functionParams = + makeNixFunctionParams buildDhallDirectoryPackage nixDependencies + let nixExpression = Nix.mkFunction - (Nix.mkParamset - ( [ (buildDhallDirectoryPackage, Nothing) ] - <> nub (fmap functionParameter nixDependencies) - ) - False - ) + (Nix.mkParamset functionParams False) ( Nix.mkSym buildDhallDirectoryPackage @@ Nix.attrsE [ ("name", Nix.mkStr finalName) @@ -701,6 +837,7 @@ data Error = MissingSemanticIntegrityCheck URL | NotAValidGitHubRepositoryURL URL + | NotAValidGistRepositoryURL URL | UnsupportedDomainDependency URL Text | RepositoryIsNotAValidURI Text | UnsupportedURIScheme Text String @@ -749,6 +886,22 @@ ↳ https://raw.githubusercontent.com/$${owner}/$${repository}/$${revision}/… |] + NotAValidGistRepositoryURL url -> + let dependency = Dhall.Core.pretty url + + in [NeatInterpolation.text| +Error: Not a valid gist repository URL + +Your Dhall package appears to depend on the following import: + +↳ $dependency + +... which is missing one or more path components that a raw GitHub import would +normally have. The URL should minimally have the following path components: + +↳ https://gist.githubusercontent.com/$${owner}/$${id}/raw/$${revision}/… +|] + UnsupportedDomainDependency url authority -> let dependency = Dhall.Core.pretty url @@ -758,6 +911,7 @@ This tool currently only translates the following domains into Nix dependencies: * raw.githubusercontent.com +* gist.githubusercontent.com * prelude.dhall-lang.org One of the Dhall project's dependencies: @@ -942,7 +1096,7 @@ , rev : Text , path : Text , sha256 : Text - , fetchSubmodules : Bool + , fetchSubmodules : Bool } ... but JSON decoding failed with the following error: diff --git a/dhall-nixpkgs.cabal b/dhall-nixpkgs.cabal --- a/dhall-nixpkgs.cabal +++ b/dhall-nixpkgs.cabal @@ -1,4 +1,4 @@ -Version: 1.0.6 +Version: 1.0.7 Cabal-Version: >=1.10 Name: dhall-nixpkgs Synopsis: Convert Dhall projects to Nix packages @@ -17,17 +17,21 @@ Executable dhall-to-nixpkgs Main-Is: Main.hs Build-Depends: base >= 4.11 && < 5 - , aeson >= 1.0.0.0 && < 1.6 + , aeson >= 1.0.0.0 && < 2.1 + , base16-bytestring >= 1.0.0.0 + , base64-bytestring >= 1.2.1.0 + , bytestring < 0.12 , data-fix , dhall >= 1.32.0 && < 1.41 , foldl < 1.5 , hnix >= 0.10.1 && < 0.15 , lens-family-core >= 1.0.0 && < 2.2 - , megaparsec >= 7.0.0 && < 9.2 + -- megaparsec follows SemVer: https://github.com/mrkkrp/megaparsec/issues/469#issuecomment-927918469 + , megaparsec >= 7.0.0 && < 10 , mmorph < 1.3 , neat-interpolation < 0.6 , optparse-applicative >= 0.14.0.0 && < 0.17 - , prettyprinter >= 1.5.1 && < 1.8 + , prettyprinter >= 1.7.0 && < 1.8 , text >= 0.11.1.0 && < 1.3 , transformers >= 0.2.0.0 && < 0.6 , turtle < 1.6