diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,22 @@ +# Revision History for language-nix + +## 2.3.0 + +* `Language.Nix.Identifier` now exports `nixKeywords` which lists + keywords in Nix that are parseable as simple identifiers, but have + special meaning, like `assert` and `with`. Consequently, they can't + be used as (simple) identifiers in Nix code. + + `quote`, `needsQuoting` and `Pretty` will take this list into account + and quote such identifiers. However, `HasParser` will _not_ reject them + even if they are unquoted. +* Resolved discrepancies between `Language.Nix.Identifier` and Nix w.r.t. + quoting and escaping: + + - Fixed missing escaping of some Nix syntax elements, e.g. in the case of + `ident # "${foo}"`. + - Pretty printing `Identifier`s will no longer produce escape sequences + Haskell supports, but Nix doesn't. + - Parsing `Identifier`s won't interpret escape sequences that Nix wouldn't + understand. +* Added an hspec/QuickCheck based test suite. diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,9 @@ +language-nix +============ + +[![hackage release](https://img.shields.io/hackage/v/language-nix.svg?label=hackage)](http://hackage.haskell.org/package/language-nix) +[![stackage LTS package](http://stackage.org/package/language-nix/badge/lts)](http://stackage.org/lts/package/language-nix) +[![stackage Nightly package](http://stackage.org/package/language-nix/badge/nightly)](http://stackage.org/nightly/package/language-nix) +[![travis build status](https://img.shields.io/travis/peti/language-nix/master.svg?label=travis+build)](https://travis-ci.org/peti/language-nix) + +This Haskell library contains data types and useful functions to represent and manipulate the Nix language. It is used by tooling like cabal2nix and is not meant to be a general purpose nix parser or even interpreter. Have a look at [hnix (in Haskell)](https://github.com/haskell-nix/hnix) or [nix (the original in C++)](https://github.com/nixos/nix) instead. diff --git a/language-nix.cabal b/language-nix.cabal --- a/language-nix.cabal +++ b/language-nix.cabal @@ -1,5 +1,5 @@ name: language-nix -version: 2.2.0 +version: 2.3.0 synopsis: Data types and functions to represent the Nix language description: Data types and useful functions to represent and manipulate the Nix language. @@ -7,23 +7,25 @@ license-file: LICENSE author: Peter Simons maintainer: simons@cryp.to -tested-with: GHC == 8.2.2, GHC == 8.4.4, GHC == 8.6.5, GHC == 8.8.1 +tested-with: GHC == 8.10.7, GHC == 9.0.2, GHC == 9.2.8, GHC == 9.4.8, GHC == 9.6.7 || == 9.8.4 || == 9.10.2 || == 9.12.2 category: Distribution, Language, Nix -homepage: https://github.com/peti/language-nix#readme -bug-reports: https://github.com/peti/language-nix/issues +homepage: https://github.com/NixOS/cabal2nix/tree/master/language-nix#readme +bug-reports: https://github.com/NixOS/cabal2nix/issues build-type: Simple -cabal-version: >= 1.10 +cabal-version: 1.18 +extra-doc-files: README.md + CHANGELOG.md source-repository head type: git - location: https://github.com/peti/language-nix + location: https://github.com/NixOS/cabal2nix + subdir: language-nix library exposed-modules: Language.Nix Language.Nix.Binding Language.Nix.Identifier Language.Nix.Path - other-modules: Paths_language_nix hs-source-dirs: src build-depends: base < 5, QuickCheck, deepseq, lens, parsec-class, pretty default-language: Haskell2010 @@ -31,3 +33,17 @@ TemplateHaskell GeneralizedNewtypeDeriving DeriveGeneric + +test-suite hspec + type: exitcode-stdio-1.0 + hs-source-dirs: test + main-is: hspec.hs + build-depends: base + , language-nix + , hspec + , QuickCheck + , lens + , parsec-class + , pretty + , process + default-language: Haskell2010 diff --git a/src/Language/Nix/Binding.hs b/src/Language/Nix/Binding.hs --- a/src/Language/Nix/Binding.hs +++ b/src/Language/Nix/Binding.hs @@ -6,7 +6,6 @@ import Control.DeepSeq import Control.Lens -import Data.Maybe import Data.String import GHC.Generics ( Generic ) import Language.Nix.Identifier diff --git a/src/Language/Nix/Identifier.hs b/src/Language/Nix/Identifier.hs --- a/src/Language/Nix/Identifier.hs +++ b/src/Language/Nix/Identifier.hs @@ -4,8 +4,15 @@ {-# LANGUAGE TemplateHaskell #-} module Language.Nix.Identifier - ( Identifier, ident, quote, needsQuoting + ( -- * Type-safe Identifiers + Identifier, ident , parseSimpleIdentifier, parseQuotedIdentifier + -- * String Predicates + , needsQuoting + -- * Internals + -- TODO: only required by the language-nix test suite, unexport? + , nixKeywords + , quote ) where @@ -23,33 +30,54 @@ -- (and viewed) with the 'ident' isomorphism. For the sake of convenience, -- @Identifier@s are an instance of the 'IsString' class. -- --- Reasonable people restrict themselves to identifiers of the form --- @[a-zA-Z_][a-zA-Z0-9_'-]*@, because these don't need quoting. The --- methods of the 'Text' class can be used to parse and pretty-print an --- identifier with proper quoting: +-- It is usually wise to only use identifiers of the form +-- @[a-zA-Z_][a-zA-Z0-9_'-]*@, because these don't need quoting. +-- Consequently, they can appear almost anywhere in a Nix expression +-- (whereas quoted identifiers e.g. can't be used in function patterns). +-- The methods of the 'Pretty' class can be used to print an identifier +-- with proper quoting: -- -- >>> pPrint (ident # "test") -- test -- >>> pPrint (ident # "foo.bar") -- "foo.bar" -- --- prop> \str -> Just (ident # str) == parseM "Ident" (quote str) --- prop> \i -> Just (i :: Identifier) == parseM "Ident" (prettyShow i) +-- The 'HasParser' class allows parsing rendered identifiers even if they are +-- quoted: +-- +-- >>> parseM "Identifier" "hello" :: Maybe Identifier +-- Just (Identifier "hello") +-- >>> parseM "Identifier" "\"3rd party\"" :: Maybe Identifier +-- Just (Identifier "3rd party") +-- +-- __Warning__: Identifiers /may not/ contain @\'\\0\'@, but this is not +-- checked during construction! +-- +-- See also . declareLenses [d| newtype Identifier = Identifier { ident :: String } deriving (Show, Eq, Ord, IsString, Generic) |] -- ^ An isomorphism that allows conversion of 'Identifier' from/to the -- standard 'String' type via 'review'. -- --- prop> \str -> fromString str == ident # str --- prop> \str -> set ident str undefined == ident # str --- prop> \str -> view ident (review ident str) == str +-- >>> ident # "hello" +-- Identifier "hello" +-- >>> from ident # fromString "hello" +-- "hello" instance NFData Identifier where rnf (Identifier str) = rnf str instance Arbitrary Identifier where - arbitrary = Identifier <$> listOf1 arbitraryUnicodeChar + arbitrary = Identifier <$> oneof + [ -- almost always needs quoting, unreasonable + listOf1 (nonNul arbitraryUnicodeChar) + -- almost always needs quoting, reasonable-ish + , listOf1 (nonNul arbitraryPrintableChar) + -- rarely needs quoting + , listOf1 (arbitraryASCIIChar `suchThat` isSimpleChar) ] + where nonNul g = g `suchThat` (/= '\0') + isSimpleChar c = isAlphaNum c || c `elem` "_-'" shrink (Identifier i) = map Identifier (shrink i) instance CoArbitrary Identifier @@ -57,14 +85,26 @@ instance Pretty Identifier where pPrint = view (ident . to quote . to text) +-- | Note that this parser is more lenient than Nix w.r.t. simple identifiers, +-- since it will accept 'nixKeywords'. +-- +-- Naturally, it does not support string interpolation, but does not reject +-- strings that contain them. E.g. the string literal @"hello ${world}"@ +-- will contain @${world}@ verbatim after parsing. Do not rely on this +-- behavior, as it may be changed in the future. instance HasParser Identifier where parser = parseQuotedIdentifier <|> parseSimpleIdentifier -- | Parsec parser for simple identifiers, i.e. those that don't need quoting. +-- The parser is equivalent to the regular expression @^[a-zA-Z_][a-zA-Z0-9_'-]*$@ +-- which the Nix parser uses. +-- +-- Note that this parser will accept keywords which would not be parsed as +-- identifiers by Nix, see 'nixKeywords'. parseSimpleIdentifier :: CharParser st tok m Identifier parseSimpleIdentifier = do - c <- satisfy (\x -> x == '_' || isAlpha x) - cs <- many (satisfy (\x -> x `elem` "_'-" || isAlphaNum x)) + c <- satisfy (\x -> x == '_' || (isAscii x && isAlpha x)) + cs <- many (satisfy (\x -> x `elem` "_'-" || (isAscii x && isAlphaNum x))) return (Identifier (c:cs)) -- | 'ReadP' parser for quoted identifiers, i.e. those that /do/ need @@ -73,29 +113,69 @@ parseQuotedIdentifier = Identifier <$> qstring where qstring :: CharParser st tok m String - qstring = do txt <- between (P.char '"') (P.char '"') (many qtext) - return (read ('"' : concat txt ++ ['"'])) + qstring = between (P.char '"') (P.char '"') (many qtext) - qtext :: CharParser st tok m String - qtext = quotedPair <|> many1 (P.noneOf "\\\"") + qtext :: CharParser st tok m Char + qtext = quotedPair <|> P.noneOf "\\\"" - quotedPair :: CharParser st tok m String + quotedPair :: CharParser st tok m Char quotedPair = do - c1 <- P.char '\\' - c2 <- anyChar - return [c1,c2] + _ <- P.char '\\' + c <- anyChar + -- See https://github.com/NixOS/nix/blob/2d83bc6b83763290e9bbf556209927ba469956aa/src/libexpr/lexer.l#L54-L60 + return $ case c of + 'n' -> '\n' + 't' -> '\t' + 'r' -> '\r' + -- Note that this handles actual escapes like \" and \\ and + -- bogus cases like \f which Nix doesn't fail on (despite not + -- supporting it), but simply maps to plain f + _ -> c -- | Checks whether a given string needs quoting when interpreted as an --- 'Identifier'. Simple identifiers that don't need quoting match the --- regular expression @^[a-zA-Z_][a-zA-Z0-9_'-]*$@. +-- 'Identifier'. needsQuoting :: String -> Bool -needsQuoting = isLeft . runParser (parseSimpleIdentifier >> eof) () "" +needsQuoting s = + s `elem` nixKeywords + || isLeft (runParser (parseSimpleIdentifier >> eof) () "" s) +-- | List of strings that are parseable as simple identifiers (see +-- 'parseSimpleIdentifier') in isolation, but won't be accepted by Nix because +-- [keywords](https://nix.dev/manual/nix/2.30/language/identifiers.html#keywords) +-- take precedence. +nixKeywords :: [String] +nixKeywords = + [ "assert", "with", "if", "then", "else", "let", "in", "rec", "inherit", "or" ] + -- | Helper function to quote a given identifier string if necessary. +-- Usually, one should use the 'Pretty' instance of 'Identifier' instead. -- -- >>> putStrLn (quote "abc") -- abc -- >>> putStrLn (quote "abc.def") -- "abc.def" +-- >>> putStrLn (quote "$foo") +-- "$foo" +-- >>> putStrLn (quote "${foo}") +-- "\${foo}" quote :: String -> String -quote s = if needsQuoting s then show s else s +quote s = if needsQuoting s then '"' : quote' s else s + where + quote' (c1:c2:cs) = escapeChar c1 (Just c2) ++ quote' (c2:cs) + quote' (c:cs) = escapeChar c Nothing ++ quote' cs + quote' "" = "\"" + +escapeChar :: Char -> Maybe Char -> String +escapeChar c1 c2 = + case c1 of + -- supported escape sequences, see quotedPair above + -- N.B. technically, we only need to escape \r (since Nix converts raw \r to \n), + -- but it's nicer to escape what we can. + '\n' -> "\\n" + '\t' -> "\\t" + '\r' -> "\\r" + -- syntactically significant in doubly quoted strings + '\\' -> "\\\\" + '"' -> "\\\"" + '$' | c2 == Just '{' -> "\\$" + _ -> [c1] diff --git a/src/Language/Nix/Path.hs b/src/Language/Nix/Path.hs --- a/src/Language/Nix/Path.hs +++ b/src/Language/Nix/Path.hs @@ -5,7 +5,6 @@ import Control.DeepSeq import Control.Lens -import Data.Maybe import Data.String import GHC.Generics ( Generic ) import Language.Nix.Identifier diff --git a/test/hspec.hs b/test/hspec.hs new file mode 100644 --- /dev/null +++ b/test/hspec.hs @@ -0,0 +1,100 @@ +{-# LANGUAGE ScopedTypeVariables #-} +module Main (main) where + +import Control.Exception +import Control.Lens +import Control.Monad (forM_) +import Data.Char (isAscii, isSpace) +import Data.List (dropWhileEnd) +import Data.String (fromString) +import Language.Nix.Identifier +import System.Process (callProcess, readCreateProcess, proc) +import Test.Hspec +import Test.QuickCheck +import Text.Parsec.Class (parseM) +import Text.PrettyPrint.HughesPJClass (prettyShow) + +main :: IO () +main = hspec $ do + describe "Language.Nix.Identifier" $ do + describe "ident" $ do + it "is equivalent to fromString" $ + stringIdentProperty $ \str -> fromString str == ident # str + it "allows recovering the original string after conversion to Identifier" $ + stringIdentProperty $ \str -> view ident (review ident str) == str + it "can be used as a setter" $ + stringIdentProperty $ \str -> set ident str undefined == ident # str + + describe "HasParser Identifier" $ do + it "can parse the result of prettyShow" $ + identProperty $ \i -> parseM "Identifier" (prettyShow i) == Just (i :: Identifier) + it "can parse the result of quote" $ + stringIdentProperty $ \str -> parseM "Identifier" (quote str) == Just (ident # str) + it "parses redundant escape sequences" $ + forM_ + [ ("\"\\f\"", "f") + , ("\"echo \\$var\"", "echo $var") + , ("\"\\h\\e\\l\\l\\o\\ \\w\\or\\l\\d\"", "hello world") + -- \t and \n don't need to be escaped, though it's advisable + , ("\"only\\ttechnically\nredundant\"", "only\ttechnically\nredundant") + ] + $ \(i, e) -> do + let e' = Just (ident # e) + parseM "Identifier" i `shouldBe` e' + parseM "Identifier" ("\"" ++ e ++ "\"") `shouldBe` e' + + describe "nixKeywords" $ do + it "are quoted" $ forM_ nixKeywords $ \str -> do + shouldSatisfy str needsQuoting + prettyShow (ident # str) `shouldBe` "\"" ++ str ++ "\"" + + describe "needsQuoting" $ do + it "if string contains non ASCII characters" $ stringIdentProperty $ \s -> + any (not . isAscii) s ==> needsQuoting s + it "if string contains spaces" $ stringIdentProperty $ \s -> + any isSpace s ==> needsQuoting s + it "if length is zero" $ shouldSatisfy "" needsQuoting + + describe "nix-instantiate" $ do + nixInstantiate <- runIO $ do + (callProcess nixInstantiateBin [ "--version" ] >> pure (Just nixInstantiateBin)) + `catch` (\(_ :: SomeException) -> pure Nothing) + let nix :: Example a => String -> (String -> a) -> SpecWith (Arg a) + nix str spec = + case nixInstantiate of + Nothing -> it str $ \_ -> + pendingWith (nixInstantiateBin ++ " could not be found or executed") + Just exec -> it str $ spec exec + + nix "accepts our identifiers and reproduces them" $ \exec -> identProperty $ \i -> ioProperty $ do + let expAttr = prettyShow i + expr = "{" ++ expAttr ++ "=null;}" + + out <- readCreateProcess (proc exec ["--eval", "--strict", "-E", expr]) "" + pure $ + extractIdentSyntax out === expAttr + .&&. parseM "Identifier" (extractIdentSyntax out) === Just i + +nixInstantiateBin :: String +nixInstantiateBin = "nix-instantiate" + +stringIdentProperty :: Testable prop => (String -> prop) -> Property +stringIdentProperty p = property $ \s -> + '\0' `notElem` s ==> classify (needsQuoting s) "need quoting" $ p s + +identProperty :: Testable prop => (Identifier -> prop) -> Property +identProperty p = property $ \i -> + classify (needsQuoting (from ident # i)) "need quoting" $ p i + +-- | Given the (pretty) printed representation of the Nix value produced by the +-- expression @{ ${ident} = null; }@, for any value of @ident@, extract the +-- part that represents the identifier. +-- +-- Note that pretty printing is buggy in some versions of Nix and the result +-- may not actually be valid Nix syntax. +extractIdentSyntax :: String -> String +extractIdentSyntax = + dropWhileEnd (`elem` "= \n\t") -- remove "… = " + . dropWhileEnd (`elem` "null") -- remove "null" + . dropWhileEnd (`elem` ";} \n\t") -- remove "…; }" + . dropWhile (`elem` "{ \n\t") -- remove "{ …"