diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,19 @@ +# Version [0.1.1.0](https://github.com/expipiplus1/update-nix-fetchgit/compare/0.1.0.0...0.1.1.0) + +* Changelog started. Previous release was `0.1.0.0`. + +* Additions + * `update-nix-fetchgit` will pass any extra arguments after the filename to `nix-prefetch-git`: + + ``` + update-nix-fetchgit filename.nix --rev refs/heads/myBranch + ``` + + * `Update.Span` module now exposes `split` + +--- + +`update-nix-fetchgit` uses [PVP Versioning][1]. + +[1]: https://pvp.haskell.org + diff --git a/README.md b/README.md new file mode 100644 --- /dev/null +++ b/README.md @@ -0,0 +1,67 @@ +# update-nix-fetchgit + +This is a command-line utility for updating `fetchgit`, `fetchgitPrivate`, and `fetchFromGitHub` calls in [Nix](http://nixos.org/nix/) expressions. This utility is meant to be used by people maintaining Nix expressions that fetch files from Git repositories. It automates the process of keeping such expressions up-to-date with the latest upstream sources. + +When you run `update-nix-fetchgit` on a file, it will: + +- Read the file and parse it as a Nix expression. +- Find all Git fetches (calls to `fetchgit`, `fetchgitPrivate`, or `fetchFromGitHub`). +- Run [`nix-prefetch-git`](https://github.com/NixOS/nixpkgs/blob/master/pkgs/build-support/fetchgit/nix-prefetch-git) to get information about the latest HEAD commit of each repository. +- Update the corresponding rev, sha256, and version attributes for each repository. +- Overwrite the original input file. + +Any `version` attribute found in the file will be updated if it is in a set that contains (directly or inderictly) a Git fetch. The version attribute will be updated to the commit date of the latest HEAD commit in the Git repository, in the time zone of the committer, in "YYYY-MM-DD" format. If the set contains multiple Git fetches, the latest such date is used. + +When this program fetches information from multiple repositories, it runs multiple instances of `nix-prefetch-git` in parallel. + + +# Usage + +Pass the name of the file to be updated as the first argument: + + update-nix-fetchgit filename.nix + +The file will be updated in place. + +## Extra arguments + +`update-nix-fetchgit` will pass any extra arguments after the filename to `nix-prefetch-git`: + + update-nix-fetchgit filename.nix --rev refs/heads/myBranch + + +# Example + +Here is an example of a Nix expression that can be updated by this program: + +```nix +{ stdenv, fetchgit }: + +stdenv.mkDerivation rec { + name = "foo-${version}"; + version = "2016-07-13"; + src = fetchgit { + url = "git://midipix.org/slibtool"; + rev = "4f56fd184ef6020626492a6f954a486d54f8b7ba"; + sha256 = "0nmyp5yrzl9dbq85wyiimsj9fklb8637a1936nw7zzvlnzkgh28n"; + }; +} +``` + +The `rev`, `sha256`, and `version` attributes will all be updated. + + +# Building from source + +The recommended way to build this program from source for development purposes is to download and run `nix-shell` in the top-level source directory and then run `cabal build`. + + +# More documentation + +You can run `update-nix-fetchgit --help` or `man update-nix-fetchgit` for more documentation. + + +# Authors + +- [expipiplus1](https://github.com/expipiplus1) - I'm `jophish` on Freenode; say hi! +- [DavidEGrayson](https://github.com/DavidEGrayson) diff --git a/app/Main.hs b/app/Main.hs --- a/app/Main.hs +++ b/app/Main.hs @@ -2,38 +2,19 @@ module Main where -import Data.Text.IO (readFile, writeFile) -import Prelude hiding (readFile, writeFile) -import System.Environment (getArgs) -import System.Exit -import System.IO (hPutStrLn, stderr) -import Update.Nix.FetchGit -import Update.Nix.FetchGit.Utils -import Update.Nix.FetchGit.Warning -import Update.Span +import qualified Data.Text +import qualified System.Environment +import qualified System.Exit +import qualified Update.Nix.FetchGit + main :: IO () main = -- Super simple command line parsing at the moment, just look for one - -- filename. - getArgs >>= \case - [filename] -> do - t <- readFile filename - -- Get the updates from this file. - updatesFromFile filename >>= \case - -- If we have any errors, print them and finish. - Left ws -> printErrorAndExit ws - Right us -> - -- Update the text of the file in memory. - case updateSpans us t of - -- If updates are needed, write to the file. - t' | t' /= t -> writeFile filename t' - _ -> return () + -- filename and optionally pass extra arguments to `nix-prefetch-git`. + System.Environment.getArgs >>= \case + [filename] -> Update.Nix.FetchGit.processFile filename [] + (filename:args) -> Update.Nix.FetchGit.processFile filename (map Data.Text.pack args) _ -> do - putStrLn "Usage: update-nix-fetchgit filename" - exitWith (ExitFailure 1) - -printErrorAndExit :: Warning -> IO () -printErrorAndExit e = do - hPutStrLn stderr (formatWarning e) - exitWith (ExitFailure 1) + putStrLn "Usage: update-nix-fetchgit filename []" + System.Exit.exitFailure diff --git a/src/Update/Nix/FetchGit.hs b/src/Update/Nix/FetchGit.hs --- a/src/Update/Nix/FetchGit.hs +++ b/src/Update/Nix/FetchGit.hs @@ -3,13 +3,14 @@ module Update.Nix.FetchGit ( updatesFromFile + , processFile ) where import Control.Concurrent.Async (mapConcurrently) import Control.Error import Data.Foldable (toList) import Data.Generics.Uniplate.Data (para) -import Data.Text (pack) +import Data.Text (Text, pack) import Nix.Expr import Update.Nix.FetchGit.Prefetch import Update.Nix.FetchGit.Types @@ -17,18 +18,45 @@ import Update.Nix.FetchGit.Warning import Update.Span +import qualified Data.Text.IO +import qualified System.IO +import qualified System.Exit + -------------------------------------------------------------------------------- -- Tying it all together -------------------------------------------------------------------------------- +-- | Provided FilePath, update Nix file in-place +processFile :: FilePath -> [Text] -> IO () +processFile filename args = do + t <- Data.Text.IO.readFile filename + -- Get the updates from this file. + updatesFromFile filename args >>= \case + -- If we have any errors, print them and finish. + Left ws -> printErrorAndExit ws + Right us -> + -- Update the text of the file in memory. + case updateSpans us t of + -- If updates are needed, write to the file. + t' | t' /= t -> do + Data.Text.IO.writeFile filename t' + putStrLn $ "Made " ++ (show $ length us) ++ " changes" + + _ -> putStrLn "No updates" + where + printErrorAndExit :: Warning -> IO () + printErrorAndExit e = do + System.IO.hPutStrLn System.IO.stderr (formatWarning e) + System.Exit.exitFailure + -- | Given the path to a Nix file, returns the SpanUpdates -- all the parts of the file we want to update. -updatesFromFile :: FilePath -> IO (Either Warning [SpanUpdate]) -updatesFromFile f = runExceptT $ do +updatesFromFile :: FilePath -> [Text] -> IO (Either Warning [SpanUpdate]) +updatesFromFile f extraArgs = runExceptT $ do expr <- ExceptT $ ourParseNixFile f treeWithArgs <- hoistEither $ exprToFetchTree expr treeWithLatest <- ExceptT $ - sequenceA <$> mapConcurrently getFetchGitLatestInfo treeWithArgs + sequenceA <$> mapConcurrently (getFetchGitLatestInfo extraArgs) treeWithArgs pure (fetchTreeToSpanUpdates treeWithLatest) -------------------------------------------------------------------------------- @@ -40,19 +68,19 @@ exprToFetchTree = para $ \e subs -> case e of -- If it is a call (application) of fetchgit, record the -- arguments since we will need to update them. - AnnE _ (NApp function (AnnE _ (NSet bindings))) + AnnE _ (NBinary NApp function (AnnE _ (NSet _rec bindings))) | extractFuncName function == Just "fetchgit" || extractFuncName function == Just "fetchgitPrivate" -> FetchNode <$> extractFetchGitArgs bindings -- Similarly, record calls to fetchFromGitHub. - AnnE _ (NApp function (AnnE _ (NSet bindings))) + AnnE _ (NBinary NApp function (AnnE _ (NSet _rec bindings))) | extractFuncName function == Just "fetchFromGitHub" -> FetchNode <$> extractFetchFromGitHubArgs bindings -- If it is an attribute set, find any attributes in it that we -- might want to update. - AnnE _ (NSet bindings) + AnnE _ (NSet _rec bindings) -> Node <$> findAttr "version" bindings <*> sequenceA subs -- If this is something uninteresting, just wrap the sub-trees. @@ -77,9 +105,9 @@ -- Getting updated information from the internet. -------------------------------------------------------------------------------- -getFetchGitLatestInfo :: FetchGitArgs -> IO (Either Warning FetchGitLatestInfo) -getFetchGitLatestInfo args = runExceptT $ do - o <- ExceptT (nixPrefetchGit (extractUrlString $ repoLocation args)) +getFetchGitLatestInfo :: [Text] -> FetchGitArgs -> IO (Either Warning FetchGitLatestInfo) +getFetchGitLatestInfo extraArgs args = runExceptT $ do + o <- ExceptT (nixPrefetchGit extraArgs (extractUrlString $ repoLocation args)) d <- hoistEither (parseISO8601DateToDay (date o)) pure $ FetchGitLatestInfo args (rev o) (sha256 o) d diff --git a/src/Update/Nix/FetchGit/Prefetch.hs b/src/Update/Nix/FetchGit/Prefetch.hs --- a/src/Update/Nix/FetchGit/Prefetch.hs +++ b/src/Update/Nix/FetchGit/Prefetch.hs @@ -10,7 +10,7 @@ import Control.Monad.IO.Class (liftIO) import Data.Aeson (FromJSON, decode) import Data.ByteString.Lazy.UTF8 (fromString) -import Data.Text +import Data.Text (Text, pack, unpack) import GHC.Generics import System.Exit (ExitCode (..)) import System.Process (readProcessWithExitCode) @@ -26,11 +26,12 @@ deriving (Show, Generic, FromJSON) -- | Run nix-prefetch-git -nixPrefetchGit :: Text -- ^ The URL to prefetch +nixPrefetchGit :: [Text] -- ^ Extra arguments for nix-prefetch-git + -> Text -- ^ The URL to prefetch -> IO (Either Warning NixPrefetchGitOutput) -nixPrefetchGit prefetchURL = runExceptT $ do +nixPrefetchGit extraArgs prefetchURL = runExceptT $ do (exitCode, nsStdout, nsStderr) <- liftIO $ - readProcessWithExitCode "nix-prefetch-git" [unpack prefetchURL] "" + readProcessWithExitCode "nix-prefetch-git" (map unpack extraArgs ++ [unpack prefetchURL]) "" hoistEither $ case exitCode of ExitFailure e -> Left (NixPrefetchGitFailed e (pack nsStderr)) ExitSuccess -> pure () diff --git a/src/Update/Nix/FetchGit/Utils.hs b/src/Update/Nix/FetchGit/Utils.hs --- a/src/Update/Nix/FetchGit/Utils.hs +++ b/src/Update/Nix/FetchGit/Utils.hs @@ -11,21 +11,27 @@ , extractFuncName , extractAttr , findAttr + , matchAttr , exprText , exprSpan , parseISO8601DateToDay , formatWarning ) where -import Control.Error (lastMay) -import Data.Generics.Uniplate.Data (transform) -import Data.Maybe (catMaybes) -import Data.Monoid ((<>)) -import Data.Text (Text, unpack, splitOn) -import Data.Time (parseTimeM, defaultTimeLocale) -import Nix.Parser (parseNixFileLoc, Result(..)) -import Text.Trifecta.Result (_errDoc) -import Nix.Expr +import Data.Maybe ( catMaybes ) +import Data.List.NonEmpty as NE +import Data.Text ( Text + , unpack + , splitOn + ) +import Data.Time ( parseTimeM + , defaultTimeLocale + ) +import Nix.Parser ( parseNixFileLoc + , Result(..) + ) +import Nix.Reduce +import Nix.Expr hiding ( SourcePos ) import Update.Nix.FetchGit.Types import Update.Nix.FetchGit.Warning import Update.Span @@ -33,16 +39,8 @@ ourParseNixFile :: FilePath -> IO (Either Warning NExprLoc) ourParseNixFile f = parseNixFileLoc f >>= \case - Failure parseError -> pure $ Left (CouldNotParseInput (_errDoc parseError)) - Success expr -> pure $ pure $ fixNixSets expr - --- Convert all NRecSet values (recursive sets) to NSet values because --- we do not care about the distinction between NRecSet and NSet and --- we want our program to treat both types equally. -fixNixSets :: NExprLoc -> NExprLoc -fixNixSets = transform fix - where fix (AnnE s (NRecSet bindings)) = AnnE s (NSet bindings) - fix n = n + Failure parseError -> pure $ Left (CouldNotParseInput parseError) + Success expr -> pure <$> reduceExpr Nothing expr -- | Get the url from either a nix expression for the url or a repo and owner -- expression. @@ -65,22 +63,17 @@ (AnnE _ (NStr (DoubleQuoted [Plain t]))) -> pure t e -> Left (NotAString e) --- | Get the 'SourceSpan' covering a particular expression. -exprSpan :: NExprLoc -> SourceSpan -exprSpan expr = SourceSpan (deltaToSourcePos begin) (deltaToSourcePos end) - where (AnnE (SrcSpan begin end) _) = expr - --- | Go from a 'Delta' to a 'SourcePos'. -deltaToSourcePos :: Delta -> SourcePos -deltaToSourcePos delta = SourcePos line column - where (Directed _ line column _ _) = delta +-- | Get the 'SrcSpan' covering a particular expression. +exprSpan :: NExprLoc -> SrcSpan +exprSpan (AnnE s _) = s +exprSpan _ = error "unreachable" -- TODO: Add pattern completeness to hnix -- | Given an expression that is supposed to represent a function, -- extracts the name of the function. If we cannot figure out the -- function name, returns Nothing. extractFuncName :: NExprLoc -> Maybe Text extractFuncName (AnnE _ (NSym name)) = Just name -extractFuncName (AnnE _ (NSelect _ (lastMay -> Just (StaticKey name)) _)) = Just name +extractFuncName (AnnE _ (NSelect _ (NE.last -> StaticKey name) _)) = Just name extractFuncName _ = Nothing -- | Extract a named attribute from an attrset. @@ -102,9 +95,9 @@ -- Nothing. matchAttr :: Text -> Binding a -> Maybe a matchAttr t = \case - NamedVar [StaticKey t'] x | t == t' -> Just x - NamedVar _ _ -> Nothing - Inherit _ _ -> Nothing + NamedVar (StaticKey t' :|[]) x _ | t == t' -> Just x + NamedVar _ _ _ -> Nothing + Inherit _ _ _ -> Nothing -- Takes an ISO 8601 date and returns just the day portion. parseISO8601DateToDay :: Text -> Either Warning Day @@ -122,7 +115,7 @@ "Error: The \"" <> unpack attrName <> "\" attribute appears twice in a set." formatWarning (NotAString expr) = "Error: The expression at " - <> (prettyPrintSourcePos . sourceSpanBegin . exprSpan) expr + <> (prettyPrintSourcePos . spanBegin . exprSpan) expr <> " is not a string literal." formatWarning (NixPrefetchGitFailed exitCode errorOutput) = "Error: nix-prefetch-git failed with exit code " <> show exitCode diff --git a/src/Update/Nix/FetchGit/Warning.hs b/src/Update/Nix/FetchGit/Warning.hs --- a/src/Update/Nix/FetchGit/Warning.hs +++ b/src/Update/Nix/FetchGit/Warning.hs @@ -4,9 +4,10 @@ import Data.Text import Nix.Expr -import Text.PrettyPrint.ANSI.Leijen (Doc) +import Data.Text.Prettyprint.Doc +import Data.Void -data Warning = CouldNotParseInput Doc +data Warning = CouldNotParseInput (Doc Void) | MissingAttr Text | DuplicateAttrs Text | NotAString NExprLoc diff --git a/src/Update/Span.hs b/src/Update/Span.hs --- a/src/Update/Span.hs +++ b/src/Update/Span.hs @@ -6,39 +6,26 @@ module Update.Span ( SpanUpdate(..) + , SrcSpan(..) , SourcePos(..) - , SourceSpan(..) , updateSpan , updateSpans , linearizeSourcePos , prettyPrintSourcePos + , split ) where import Control.Exception (assert) import Data.Data (Data) import Data.Int (Int64) import Data.List (genericTake, sortOn) -import Data.Monoid ((<>)) import Data.Text (Text, length, lines, splitAt) import Prelude hiding (length, lines, splitAt) - --- | A position in a text file -data SourcePos = SourcePos{ sourcePosRow :: Int64, - sourcePosColumn :: Int64 - } - deriving (Show, Eq, Ord, Data) - --- | A span of characters with a beginning and an end. --- --- 'spanEnd' must be greater than 'spanBegin' -data SourceSpan = SourceSpan{ sourceSpanBegin :: SourcePos - , sourceSpanEnd :: SourcePos - } - deriving (Show, Data) +import Nix.Expr.Types.Annotated -- | A span and some text to replace it with. -- They don't have to be the same length. -data SpanUpdate = SpanUpdate{ spanUpdateSpan :: SourceSpan +data SpanUpdate = SpanUpdate{ spanUpdateSpan :: SrcSpan , spanUpdateContents :: Text } deriving (Show, Data) @@ -46,7 +33,7 @@ -- | Update many spans in a file. They must be non-overlapping. updateSpans :: [SpanUpdate] -> Text -> Text updateSpans us t = - let sortedSpans = sortOn (sourceSpanBegin . spanUpdateSpan) us + let sortedSpans = sortOn (spanBegin . spanUpdateSpan) us anyOverlap = any (uncurry overlaps) (zip <*> tail $ spanUpdateSpan <$> sortedSpans) in @@ -56,20 +43,23 @@ -- | Update a single span of characters inside a text value. If you're updating -- multiples spans it's best to use 'updateSpans'. updateSpan :: SpanUpdate -> Text -> Text -updateSpan (SpanUpdate (SourceSpan b e) r) t = +updateSpan (SpanUpdate (SrcSpan b e) r) t = let (before, _) = split b t (_, end) = split e t in before <> r <> end -- | Do two spans overlap -overlaps :: SourceSpan -> SourceSpan -> Bool -overlaps (SourceSpan b1 e1) (SourceSpan b2 e2) = +overlaps :: SrcSpan -> SrcSpan -> Bool +overlaps (SrcSpan b1 e1) (SrcSpan b2 e2) = b2 >= b1 && b2 < e1 || e2 >= b1 && e2 < e1 -- | Split some text at a particular 'SourcePos' split :: SourcePos -> Text -> (Text, Text) -split (SourcePos row col) t = - splitAt (fromIntegral (linearizeSourcePos t row col)) t +split (SourcePos _ row col) t = splitAt + (fromIntegral + (linearizeSourcePos t (fromIntegral (unPos row - 1)) (fromIntegral (unPos col - 1))) + ) + t -- | Go from a line and column representation to a single character offset from -- the beginning of the text. @@ -83,5 +73,5 @@ where lineCharOffset = sum . fmap ((+1) . length) . genericTake l . lines $ t prettyPrintSourcePos :: SourcePos -> String -prettyPrintSourcePos (SourcePos row column) = +prettyPrintSourcePos (SourcePos _ row column) = "line " <> show row <> " column " <> show column diff --git a/update-nix-fetchgit.cabal b/update-nix-fetchgit.cabal --- a/update-nix-fetchgit.cabal +++ b/update-nix-fetchgit.cabal @@ -1,7 +1,8 @@ name: update-nix-fetchgit -version: 0.1.0.0 +version: 0.1.1.0 synopsis: A program to update fetchgit values in Nix expressions -description: Please see README.md +description: This command-line utility is meant to be used by people maintaining Nix expressions that fetch files from Git repositories. + It automates the process of keeping such expressions up-to-date with the latest upstream sources. homepage: https://github.com/expipiplus1/update-nix-fetchgit#readme license: BSD3 license-file: LICENSE @@ -10,8 +11,11 @@ copyright: 2015 Joe Hermaszewski category: Nix build-type: Simple --- extra-source-files: cabal-version: >=1.10 +extra-source-files: + CHANGELOG.md + LICENSE + README.md library hs-source-dirs: src @@ -22,20 +26,20 @@ , Update.Nix.FetchGit.Utils other-modules: Update.Nix.FetchGit.Types build-depends: base >= 4.7 && < 5 - , aeson >= 0.9 && < 0.12 - , ansi-wl-pprint >= 0.6 && < 0.7 - , async >= 2.1 && < 2.2 - , bytestring >= 0.10 && < 0.11 - , data-fix >= 0.0 && < 0.1 - , hnix >= 0.3.2 && < 0.4 - , process >= 1.2 && < 1.5 - , text >= 1.2 && < 1.3 - , time >= 1.5 && < 1.7 - , transformers >= 0.4 && < 0.6 - , trifecta >= 1.6 && < 1.7 - , uniplate >= 1.6 && < 1.7 - , utf8-string >= 1.0 && < 1.1 - , errors >= 2.1 && < 2.2 + , aeson >= 0.9 + , async >= 2.1 + , bytestring >= 0.10 + , data-fix >= 0.0 + , errors >= 2.1 + , hnix >= 0.8 + , prettyprinter + , process >= 1.2 + , text >= 1.2 + , time >= 1.5 + , transformers >= 0.4 + , trifecta >= 1.6 + , uniplate >= 1.6 + , utf8-string >= 1.0 default-language: Haskell2010 ghc-options: -Wall @@ -44,7 +48,7 @@ main-is: Main.hs ghc-options: -threaded -rtsopts -with-rtsopts=-N -Wall build-depends: base - , text >= 1.2 && < 1.3 + , text >= 1.2 , update-nix-fetchgit default-language: Haskell2010