packages feed

nixpkgs-update (empty) → 0.2.0

raw patch · 17 files changed

+1630/−0 lines, 17 filesdep +basedep +directorydep +doctestsetup-changed

Dependencies added: base, directory, doctest, errors, filepath, github, mtl, neat-interpolation, optparse-applicative, regex-applicative, shelly, text, time, unix, vector

Files

+ LICENSE view
@@ -0,0 +1,1 @@+To the extent possible under law, Ryan Mulligan has waived all copyright and related or neighboring rights to repology-api. This work is published from: United States.
+ README.md view
@@ -0,0 +1,24 @@+# nixpkgs-update++[![Build Status](https://travis-ci.org/ryantm/nixpkgs-update.svg?branch=master)](https://travis-ci.org/ryantm/nixpkgs-update)++Scripts to try to update nixpkgs packages. Uses `hub` to automatically make PRs.++# Instructions++1. Clone this repo and build the tool:+    ```+    git clone https://github.com/ryantm/nixpkgs-update && cd nixpkgs-update+    nix run nixpkgs.cabal2nix -c cabal2nix --shell --hpack . > shell.nix && nix-build shell.nix+    ```+2. Get a list of oudated packages and place them in a `packages-to-update.txt` file in the root directory of this repository.+    ```+    git clone https://github.com/ryantm/repology-api.git && cd repology-api+    nix run nixpkgs.cabal2nix -c cabal2nix --shell --hpack . > shell.nix && nix-build shell.nix && result/bin/repology-api > ../packages-to-update.txt+    ```+3. Return back `cd ..` and run the tool `nix run -f '<nixpkgs>' gitAndTools.hub gist jq tree -c result/bin/nixpkgs-update --update`++# Prior work++* https://github.com/NixOS/nixpkgs/blob/master/pkgs/common-updater/scripts/update-source-version+* https://github.com/NixOS/nixpkgs/tree/master/pkgs/build-support/upstream-updater
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ nixpkgs-update.cabal view
@@ -0,0 +1,86 @@+-- This file has been generated from package.yaml by hpack version 0.28.2.+--+-- see: https://github.com/sol/hpack+--+-- hash: dbfac301b2851ef0eb87cd2dc7989a2970db94cdaa891199031c4115cbb6f01c++name:           nixpkgs-update+version:        0.2.0+synopsis:       Tool for semi-automatic updating of nixpkgs repository+description:    nixpkgs-update provides tools for updating of nixpkgs packages in a semi-automatic way. Mainly, it is used to run the GitHub bot @r-ryantm, but the underlying update mechanisms should be generally useful and in a later version should be exposed as a command-line tool.+category:       Web+homepage:       https://github.com/ryantm/nixpkgs-update#readme+bug-reports:    https://github.com/ryantm/nixpkgs-update/issues+author:         Ryan Mulligan+maintainer:     ryan@ryantm.com+copyright:      2018 Ryan Mulligan+license:        PublicDomain+license-file:   LICENSE+build-type:     Simple+cabal-version:  >= 1.10+extra-source-files:+    README.md++source-repository head+  type: git+  location: https://github.com/ryantm/nixpkgs-update++executable nixpkgs-update+  main-is: Main.hs+  other-modules:+      Blacklist+      Check+      Clean+      DeleteMerged+      File+      GH+      Git+      Nix+      Outpaths+      Update+      Utils+      Paths_nixpkgs_update+  hs-source-dirs:+      src+  build-depends:+      base >=4.7 && <5+    , directory >=1.3 && <1.4+    , errors+    , filepath+    , github+    , mtl+    , neat-interpolation >=0.3 && <0.4+    , optparse-applicative+    , regex-applicative+    , shelly+    , text+    , time >=1.8 && <1.10+    , unix+    , vector+  default-language: Haskell2010++test-suite doctests+  type: exitcode-stdio-1.0+  main-is: doctests.hs+  other-modules:+      Paths_nixpkgs_update+  hs-source-dirs:+      test+  ghc-options: -threaded+  build-depends:+      base >=4.7 && <5+    , directory >=1.3 && <1.4+    , doctest+    , errors+    , filepath+    , github+    , mtl+    , neat-interpolation >=0.3 && <0.4+    , optparse-applicative+    , regex-applicative+    , shelly+    , text+    , time >=1.8 && <1.10+    , unix+    , vector+  default-language: Haskell2010
+ src/Blacklist.hs view
@@ -0,0 +1,142 @@+{-# LANGUAGE OverloadedStrings #-}++module Blacklist+  ( packageName+  , content+  , srcUrl+  , attrPath+  , checkResult+  ) where++import Data.Foldable (find)+import Data.Text (Text)+import qualified Data.Text as T++type Blacklist = [(Text -> Bool, Text)]++srcUrl :: Text -> Either Text ()+srcUrl = blacklister srcUrlList++attrPath :: Text -> Either Text ()+attrPath = blacklister attrPathList++packageName :: Text -> Either Text ()+packageName name =+  if (name == "elementary-xfce-icon-theme") -- https://github.com/ryantm/nixpkgs-update/issues/63+    then Right ()+    else blacklister nameList name++content :: Text -> Either Text ()+content = blacklister contentList++checkResult :: Text -> Either Text ()+checkResult = blacklister checkResultList++srcUrlList :: Blacklist+srcUrlList =+  [(("gnome" `T.isInfixOf`), "Packages from gnome are currently blacklisted.")]++attrPathList :: Blacklist+attrPathList =+  [ prefix+      "lua"+      "Packages for lua are currently blacklisted. https://github.com/NixOS/nixpkgs/pull/37501#issuecomment-375169646"+  , prefix "lxqt" "Packages for lxqt are currently blacklisted."+  , prefix+      "altcoins.bitcoin-xt"+      "nix-prefetch-url has infinite redirect https://github.com/NixOS/nix/issues/2225 remove after Nix upgrade that includes https://github.com/NixOS/nix/commit/b920b908578d68c7c80f1c1e89c42784693e18d5."+  , prefix+      "altcoins.bitcoin"+      "@roconnor asked for a blacklist on this until something can be done with GPG signatures https://github.com/NixOS/nixpkgs/commit/77f3ac7b7638b33ab198330eaabbd6e0a2e751a9"+  , eq "sqlite-interactive" "it is an override"+  , eq "harfbuzzFull" "it is an override"+  ]++nameList :: Blacklist+nameList =+  [ prefix "r-" "we don't know how to find the attrpath for these"+  , infixOf "jquery" "this isn't a real package"+  , infixOf "google-cloud-sdk" "complicated package"+  , infixOf "github-release" "complicated package"+  , infixOf+      "libxc"+      "currently people don't want to update this https://github.com/NixOS/nixpkgs/pull/35821"+  , infixOf "perl" "currently don't know how to update perl"+  , infixOf "python" "currently don't know how to update python"+  , infixOf "cdrtools" "We keep downgrading this by accident."+  , infixOf "gst" "gstreamer plugins are kept in lockstep."+  , infixOf "electron" "multi-platform srcs in file."+  , infixOf+      "linux-headers"+      "Not updated until many packages depend on it (part of stdenv)."+  , infixOf "xfce" "@volth asked to not update xfce"+  , infixOf "cmake-cursesUI-qt4UI" "Derivation file is complicated"+  , infixOf "iana-etc" "@mic92 takes care of this package"+  , infixOf+      "checkbashism"+      "needs to be fixed, see https://github.com/NixOS/nixpkgs/pull/39552"+  , eq "isl" "multi-version long building package"+  , infixOf "qscintilla" "https://github.com/ryantm/nixpkgs-update/issues/51"+  , eq "itstool" "https://github.com/NixOS/nixpkgs/pull/41339"+  , eq+      "wire-desktop"+      "nixpkgs-update cannot handle this derivation https://github.com/NixOS/nixpkgs/pull/42936#issuecomment-402282692"+  , infixOf+      "virtualbox"+      "nixpkgs-update cannot handle updating the guest additions https://github.com/NixOS/nixpkgs/pull/42934"+  , eq+      "avr-binutils"+      "https://github.com/NixOS/nixpkgs/pull/43787#issuecomment-408649537"+  , eq+      "iasl"+      "two updates had to be reverted, https://github.com/NixOS/nixpkgs/pull/46272"+  , eq+      "meson"+      "https://github.com/NixOS/nixpkgs/pull/47024#issuecomment-423300633"+  , eq+      "burp"+      "temporary blacklist until better versioning schema https://github.com/NixOS/nixpkgs/pull/46298#issuecomment-419536301"+  ]++contentList :: Blacklist+contentList =+  [ infixOf "DO NOT EDIT" "Derivation file says not to edit it."+  , infixOf "Do not edit!" "Derivation file says not to edit it."+    -- Skip packages that have special builders+  , infixOf "buildGoPackage" "Derivation contains buildGoPackage."+  , infixOf "buildRustCrate" "Derivation contains buildRustCrate."+  , infixOf "buildPythonPackage" "Derivation contains buildPythonPackage."+  , infixOf "buildRubyGem" "Derivation contains buildRubyGem."+  , infixOf "bundlerEnv" "Derivation contains bundlerEnv."+  , infixOf "buildPerlPackage" "Derivation contains buildPerlPackage."+  ]++checkResultList :: Blacklist+checkResultList =+  [ infixOf+      "busybox"+      "- busybox result is not automatically checked, because some binaries kill the shell"+  , infixOf+      "fcitx"+      "- fcitx result is not automatically checked, because some binaries gets stuck in daemons"+  , infixOf+      "x2goclient"+      "- x2goclient result is not automatically checked, because some binaries don't timeout properly"+  ]++blacklister :: Blacklist -> Text -> Either Text ()+blacklister blacklist input =+  case result of+    Nothing -> Right ()+    Just msg -> Left msg+  where+    result = snd <$> find (\(isBlacklisted, _) -> isBlacklisted input) blacklist++prefix :: Text -> Text -> (Text -> Bool, Text)+prefix part reason = ((part `T.isPrefixOf`), reason)++infixOf :: Text -> Text -> (Text -> Bool, Text)+infixOf part reason = ((part `T.isInfixOf`), reason)++eq :: Text -> Text -> (Text -> Bool, Text)+eq part reason = ((part ==), reason)
+ src/Check.hs view
@@ -0,0 +1,145 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module Check+  ( result+  ) where++import Control.Applicative (many)+import Control.Monad (forM)+import Data.Char (isSpace)+import Data.Maybe (isJust)+import Data.Semigroup ((<>))+import qualified Data.Text as T+import Data.Text (Text)+import Prelude hiding (FilePath)+import Shelly+import qualified Text.Regex.Applicative as RE+import Text.Regex.Applicative (RE, (=~))+import Utils (Options(..), UpdateEnv (..), Version, canFail, succeded)++default (T.Text)++data BinaryCheck = BinaryCheck+  { filePath :: FilePath+  , zeroExitCode :: Bool+  , versionPresent :: Bool+  }++-- | Construct regex: [^\.]*${version}\.*\s*+versionRegex :: Text -> RE Char ()+versionRegex version =+  (\_ _ _ _ -> ()) <$> many (RE.psym (/= '.')) <*> RE.string (T.unpack version) <*>+  many (RE.sym '.') <*>+  many (RE.psym isSpace)++-- | Run a program with provided argument and report whether the output+-- mentions the expected version+checkBinary :: Text -> Version -> FilePath -> Sh BinaryCheck+checkBinary argument expectedVersion program =+  catchany_sh+  (do+      stdout <- canFail $ cmd "timeout" "-k" "2" "1" program argument+      code <- lastExitCode+      stderr <- lastStderr+      let hasVersion =+            isJust $ (T.unpack . T.unwords . T.lines $ stdout <> "\n" <> stderr) =~+            versionRegex expectedVersion+      return $ BinaryCheck program (code == 0) hasVersion)+  (\ _ -> return $ BinaryCheck program False False)++checks :: [Version -> FilePath -> Sh BinaryCheck]+checks =+  [ checkBinary ""+  , checkBinary "-V"+  , checkBinary "-v"+  , checkBinary "--version"+  , checkBinary "version"+  , checkBinary "-h"+  , checkBinary "--help"+  , checkBinary "help"+  ]++someChecks :: BinaryCheck -> [Sh BinaryCheck] -> Sh BinaryCheck+someChecks best [] = return best+someChecks best (c:rest) = do+  current <- c+  let nb = newBest current+  case nb of+    BinaryCheck _ True True -> return nb+    _ -> someChecks nb rest+  where+    newBest :: BinaryCheck -> BinaryCheck+    newBest (BinaryCheck _ currentExit currentVersionPresent) =+      BinaryCheck+      (filePath best)+      (zeroExitCode best || currentExit)+      (versionPresent best || currentVersionPresent)++-- | Run a program with various version or help flags and report+-- when they succeded+runChecks :: Version -> FilePath -> Sh BinaryCheck+runChecks expectedVersion program =+  someChecks (BinaryCheck program False False) checks'+  where+    checks' = map (\c -> c expectedVersion program) checks++checkReport :: BinaryCheck -> Text+checkReport (BinaryCheck p False False) = "- Warning: no invocation of " <> toTextIgnore p <> " had a zero exit code or showed the expected version"+checkReport (BinaryCheck p _ _) = "- " <> toTextIgnore p <> " passed the binary check."++successfullCheck :: BinaryCheck -> Bool+successfullCheck (BinaryCheck _ False False) = False+successfullCheck _ = True++result :: UpdateEnv -> FilePath  -> Sh Text+result updateEnv resultPath = do+  let expectedVersion = newVersion updateEnv+  home <- get_env_text "HOME"+  let logFile = workingDir (options updateEnv) </> "check-result-log.tmp"+  setenv "EDITOR" "echo"+  setenv "HOME" "/homeless-shelter"+  let addToReport input = appendfile logFile (input <> "\n")+  tempdir <- fromText . T.strip <$> cmd "mktemp" "-d"+  chdir tempdir $ do+    rm_f logFile+    let binaryDir = resultPath </> "bin"+    binExists <- test_d binaryDir+    binaries <-+      if binExists+        then findWhen test_f (resultPath </> "bin")+        else return []+    checks <- forM binaries $ \binary -> runChecks expectedVersion binary+    addToReport (T.intercalate "\n" (map checkReport checks))+    let passedZeroExitCode = (T.pack . show)+          (foldl+           (\sum c -> if zeroExitCode c then sum + 1 else sum) 0 checks :: Int)+        passedVersionPresent = (T.pack . show)+          (foldl+           (\sum c -> if versionPresent c then sum + 1 else sum) 0 checks :: Int)+        numBinaries = (T.pack . show) (length binaries)++    addToReport ("- " <> passedZeroExitCode <> " of " <> numBinaries <> " passed binary check by having a zero exit code.")+    addToReport ("- " <> passedVersionPresent <> " of " <> numBinaries <> " passed binary check by having the new version present in output.")+    canFail $ cmd "grep" "-r" expectedVersion resultPath+    whenM ((== 0) <$> lastExitCode) $+      addToReport $+        "- found " <> expectedVersion <> " with grep in " <>+        toTextIgnore resultPath+    whenM+      (null <$>+       findWhen+         (\path ->+            ((expectedVersion `T.isInfixOf` toTextIgnore path) &&) <$>+            test_f path)+         resultPath) $+      addToReport $+        "- found " <> expectedVersion <> " in filename of file in " <>+        toTextIgnore resultPath+    setenv "HOME" home+    gist1 <- cmd "tree" resultPath -|- cmd "gist"+    unless (T.null gist1) $ addToReport $ "- directory tree listing: " <> T.strip gist1+    gist2 <- cmd "du" "-h" resultPath -|- cmd "gist"+    unless (T.null gist2) $ addToReport $ "- du listing: " <> T.strip gist2+  canFail $ readfile logFile
+ src/Clean.hs view
@@ -0,0 +1,115 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module Clean+  ( fixSrcUrl+  ) where++import Control.Applicative ((<|>), some)+import Control.Error+import Control.Monad (forM_)+import Control.Monad.Trans+import Data.Function ((&))+import Data.Maybe (isNothing)+import Data.Semigroup ((<>))+import qualified Data.Text as T+import Data.Text (Text)+import qualified File+import Prelude hiding (FilePath)+import Shelly+import qualified Text.Regex.Applicative as RE+import Text.Regex.Applicative (RE, (=~))+import Utils (UpdateEnv(..), Version, canFail, setupNixpkgs, succeded)++default (T.Text)++-- | Construct regex: ${version}[^/]+\.(tar|zip)+archiveRegex :: Version -> RE Char ()+archiveRegex version =+  (\_ _ _ _ -> ()) <$> RE.string (T.unpack version) <*> some (RE.psym (/= '/')) <*>+  RE.sym '.' <*>+  (RE.string "tar" <|> RE.string "zip")++fixSrcUrl :: UpdateEnv -> FilePath -> Text -> Text -> Sh Text+fixSrcUrl updateEnv derivationFile attrPath oldSrcUrl = do+  setupNixpkgs+  oldDerivationName <-+    T.strip <$>+    cmd "nix" "eval" "-f" "." "--raw" ("pkgs." <> attrPath <> ".name")+  let newDerivationName =+        T.replace+          (oldVersion updateEnv)+          (newVersion updateEnv)+          oldDerivationName+  name <-+    T.strip <$>+    cmd+      "nix"+      "eval"+      "--raw"+      ("(let pkgs = import ./. {}; in (builtins.parseDrvName pkgs." <> attrPath <>+       ".name).name)")+  whenM+    (succeded $+     cmd "grep" "-q" ("name = \"" <> newDerivationName <> "\"") derivationFile) $+    -- Separate name and version+   do+    let newName = name <> "-${version}"+    File.replace newDerivationName newName derivationFile+    cmd "grep" "-q" ("name = \"" <> newName <> "\"") derivationFile+    cmd+      "sed"+      "-i"+      ("s|^\\([ ]*\\)\\(name = \"" <> name <>+       "-${version}\";\\)|\\1\\2\n\\1version = \"" <>+       newVersion updateEnv <>+       "\";|")+      derivationFile+    cmd+      "grep"+      "-q"+      ("version = \"" <> newVersion updateEnv <> "\";")+      derivationFile+  -- Obtain download URLs from repology+  -- TODO: use repology-api package+  downloads <-+    cmd+      "curl"+      "-s"+      ("https://repology.org/api/v1/metapackage/" <> packageName updateEnv) -|-+    cmd "jq" ".[].downloads | select(values) | .[]"+  let filteredDownloads =+        downloads & T.lines &+        filter+          (\url ->+             newVersion updateEnv `T.isInfixOf` url &&+             isNothing (T.unpack url =~ archiveRegex (newVersion updateEnv))) &+        map (T.replace "\"" "")+  forResult <-+    runExceptT $+    runExceptRT $+    forM_ filteredDownloads $ \downloadUrl -> do+      let oldUrl =+            T.replace+              (oldVersion updateEnv)+              "${version}"+              (T.replace oldDerivationName "${name}" oldSrcUrl)+      let newUrl =+            T.replace+              (newVersion updateEnv)+              "${version}"+              (T.replace newDerivationName "${name}" downloadUrl)+      lift $ File.replace oldUrl newUrl derivationFile+      lift $ cmd "grep" "-q" ("url = \"" <> newUrl <> "\";") derivationFile+      whenM+        (lift $+         succeded $+         cmd "grep" "-q" ("url = \"" <> newUrl <> "\";") derivationFile) $ do+        hash <-+          lift $ canFail $ cmd "nix-prefetch-url" "-A" (attrPath <> ".src")+        succeedT hash+      return ()+  case forResult of+    Right hash -> return hash+    _ -> cmd "false"
+ src/DeleteMerged.hs view
@@ -0,0 +1,52 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module DeleteMerged+  ( deleteMerged+  , deleteDone+  ) where++import Control.Monad (forM_)+import Data.Function ((&))+import Data.Maybe (mapMaybe)+import Data.Semigroup ((<>))+import qualified Data.Text as T+import qualified Data.Text.IO as T+import qualified Git+import qualified GH+import Shelly+import Utils (Options, ourShell, setupNixpkgs)+import qualified Data.Vector as V++default (T.Text)++-- | Delete the already merged branches both from local and remote repository+deleteMerged :: Options -> IO ()+deleteMerged o =+  ourShell o $ do+    setupNixpkgs+    Git.fetch+    Git.cleanAndResetToMaster+    mergedRemoteBranches <- T.lines <$> cmd "git" "branch" "-ra" "--merged"+    let mergedRemoteAutoUpdateBranches =+          mergedRemoteBranches & filter ("origin/auto-update/" `T.isInfixOf`) &+          mapMaybe (T.stripPrefix "remotes/origin/" . T.strip)+    forM_ mergedRemoteAutoUpdateBranches $ \branch ->+      cmd "git" "push" "origin" (":" <> branch)+    mergedBranches <- T.lines <$> cmd "git" "branch" "-a" "--merged"+    let mergedAutoUpdateBranches =+          mergedBranches & filter ("auto-update/" `T.isInfixOf`) & map T.strip+    forM_ mergedAutoUpdateBranches $ \branch -> cmd "git" "branch" "-d" branch++deleteDone :: Options -> IO ()+deleteDone o =+  ourShell o $ do+    setupNixpkgs+    Git.fetch+    Git.cleanAndResetToMaster+    result <- liftIO $ GH.closedAutoUpdateRefs o+    case result of+      Left error -> liftIO $ T.putStrLn error+      Right refs ->+        V.sequence_ (fmap (\r -> Git.deleteBranch ("auto-update/" <> r)) refs)
+ src/File.hs view
@@ -0,0 +1,19 @@+module File+  ( replace+  ) where++import           Data.Text (Text)+import qualified Data.Text as T+import           Data.Text.IO as T+import qualified Shelly+import           Shelly (liftIO, toTextIgnore, Sh)++replaceIO :: Text -> Text -> FilePath -> IO ()+replaceIO find r file = do+  contents <- T.readFile file+  T.writeFile file (T.replace find r contents)++replace :: Text -> Text -> Shelly.FilePath -> Sh ()+replace find r file = liftIO $ replaceIO find r f+  where+    f = (T.unpack . toTextIgnore) file
+ src/GH.hs view
@@ -0,0 +1,109 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module GH+  ( GH.releaseUrl+  , compareUrl+  , pr+  , closedAutoUpdateRefs+  ) where++import Control.Category ((>>>))+import Control.Error+import Data.Bifunctor (bimap, first, second)+import Data.Function ((&))+import Data.Maybe (fromJust)+import Data.Semigroup ((<>))+import qualified Data.Text as T+import Data.Text (Text)+import qualified Data.Text.Encoding as T+import qualified Data.Vector as V+import Data.Vector (Vector)+import GitHub+import GitHub.Data.Name (Name(..), untagName)+import GitHub.Endpoints.GitData.References (references')+import GitHub.Endpoints.Repos.Releases (releaseByTagName)+import Shelly hiding (tag)+import Utils++gReleaseUrl :: URLParts -> IO (Either Text Text)+gReleaseUrl (URLParts owner repo tag) =+  bimap (T.pack . show) (getUrl . releaseHtmlUrl) <$>+  releaseByTagName owner repo tag++releaseUrl :: Text -> IO (Either Text Text)+releaseUrl url =+  runExceptT $ do+    urlParts <- ExceptT $ parseURL url+    ExceptT $ gReleaseUrl urlParts++pr :: Text -> Sh ()+pr = cmd "hub" "pull-request" "-m"++data URLParts = URLParts+  { owner :: Name Owner+  , repo :: Name Repo+  , tag :: Text+  }++parseURL :: Text -> IO (Either Text URLParts)+parseURL url =+  runExceptT $ do+    tryAssert+      ("GitHub: " <> url <> " is not a GitHub URL.")+      ("https://github.com/" `T.isPrefixOf` url)+    let parts = T.splitOn "/" url+    owner <- N <$> tryAt ("GitHub: owner part missing from " <> url) parts 3+    repo <- N <$> tryAt ("GitHub: repo part missing from " <> url) parts 4+    tagPart <- tryAt ("GitHub: tag part missing from " <> url) parts 6+    tag <-+      tryJust+        ("GitHub: tag part missing .tar.gz suffix " <> url)+        (T.stripSuffix ".tar.gz" tagPart)+    return $ URLParts owner repo tag++compareUrl :: Text -> Text -> IO (Either Text Text)+compareUrl urlOld urlNew =+  runExceptT $ do+    oldParts <- ExceptT $ parseURL urlOld+    newParts <- ExceptT $ parseURL urlNew+    return $+      "https://github.com/" <> untagName (owner newParts) <> "/" <>+      untagName (repo newParts) <>+      "/compare/" <>+      tag oldParts <>+      "..." <>+      tag newParts++--deleteDoneBranches :: IO ()+--deleteDoneBranches = do+autoUpdateRefs :: Options -> IO (Either Text (Vector Text))+autoUpdateRefs o =+  references' (Just (OAuth (T.encodeUtf8 (githubToken o)))) "r-ryantm" "nixpkgs" &+  fmap+    (first (T.pack . show) >>>+     second (fmap gitReferenceRef >>> V.mapMaybe (T.stripPrefix prefix)))+  where+    prefix = "refs/heads/auto-update/"++openPRWithAutoUpdateRef :: Options -> Text -> IO (Either Text Bool)+openPRWithAutoUpdateRef o ref =+  (executeRequest (OAuth (T.encodeUtf8 (githubToken o))) $+   pullRequestsForR+     "nixos"+     "nixpkgs"+     (optionsHead ("r-ryantm:auto-update/" <> ref) <> stateOpen)+     FetchAll) &+  fmap (first (T.pack . show) >>> second (\v -> not (V.null v)))++refShouldBeDeleted :: Options -> Text -> IO Bool+refShouldBeDeleted o ref =+  not <$> either (const True) id <$> openPRWithAutoUpdateRef o ref++closedAutoUpdateRefs :: Options -> IO (Either Text (Vector Text))+closedAutoUpdateRefs o =+  runExceptT $ do+    aur :: Vector Text <- ExceptT $ autoUpdateRefs o+    ExceptT (Right <$> V.filterM (refShouldBeDeleted o) aur)
+ src/Git.hs view
@@ -0,0 +1,97 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module Git+  ( cleanAndResetToMaster+  , cleanAndResetToStaging+  , cleanup+  , fetchIfStale+  , fetch+  , push+  , checkoutAtMergeBase+  , autoUpdateBranchExists+  , commit+  , headHash+  , deleteBranch+  ) where++import Data.Semigroup ((<>))+import qualified Data.Text as T+import Data.Text (Text)+import Data.Time.Clock (UTCTime, addUTCTime, diffUTCTime, getCurrentTime)+import Shelly+import System.Directory (getModificationTime, getHomeDirectory)+import Utils (Options(..), UpdateEnv(..), branchName, canFail)++default (T.Text)++clean :: Sh ()+clean = cmd "git" "clean" "-fdx"++cleanAndResetTo :: Text -> Text -> Sh ()+cleanAndResetTo branch target = do+  cmd "git" "reset" "--hard"+  clean+  cmd "git" "checkout" "-B" branch target+  cmd "git" "reset" "--hard" target+  clean++cleanAndResetToMaster :: Sh ()+cleanAndResetToMaster = cleanAndResetTo "master" "upstream/master"++cleanAndResetToStaging :: Sh ()+cleanAndResetToStaging = cleanAndResetTo "staging" "upstream/staging"++cleanup :: Text -> Sh ()+cleanup branchName = do+  cleanAndResetToMaster+  canFail $ cmd "git" "branch" "-D" branchName++staleFetchHead :: IO Bool+staleFetchHead = do+  home <- getHomeDirectory+  let fetchHead = home <> "/.cache/nixpkgs/.git/FETCH_HEAD"+  oneHourAgo <- addUTCTime (fromInteger $ -60 * 60) <$> getCurrentTime+  fetchedLast <- getModificationTime fetchHead+  return (fetchedLast < oneHourAgo)++fetchIfStale :: Sh ()+fetchIfStale =+  whenM+    (liftIO staleFetchHead)+    fetch++fetch :: Sh ()+fetch = canFail $ cmd "git" "fetch" "-q" "--prune" "--multiple" "upstream" "origin"++push :: UpdateEnv -> Sh ()+push updateEnv =+  run_+    "git"+    (["push", "--force", "--set-upstream", "origin", branchName updateEnv] +++     ["--dry-run" | dryRun (options updateEnv)])++checkoutAtMergeBase :: Text -> Sh ()+checkoutAtMergeBase branchName = do+  base <-+    T.strip <$> cmd "git" "merge-base" "upstream/master" "upstream/staging"+  cmd "git" "checkout" "-B" branchName base++autoUpdateBranchExists :: Text -> Sh Bool+autoUpdateBranchExists packageName = do+  remotes <-+    map T.strip . T.lines <$> (silently $ cmd "git" "branch" "--remote")+  return $ ("origin/auto-update/" <> packageName) `elem` remotes++commit :: Text -> Sh ()+commit = cmd "git" "commit" "-am"++headHash :: Sh Text+headHash = cmd "git" "rev-parse" "HEAD"++deleteBranch :: Text -> Sh ()+deleteBranch branchName = do+  canFail $ do+    cmd "git" "branch" "-D" branchName+    cmd "git" "push" "origin" (":" <> branchName)
+ src/Main.hs view
@@ -0,0 +1,55 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++import Control.Applicative ((<**>), (<|>))+import Control.Exception+import Data.Maybe (isJust)+import Data.Semigroup ((<>))+import qualified Data.Text as T+import Data.Text (Text)+import qualified Data.Text.IO as T+import DeleteMerged (deleteDone)+import qualified Options.Applicative as Opt+import System.Directory (getHomeDirectory)+import System.Posix.Env (getEnv)+import Update (updateAll)+import Utils (Options(..))++default (T.Text)++data Mode+  = Update+  | DeleteDone++modeParser :: Opt.Parser Mode+modeParser =+  Opt.flag'+    Update+    (Opt.long "update" <> Opt.help "Update packages (default mode)") <|>+  Opt.flag'+    DeleteDone+    (Opt.long "delete-done" <>+     Opt.help "Delete branches from PRs that were merged or closed")++programInfo :: Opt.ParserInfo Mode+programInfo =+  Opt.info+    (modeParser <**> Opt.helper)+    (Opt.fullDesc <> Opt.progDesc "Update packages in nixpkgs repository" <>+     Opt.header "nixpkgs-update")++makeOptions :: IO Options+makeOptions = do+  dryRun <- isJust <$> getEnv "DRY_RUN"+  homeDir <- T.pack <$> getHomeDirectory+  githubToken <- T.strip <$> T.readFile "github_token.txt"+  return $ Options dryRun (homeDir <> "/.nixpkgs-update") githubToken++main :: IO ()+main = do+  mode <- Opt.execParser programInfo+  options <- makeOptions+  case mode of+    DeleteDone -> deleteDone options+    Update -> updateAll options
+ src/Nix.hs view
@@ -0,0 +1,190 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module Nix+  ( nixEvalE+  , compareVersions+  , lookupAttrPath+  , getDerivationFile+  , getMaintainers+  , getOldHash+  , getSrcUrl+  , getSrcUrls+  , getIsBroken+  , build+  , getDescription+  , cachix+  , numberOfFetchers+  , oldVersionOn+  ) where++import Control.Category ((>>>))+import Control.Error+import Control.Monad (void)+import Data.Bifunctor (second)+import Data.Function ((&))+import Data.Semigroup ((<>))+import Data.Text (Text)+import qualified Data.Text as T+import Prelude hiding (FilePath)+import Shelly (FilePath, Sh, cmd, fromText, run, setStdin, toTextIgnore)+import Utils (UpdateEnv(..), rewriteError, shE)++data Raw+  = Raw+  | NoRaw++rawOpt :: Raw -> [Text]+rawOpt Raw = ["--raw"]+rawOpt NoRaw = []++nixEvalE :: Raw -> Text -> Sh (Either Text Text)+nixEvalE raw expr =+  run "nix" (["eval", "-f", "."] <> rawOpt raw <> [expr]) &+  (fmap T.strip >>> shE >>> rewriteError ("nix eval failed for " <> expr))++-- Error if the "new version" is actually newer according to nix+compareVersions :: UpdateEnv -> Sh (Either Text ())+compareVersions updateEnv = do+  versionComparison <-+    nixEvalE+      NoRaw+      ("(builtins.compareVersions \"" <> newVersion updateEnv <> "\" \"" <>+       oldVersion updateEnv <>+       "\")")+  return $+    case versionComparison of+      Right "1" -> Right ()+      Right a ->+        Left $+        newVersion updateEnv <> " is not newer than " <> oldVersion updateEnv <>+        " according to Nix; versionComparison: " <>+        a+      Left a -> Left a+ -- This is extremely slow but gives us the best results we know of++lookupAttrPath :: UpdateEnv -> Sh (Either Text Text)+lookupAttrPath updateEnv =+  cmd+    "nix-env"+    "-qa"+    (packageName updateEnv <> "-" <> oldVersion updateEnv)+    "-f"+    "."+    "--attr-path"+    "--arg"+    "config"+    "{ allowBroken = true; allowUnfree = true; allowAliases = false; }" &+  (fmap (head . T.words . head . T.lines) >>>+   shE >>>+   rewriteError "nix-env -q failed to find package name with old version")++getDerivationFile :: UpdateEnv -> Text -> Sh (Either Text FilePath)+getDerivationFile updateEnv attrPath =+  cmd "env" "EDITOR=echo" "nix" "edit" attrPath "-f" "." &+  (fmap T.strip >>>+   fmap fromText >>> shE >>> rewriteError "Couldn't find derivation file.")++getHash :: Text -> Sh (Either Text Text)+getHash attrPath =+  nixEvalE Raw ("pkgs." <> attrPath <> ".src.drvAttrs.outputHash")++getOldHash :: Text -> Sh (Either Text Text)+getOldHash attrPath =+  getHash attrPath &+  rewriteError+    ("Could not find old output hash at " <> attrPath <>+     ".src.drvAttrs.outputHash.")++getSrcUrl :: Text -> Sh (Either Text Text)+getSrcUrl attrPath =+  nixEvalE+    Raw+    ("(let pkgs = import ./. {}; in builtins.elemAt pkgs." <> attrPath <>+     ".src.drvAttrs.urls 0)")++getMaintainers :: Text -> Sh (Either Text Text)+getMaintainers attrPath =+  nixEvalE+    Raw+    ("(let pkgs = import ./. {}; gh = m : m.github or \"\"; nonempty = s: s != \"\"; addAt = s: \"@\"+s; in builtins.concatStringsSep \" \" (map addAt (builtins.filter nonempty (map gh pkgs." <>+     attrPath <>+     ".meta.maintainers or []))))") &+  rewriteError ("Could not fetch maintainers for" <> attrPath)++readNixBool :: Either Text Text -> Either Text Bool+readNixBool (Right "true") = Right True+readNixBool (Right "false") = Right False+readNixBool (Right a) = Left ("Failed to convert expected nix boolean " <> a)+readNixBool (Left e) = Left e++getIsBroken :: Text -> Sh (Either Text Bool)+getIsBroken attrPath =+  nixEvalE+    NoRaw+    ("(let pkgs = import ./. {}; in pkgs." <> attrPath <>+     ".meta.broken or false)") &+  fmap readNixBool &+  rewriteError ("Could not get meta.broken for attrpath " <> attrPath)++getDescription :: Text -> Sh (Either Text Text)+getDescription attrPath =+  nixEvalE+    NoRaw+    ("(let pkgs = import ./. {}; in pkgs." <> attrPath <>+     ".meta.description or \"\")") &+  rewriteError ("Could not get meta.description for attrpath " <> attrPath)++getSrcAttr :: Text -> Text -> Sh (Either Text Text)+getSrcAttr attr attrPath =+  nixEvalE NoRaw ("pkgs." <> attrPath <> ".src." <> attr)++getSrcUrls :: Text -> Sh (Either Text Text)+getSrcUrls = getSrcAttr "urls"++build :: Text -> Sh (Either Text ())+build attrPath = do+  buildE <-+    shE $+    cmd+      "nix-build"+      "--option"+      "sandbox"+      "true"+      "--option"+      "restrict-eval"+      "true"+      "-A"+      attrPath+  case buildE of+    Right _ -> return $ Right ()+    Left _ -> do+      buildLogE <-+        cmd "nix" "log" "-f" "." attrPath &+        (shE >>>+         (fmap . fmap)+           (T.lines >>> reverse >>> take 30 >>> reverse >>> T.unlines))+      return $+        case buildLogE of+          Left t -> Left "nix log failed trying to get build logs"+          Right buildLog -> Left ("nix build failed.\n" <> buildLog)++cachix :: FilePath -> Sh ()+cachix resultPath = do+  setStdin (toTextIgnore resultPath)+  void $ shE $ cmd "cachix" "push" "r-ryantm"++numberOfFetchers :: Text -> Int+numberOfFetchers derivationContents =+  count "fetchurl {" + count "fetchgit {" + count "fetchFromGitHub {"+  where+    count x = T.count x derivationContents++oldVersionOn :: UpdateEnv -> Text -> Text -> Sh (Either Text ())+oldVersionOn updateEnv branchName contents =+  pure+    (assertErr+       ("Old version not present in " <> branchName <> " derivation file.")+       (oldVersion updateEnv `T.isInfixOf` contents))
+ src/Outpaths.hs view
@@ -0,0 +1,79 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module Outpaths+where++import           Data.Semigroup ((<>))+import           Data.Text (Text)+import qualified Data.Text as T+import           NeatInterpolation (text)+import           Shelly+import           Utils+default (Text)++outPathsExpr = [text|++(let+  lib = import ./lib;+  hydraJobs = import ./pkgs/top-level/release.nix+    # Compromise: accuracy vs. resources needed for evaluation.+    {+      supportedSystems = [+        "aarch64-linux"+        "i686-linux"+        "x86_64-linux"+        "x86_64-darwin"+      ];++      nixpkgsArgs = {+        config = {+          allowBroken = true;+          allowUnfree = true;+          allowInsecurePredicate = x: true;+          checkMeta = true;++          handleEvalIssue = reason: errormsg:+            if reason == "unknown-meta"+              then abort errormsg+              else true;++          inHydra = true;+        };+      };+    };+  recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };++  # hydraJobs leaves recurseForDerivations as empty attrmaps;+  # that would break nix-env and we also need to recurse everywhere.+  tweak = lib.mapAttrs+    (name: val:+      if name == "recurseForDerivations" then true+      else if lib.isAttrs val && val.type or null != "derivation"+              then recurseIntoAttrs (tweak val)+      else val+    );++  # Some of these contain explicit references to platform(s) we want to avoid;+  # some even (transitively) depend on ~/.nixpkgs/config.nix (!)+  blacklist = [+    "tarball" "metrics" "manual"+    "darwin-tested" "unstable" "stdenvBootstrapTools"+    "moduleSystem" "lib-tests" # these just confuse the output+  ];++in+  tweak (builtins.removeAttrs hydraJobs blacklist))+|]++--downloadOutPath :: Sh ()++outPath :: Sh Text+outPath = sub $ do+  Utils.setupNixpkgs+  cmd "curl" "-o" "outpaths.nix" "https://raw.githubusercontent.com/NixOS/ofborg/released/ofborg/src/outpaths.nix"+  setenv "GC_INITIAL_HEAP_SIZE" "4g"+  cmd "nix-env" "-f" "./outpaths.nix" "-qaP" "--no-name" "--out-path" "--arg" "checkMeta" "true"
+ src/Update.hs view
@@ -0,0 +1,329 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE LambdaCase #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module Update+  ( updateAll+  ) where++import qualified Blacklist+import qualified Check+import Clean (fixSrcUrl)+import Control.Category ((>>>))+import Control.Error+import Control.Exception (SomeException, throw, toException)+import Control.Monad (forM_)+import Data.Function ((&))+import Data.Maybe (fromMaybe)+import Data.Semigroup ((<>))+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.IO as T+import Data.Time.Clock (getCurrentTime)+import Data.Time.Format (defaultTimeLocale, formatTime, iso8601DateFormat)+import qualified File+import qualified GH+import qualified Git+import NeatInterpolation (text)+import qualified Nix+import Prelude hiding (FilePath)+import Shelly+import Utils+  ( ExitCode(..)+  , Options(..)+  , UpdateEnv(..)+  , Version+  , branchName+  , canFail+  , checkAttrPathVersion+  , eitherToError+  , orElse+  , ourShell+  , parseUpdates+  , rewriteError+  , setupNixpkgs+  , shE+  , tRead+  )++default (T.Text)++errorExit' :: (Text -> Sh ()) -> Text -> Text -> Sh a+errorExit' log branchName message = do+  Git.cleanup branchName+  log message+  throw (ExitCode 1)++log' logFile msg+    -- TODO: switch to Data.Time.Format.ISO8601 once time-1.9.0 is available+ = do+  runDate <-+    T.pack . formatTime defaultTimeLocale (iso8601DateFormat (Just "%H:%M:%S")) <$>+    liftIO getCurrentTime+  appendfile logFile (runDate <> " " <> msg <> "\n")++updateAll :: Options -> IO ()+updateAll options =+  ourShell options $ do+    let logFile = fromText (workingDir options) </> "ups.log"+    mkdir_p (fromText (workingDir options))+    touchfile logFile+    updates <- readfile "packages-to-update.txt"+    let log = log' logFile+    appendfile logFile "\n\n"+    log "New run of ups.sh"+    updateLoop options log (parseUpdates updates)++updateLoop ::+     Options+  -> (Text -> Sh ())+  -> [Either Text (Text, Version, Version)]+  -> Sh ()+updateLoop _ log [] = log "ups.sh finished"+updateLoop options log (Left e:moreUpdates) = do+  log e+  updateLoop options log moreUpdates+updateLoop options log (Right (package, oldVersion, newVersion):moreUpdates) = do+  log (package <> " " <> oldVersion <> " -> " <> newVersion)+  updated <-+    catch_sh+      (updatePackage log (UpdateEnv package oldVersion newVersion options))+      (\case+         ExitCode 0 -> return True+         ExitCode _ -> return False)+  if updated+    then do+      log "SUCCESS"+      updateLoop options log moreUpdates+    else do+      log "FAIL"+      if ".0" `T.isSuffixOf` newVersion+        then let Just newNewVersion = ".0" `T.stripSuffix` newVersion+              in updateLoop+                   options+                   log+                   (Right (package, oldVersion, newNewVersion) : moreUpdates)+        else updateLoop options log moreUpdates++updatePackage :: (Text -> Sh ()) -> UpdateEnv -> Sh Bool+updatePackage log updateEnv = do+  let errorExit = errorExit' log (branchName updateEnv)+  eitherToError errorExit (pure (Blacklist.packageName (packageName updateEnv)))+  setupNixpkgs+  -- Check whether requested version is newer than the current one+  eitherToError errorExit (Nix.compareVersions updateEnv)+  -- Check whether package name is on blacklist+  Git.fetchIfStale+  whenM+    (Git.autoUpdateBranchExists (packageName updateEnv))+    (errorExit "Update branch already on origin.")+  Git.cleanAndResetToMaster+  attrPath <- eitherToError errorExit (Nix.lookupAttrPath updateEnv)+  srcUrls <- eitherToError errorExit (Nix.getSrcUrls attrPath)+  eitherToError errorExit (pure (Blacklist.srcUrl srcUrls))+  eitherToError errorExit (pure (Blacklist.attrPath attrPath))+  derivationFile <-+    eitherToError errorExit (Nix.getDerivationFile updateEnv attrPath)+  flip+    catches_sh+    [ ShellyHandler (\(ex :: ExitCode) -> throw ex)+    , ShellyHandler (\(ex :: SomeException) -> errorExit (T.pack (show ex)))+    ] $ do+    unless (checkAttrPathVersion attrPath (newVersion updateEnv)) $+      errorExit+        ("Version in attr path " <> attrPath <> " not compatible with " <>+         newVersion updateEnv)+    -- Make sure it hasn't been updated on master+    masterDerivationContents <- readfile derivationFile+    eitherToError+      errorExit+      (Nix.oldVersionOn updateEnv "master" masterDerivationContents)+    -- Make sure it hasn't been updated on staging+    Git.cleanAndResetToStaging+    stagingDerivationContents <- readfile derivationFile+    eitherToError+      errorExit+      (Nix.oldVersionOn updateEnv "staging" stagingDerivationContents)+    Git.checkoutAtMergeBase (branchName updateEnv)+    derivationContents <- readfile derivationFile+    unless (Nix.numberOfFetchers derivationContents <= 1) $+      errorExit $ "More than one fetcher in " <> toTextIgnore derivationFile+    eitherToError errorExit (pure (Blacklist.content derivationContents))+    oldHash <- eitherToError errorExit (Nix.getOldHash attrPath)+    oldSrcUrl <- eitherToError errorExit (Nix.getSrcUrl attrPath)+    File.replace (oldVersion updateEnv) (newVersion updateEnv) derivationFile+    newSrcUrl <- eitherToError errorExit (Nix.getSrcUrl attrPath)+    when (oldSrcUrl == newSrcUrl) $ errorExit "Source url did not change."+    newHash <-+      canFail (T.strip <$> cmd "nix-prefetch-url" "-A" (attrPath <> ".src")) `orElse`+      fixSrcUrl updateEnv derivationFile attrPath oldSrcUrl `orElse`+      errorExit "Could not prefetch new version URL."+    when (oldHash == newHash) $ errorExit "Hashes equal; no update necessary"+    File.replace oldHash newHash derivationFile+    eitherToError errorExit (Nix.build attrPath)+    result <-+      fromText <$>+      (T.strip <$>+       (cmd "readlink" "./result" `orElse` cmd "readlink" "./result-bin")) `orElse`+      errorExit "Could not find result link."+    publishPackage log updateEnv oldSrcUrl newSrcUrl attrPath result++publishPackage ::+     (Text -> Sh ()) -> UpdateEnv -> Text -> Text -> Text -> FilePath -> Sh Bool+publishPackage log updateEnv oldSrcUrl newSrcUrl attrPath result = do+  let errorExit = errorExit' log (branchName updateEnv)+  log ("cachix " <> (T.pack . show) result)+  Nix.cachix result+  resultCheckReport <-+    case Blacklist.checkResult (packageName updateEnv) of+      Right () -> sub (Check.result updateEnv result)+      Left msg -> pure msg+  d <- eitherToError errorExit (Nix.getDescription attrPath)+  let metaDescription =+        "\n\nmeta.description for " <> attrPath <> " is: '" <> d <> "'."+  releaseUrlResult <- liftIO $ GH.releaseUrl newSrcUrl+  releaseUrlMessage <-+    case releaseUrlResult of+      Left e -> do+        log e+        return ""+      Right msg -> return ("\n[Release on GitHub](" <> msg <> ")\n\n")+  compareUrlResult <- liftIO $ GH.compareUrl oldSrcUrl newSrcUrl+  compareUrlMessage <-+    case compareUrlResult of+      Left e -> do+        log e+        return "\n"+      Right msg -> return ("\n[Compare changes on GitHub](" <> msg <> ")\n\n")+  maintainers <- eitherToError errorExit (Nix.getMaintainers attrPath)+  let maintainersCc =+        if not (T.null maintainers)+          then "\n\ncc " <> maintainers <> " for testing."+          else ""+  let commitMsg = commitMessage updateEnv attrPath+  Git.commit commitMsg+  commitHash <- Git.headHash+  -- Try to push it three times+  Git.push updateEnv `orElse` Git.push updateEnv `orElse` Git.push updateEnv+  isBroken <- eitherToError errorExit (Nix.getIsBroken attrPath)+  untilOfBorgFree+  GH.pr+    (prMessage+       updateEnv+       isBroken+       metaDescription+       releaseUrlMessage+       compareUrlMessage+       resultCheckReport+       commitHash+       attrPath+       maintainersCc+       result)+  Git.cleanAndResetToMaster+  return True++repologyUrl :: UpdateEnv -> Text+repologyUrl updateEnv = [text|https://repology.org/metapackage/$pname/versions|]+  where+    pname = (packageName >>> T.toLower) updateEnv++commitMessage :: UpdateEnv -> Text -> Text+commitMessage updateEnv attrPath =+  let oV = oldVersion updateEnv+      nV = newVersion updateEnv+      repologyLink = repologyUrl updateEnv+   in [text|+       $attrPath: $oV -> $nV++       Semi-automatic update generated by+       https://github.com/ryantm/nixpkgs-update tools. This update was made+       based on information from+       $repologyLink+     |]++brokenWarning :: Bool -> Text+brokenWarning False = ""+brokenWarning True =+  "- WARNING: Package has meta.broken=true; Please manually test this package update and remove the broken attribute."++prMessage ::+     UpdateEnv+  -> Bool+  -> Text+  -> Text+  -> Text+  -> Text+  -> Text+  -> Text+  -> Text+  -> FilePath+  -> Text+prMessage updateEnv isBroken metaDescription releaseUrlMessage compareUrlMessage resultCheckReport commitHash attrPath maintainersCc resultPath =+  let brokenMsg = brokenWarning isBroken+      oV = oldVersion updateEnv+      nV = newVersion updateEnv+      repologyLink = repologyUrl updateEnv+      result = toTextIgnore resultPath+   in [text|+       $attrPath: $oV -> $nV++       Semi-automatic update generated by https://github.com/ryantm/nixpkgs-update tools. This update was made based on information from $repologyLink.+       $brokenMsg+       $metaDescription+       $releaseUrlMessage+       $compareUrlMessage+       <details>+       <summary>+       Checks done (click to expand)+       </summary>++       - built on NixOS+       $resultCheckReport++       </details>+       <details>+       <summary>+       Instructions to test this update (click to expand)+       </summary>++       Either download from Cachix:+       ```+       nix-store -r $result \+         --option binary-caches 'https://cache.nixos.org/ https://r-ryantm.cachix.org/' \+         --option trusted-public-keys '+         r-ryantm.cachix.org-1:gkUbLkouDAyvBdpBX0JOdIiD2/DP1ldF3Z3Y6Gqcc4c=+         cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY=+         '+       ```+       (r-ryantm's Cachix cache is only trusted for this store-path realization.)++       Or, build yourself:+       ```+       nix-build -A $attrPath https://github.com/r-ryantm/nixpkgs/archive/$commitHash.tar.gz+       ```++       After you've downloaded or built it, look at the files and if there are any, run the binaries:+       ```+       ls -la $result+       ls -la $result/bin+       ```+++       </details>+       <br/>+       $maintainersCc+    |]++untilOfBorgFree :: Sh ()+untilOfBorgFree = do+  waiting :: Int <-+    tRead <$>+    canFail+      (cmd "curl" "-s" "https://events.nix.ci/stats.php" -|-+       cmd "jq" ".evaluator.messages.waiting")+  when (waiting > 2) $ do+    sleep 60+    untilOfBorgFree
+ src/Utils.hs view
@@ -0,0 +1,182 @@+{-# LANGUAGE ExtendedDefaultRules #-}+{-# LANGUAGE OverloadedStrings #-}+{-# OPTIONS_GHC -fno-warn-type-defaults #-}++module Utils+  ( Options(..)+  , Version+  , UpdateEnv(..)+  , canFail+  , checkAttrPathVersion+  , orElse+  , setupNixpkgs+  , tRead+  , parseUpdates+  , succeded+  , ExitCode(..)+  , shE+  , rewriteError+  , eitherToError+  , branchName+  , ourShell+  ) where++import Control.Exception (Exception)+import Data.Bifunctor (first)+import Data.Semigroup ((<>))+import Data.Text (Text)+import qualified Data.Text as T+import Prelude hiding (FilePath)+import Shelly++default (T.Text)++type Version = Text++data Options = Options+  { dryRun :: Bool+  , workingDir :: Text+  , githubToken :: Text+  }++data UpdateEnv = UpdateEnv+  { packageName :: Text+  , oldVersion :: Version+  , newVersion :: Version+  , options :: Options+  }++setupNixpkgs :: Sh ()+setupNixpkgs = do+  home <- get_env_text "HOME"+  let nixpkgsPath = home </> ".cache" </> "nixpkgs"+  unlessM (test_e nixpkgsPath) $ do+    cmd "hub" "clone" "nixpkgs" nixpkgsPath -- requires that user has forked nixpkgs+    cd nixpkgsPath+    cmd "git" "remote" "add" "upstream" "https://github.com/NixOS/nixpkgs"+    cmd "git" "fetch" "upstream"+  cd nixpkgsPath+  setenv "NIX_PATH" ("nixpkgs=" <> toTextIgnore nixpkgsPath)++-- | Set environment variables needed by various programs+setUpEnvironment :: Options -> Sh ()+setUpEnvironment options = do+  setenv "PAGER" ""+  setenv "GITHUB_TOKEN" (githubToken options)++ourShell :: Options -> Sh a -> IO a+ourShell o s = shelly $ verbosely $ do+  setUpEnvironment o+  s++shE :: Sh a -> Sh (Either Text a)+shE s = do+  r <- canFail s+  status <- lastExitCode+  case status of+    0 -> return $ Right r+    c -> return $ Left ("Exit code: " <> T.pack (show c))++rewriteError :: Text -> Sh (Either Text a) -> Sh (Either Text a)+rewriteError t = fmap (first (const t))++eitherToError :: (Text -> Sh a) -> Sh (Either Text a) -> Sh a+eitherToError errorExit s = do+  e <- s+  either errorExit return e++canFail :: Sh a -> Sh a+canFail = errExit False++succeded :: Sh a -> Sh Bool+succeded s = do+  canFail s+  status <- lastExitCode+  return (status == 0)++orElse :: Sh a -> Sh a -> Sh a+orElse a b = do+  v <- canFail a+  status <- lastExitCode+  if status == 0+    then return v+    else b++infixl 3 `orElse`++branchName :: UpdateEnv -> Text+branchName ue = "auto-update/" <> packageName ue++parseUpdates :: Text -> [Either Text (Text, Version, Version)]+parseUpdates = map (toTriple . T.words) . T.lines+  where+    toTriple :: [Text] -> Either Text (Text, Version, Version)+    toTriple [package, oldVersion, newVersion] =+      Right (package, oldVersion, newVersion)+    toTriple line = Left $ "Unable to parse update: " <> T.unwords line++tRead :: Read a => Text -> a+tRead = read . T.unpack++notElemOf :: (Eq a, Foldable t) => t a -> a -> Bool+notElemOf options = not . flip elem options++-- | Similar to @breakOn@, but will not keep the pattern at the beginning of the suffix.+--+-- Examples:+--+-- > breakOn "::" "a::b::c"+-- ("a","b::c")+clearBreakOn :: Text -> Text -> (Text, Text)+clearBreakOn boundary string =+  let (prefix, suffix) = T.breakOn boundary string+   in if T.null suffix+        then (prefix, suffix)+        else (prefix, T.drop (T.length boundary) suffix)++-- | Check if attribute path is not pinned to a certain version.+-- If a derivation is expected to stay at certain version branch,+-- it will usually have the branch as a part of the attribute path.+--+-- Examples:+--+-- >>> checkAttrPathVersion "libgit2_0_25" "0.25.3"+-- True+--+-- >>> checkAttrPathVersion "owncloud90" "9.0.3"+-- True+--+-- >>> checkAttrPathVersion "owncloud-client" "2.4.1"+-- True+--+-- >>> checkAttrPathVersion "owncloud90" "9.1.3"+-- False+checkAttrPathVersion :: Text -> Version -> Bool+checkAttrPathVersion attrPath newVersion =+  if "_" `T.isInfixOf` attrPath+    then let attrVersionPart =+               let (name, version) = clearBreakOn "_" attrPath+                in if T.any (notElemOf ('_' : ['0' .. '9'])) version+                     then Nothing+                     else Just version+        -- Check assuming version part has underscore separators+             attrVersionPeriods = T.replace "_" "." <$> attrVersionPart+        -- If we don't find version numbers in the attr path, exit success.+          in maybe True (`T.isPrefixOf` newVersion) attrVersionPeriods+         -- other path+    else let attrVersionPart =+               let version = T.dropWhile (notElemOf ['0' .. '9']) attrPath+                in if T.any (notElemOf ['0' .. '9']) version+                     then Nothing+                     else Just version+        -- Check assuming version part is the prefix of the version with dots+        -- removed. For example, 91 => "9.1"+             noPeriodNewVersion = T.replace "." "" newVersion+            -- If we don't find version numbers in the attr path, exit success.+          in maybe True (`T.isPrefixOf` noPeriodNewVersion) attrVersionPart++newtype ExitCode =+  ExitCode Int+  deriving (Show)++instance Exception ExitCode
+ test/doctests.hs view
@@ -0,0 +1,3 @@+import Test.DocTest++main = doctest ["-isrc", "-XOverloadedStrings", "src/Utils.hs"]