nvfetcher 0.6.2.0 → 0.7.0.0

raw patch · 21 files changed

+273/−159 lines, 21 filesdep −validation-selectivedep ~data-defaultdep ~extradep ~regex-tdfaPVP ok

version bump matches the API change (PVP)

Dependencies removed: validation-selective

Dependency ranges changed: data-default, extra, regex-tdfa, toml-reader

API changes (from Hackage documentation)

Files

• CHANGELOG.md +11/−0
• README.md +16/−7
• app/Config.hs +3/−0
• app/Config/Common.hs +4/−4
• app/Config/PackageFetcher.hs +12/−7
• app/Config/VersionSource.hs +4/−4
• nvfetcher.cabal +7/−10
• src/NvFetcher.hs +37/−22
• src/NvFetcher/Config.hs +7/−2
• src/NvFetcher/Core.hs +65/−61
• src/NvFetcher/FetchRustGitDeps.hs +2/−1
• src/NvFetcher/NixExpr.hs +9/−5
• src/NvFetcher/NixFetcher.hs +31/−15
• src/NvFetcher/Nvchecker.hs +2/−2
• src/NvFetcher/Options.hs +17/−3
• src/NvFetcher/PackageSet.hs +12/−5
• src/NvFetcher/Types.hs +12/−4
• src/NvFetcher/Types/ShakeExtras.hs +6/−1
• test/FetchRustGitDepsSpec.hs +2/−1
• test/NixExprSpec.hs +1/−0
• test/PrefetchSpec.hs +13/−5

@@ -1,5 +1,16 @@ # Revision history for nvfetcher +## 0.7.0.0++Now nvfetcher removes all files *except* `generated.json` and `generated.nix` in `_sources` before each run. If you do want to keep those files, you can use the new CLI option `--keep-old`. In addition, a new target `purge` is introduced for resetting the state of nvfetcher by deleting the shake database saved in XDG directory. ++* Quote package name passed to nvchecker+* Add `url.name` option to specify the file name in prefetch+* Clean build dir before build+* Support keep going on fetch failure+* Add `--commit-summary`+* Support sparseCheckout+ ## 0.6.2.0  * Rework config parsing with toml-reader

@@ -103,10 +103,11 @@ To run nvfetcher as a CLI program, you'll need to provide package sources defined in TOML.  ```-Usage: nvfetcher [--version] [--help] [-o|--build-dir DIR] [--commit-changes]-                 [-l|--changelog FILE] [-j NUM] [-r|--retry NUM] [-t|--timing]-                 [-v|--verbose] [-f|--filter REGEX] [-k|--keyfile FILE] [TARGET]-                 [-c|--config FILE]+Usage: nvfetcher [--version] [--help] [-o|--build-dir DIR] [--commit-changes] +                 [-l|--changelog FILE] [-j NUM] [-r|--retry NUM] [-t|--timing] +                 [-v|--verbose] [-f|--filter REGEX] [-k|--keyfile FILE] +                 [--keep-old] [--keep-going] [TARGET] [-c|--config FILE]+   generate nix sources expr for the latest version of packages  Available options:@@ -125,7 +126,11 @@   -v,--verbose             Verbose mode   -f,--filter REGEX        Regex to filter packages to be updated   -k,--keyfile FILE        Nvchecker keyfile-  TARGET                   Two targets are available: 1.build 2.clean+  --keep-old               Don't remove old files other than generated json and+                           nix before build+  --keep-going             Don't stop if some packages failed to be fetched+  TARGET                   Three targets are available: 1.build 2.clean (remove+                           all generated files) 3.purge (remove shake db)                            (default: build)   -c,--config FILE         Path to nvfetcher TOML config                            (default: "nvfetcher.toml")@@ -188,8 +193,12 @@ - `fetch.tarball = tarball_url` - `fetch.docker = owner/name` -Optional `nix-prefetch fetchgit` config, which make sense only when the fetcher equals to `fetch.github` or `fetch.git`.-They can exist simultaneously.+Optional config for `nix-prefetch-url`, applies when the fetcher equals to `fetch.url`.+`$ver` is available in string, just like for the fetch config.++- `url.name = file_name`++Optional config for `nix-prefetch-git`, applies when the fetcher equals to `fetch.github` or `fetch.git`.  - `git.deepClone` - `git.fetchSubmodules`

@@ -78,6 +78,9 @@             not $ null gk,             "fetch.git" `notElem` keys && "fetch.github" `notElem` keys         ]+          <>+          -- url+          [KeyUnexpected pkg uk | let uk = filter ("url.name" ==) keys, not $ null uk, "fetch.url" `notElem` keys]           -- docker           <> [ KeyUnexpected pkg dk                | let dk = filter (T.isPrefixOf "docker.") keys,

@@ -7,15 +7,15 @@ import qualified Data.Text as T import TOML -githubDecoder :: Decoder (Text, Text)-githubDecoder = makeDecoder $ \case+gitHubNameDecoder :: Decoder (Text, Text)+gitHubNameDecoder = makeDecoder $ \case   v@(String s) -> case T.split (== '/') s of     [owner, repo] -> pure (owner, repo)     _ -> invalidValue "unexpected github format: it should be in the format of [owner]/[repo]" v   v -> typeMismatch v -vscodeExtensionDecoder :: Decoder (Text, Text)-vscodeExtensionDecoder = makeDecoder $ \case+vscodeExtensionNameDecoder :: Decoder (Text, Text)+vscodeExtensionNameDecoder = makeDecoder $ \case   -- assume that we can't have '.' in extension's name   v@(String s) -> case T.split (== '.') s of     [publisher, extName] -> pure (publisher, extName)

@@ -50,7 +50,8 @@ data GitOptions = GitOptions   { goDeepClone :: Maybe Bool,     goFetchSubmodules :: Maybe Bool,-    goLeaveDotGit :: Maybe Bool+    goLeaveDotGit :: Maybe Bool,+    goSparseCheckout :: Maybe [Text]   }   deriving (Eq, Generic) @@ -60,6 +61,7 @@     <$> getFieldsOpt ["git", "deepClone"]     <*> getFieldsOpt ["git", "fetchSubmodules"]     <*> getFieldsOpt ["git", "leaveDotGit"]+    <*> getFieldsOpt ["git", "sparseCheckout"]  _GitOptions :: Traversal' (NixFetcher f) GitOptions _GitOptions f x@FetchGit {..} =@@ -68,23 +70,25 @@         & deepClone .~ fromMaybe False goDeepClone         & fetchSubmodules .~ fromMaybe False goFetchSubmodules         & leaveDotGit .~ fromMaybe False goLeaveDotGit+        & sparseCheckout .~ fromMaybe [] goSparseCheckout   )-    <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit))+    <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit) (Just _sparseCheckout)) _GitOptions f x@FetchGitHub {..} =   ( \GitOptions {..} ->       x         & deepClone .~ fromMaybe False goDeepClone         & fetchSubmodules .~ fromMaybe False goFetchSubmodules         & leaveDotGit .~ fromMaybe False goLeaveDotGit+        & sparseCheckout .~ fromMaybe [] goSparseCheckout   )-    <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit))+    <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit) (Just _sparseCheckout)) _GitOptions _ x = pure x  --------------------------------------------------------------------------------  gitHubDecoder :: Decoder PackageFetcher gitHubDecoder = do-  (owner, repo) <- getFieldsWith githubDecoder ["fetch", "github"]+  (owner, repo) <- getFieldsWith gitHubNameDecoder ["fetch", "github"]   gitOptions <- gitOptionsDecoder   pure $ \v -> gitHubFetcher (owner, repo) v & _GitOptions .~ gitOptions @@ -104,19 +108,20 @@ --------------------------------------------------------------------------------  openVsxDecoder :: Decoder PackageFetcher-openVsxDecoder = openVsxFetcher <$> getFieldsWith vscodeExtensionDecoder ["fetch", "openvsx"]+openVsxDecoder = openVsxFetcher <$> getFieldsWith vscodeExtensionNameDecoder ["fetch", "openvsx"]  --------------------------------------------------------------------------------  vscodeMarketplaceDecoder :: Decoder PackageFetcher-vscodeMarketplaceDecoder = vscodeMarketplaceFetcher <$> getFieldsWith vscodeExtensionDecoder ["fetch", "vsmarketplace"]+vscodeMarketplaceDecoder = vscodeMarketplaceFetcher <$> getFieldsWith vscodeExtensionNameDecoder ["fetch", "vsmarketplace"]  --------------------------------------------------------------------------------  urlDecoder :: Decoder PackageFetcher urlDecoder = do   url <- getFields ["fetch", "url"]-  pure $ \(coerce -> v) -> urlFetcher $ T.replace "$ver" v url+  name <- getFieldsOpt ["url", "name"]+  pure $ \(coerce -> v) -> urlFetcher' (T.replace "$ver" v url) (fmap (T.replace "$ver" v) name)  -------------------------------------------------------------------------------- 

@@ -77,13 +77,13 @@ --------------------------------------------------------------------------------  gitHubReleaseDecoder :: Decoder VersionSource-gitHubReleaseDecoder = uncurry GitHubRelease <$> getFieldWith githubDecoder "github"+gitHubReleaseDecoder = uncurry GitHubRelease <$> getFieldWith gitHubNameDecoder "github"  --------------------------------------------------------------------------------  gitHubTagDecoder :: Decoder VersionSource gitHubTagDecoder = do-  (_owner, _repo) <- getFieldWith githubDecoder "github_tag"+  (_owner, _repo) <- getFieldWith gitHubNameDecoder "github_tag"   _listOptions <- listOptionsDecoder   pure GitHubTag {..} @@ -142,12 +142,12 @@ --------------------------------------------------------------------------------  openVsxDecoder :: Decoder VersionSource-openVsxDecoder = uncurry OpenVsx <$> getFieldWith vscodeExtensionDecoder "openvsx"+openVsxDecoder = uncurry OpenVsx <$> getFieldWith vscodeExtensionNameDecoder "openvsx"  --------------------------------------------------------------------------------  vscodeMarketplaceDecoder :: Decoder VersionSource-vscodeMarketplaceDecoder = uncurry VscodeMarketplace <$> getFieldWith vscodeExtensionDecoder "vsmarketplace"+vscodeMarketplaceDecoder = uncurry VscodeMarketplace <$> getFieldWith vscodeExtensionNameDecoder "vsmarketplace"  -------------------------------------------------------------------------------- 

@@ -1,6 +1,6 @@ cabal-version:   2.4 name:            nvfetcher-version:         0.6.2.0+version:         0.7.0.0 synopsis:   Generate nix sources expr for the latest version of packages @@ -26,23 +26,23 @@  common common-options   build-depends:-    , aeson                 >=1.5.6    && <2.3+    , aeson                 >=1.5.6  && <2.3     , aeson-pretty-    , base                  >=4.8      && <5+    , base                  >=4.8    && <5     , binary     , binary-instances      ^>=1.0     , bytestring     , containers     , data-default          ^>=0.7.1-    , extra                 ^>=1.7.9-    , free                  ^>=5.1.5+    , extra                 ^>=1.7+    , free                  >=5.1    && <5.3     , microlens     , microlens-th     , neat-interpolation    ^>=0.5.1     , optparse-simple       ^>=0.1.1     , parsec     , prettyprinter-    , regex-tdfa            ^>=1.3.1.1+    , regex-tdfa            ^>=1.3.1     , shake                 ^>=0.19     , text     , toml-reader           ^>=0.2@@ -90,10 +90,7 @@     Config.PackageFetcher     Config.VersionSource -  build-depends:-    , nvfetcher-    , validation-selective-+  build-depends:  nvfetcher   ghc-options:    -threaded -rtsopts -with-rtsopts=-N  flag build-example

@@ -51,15 +51,15 @@   ) where -import Control.Monad.Extra (forM_, when, whenJust)+import Control.Monad.Extra (forM_, unless, when, whenJust, whenM) import qualified Data.Aeson as A import qualified Data.Aeson.Encode.Pretty as A import qualified Data.Aeson.Types as A import qualified Data.ByteString.Lazy.Char8 as LBS import Data.Default-import Data.List ((\\))+import Data.List (partition, (\\)) import qualified Data.Map.Strict as Map-import Data.Maybe (catMaybes, fromMaybe)+import Data.Maybe (catMaybes, fromJust, fromMaybe, isJust) import Data.Text (Text) import qualified Data.Text as T import Development.Shake@@ -103,7 +103,7 @@       { buildDir = optBuildDir,         actionAfterBuild = do           whenJust optLogPath logChangesToFile-          when optCommit commitChanges+          when optCommit (commitChanges (fromMaybe "Update" optCommitSummary))           actionAfterBuild config,         shakeConfig =           (shakeConfig config)@@ -113,7 +113,9 @@             },         filterRegex = optPkgNameFilter,         retry = optRetry,-        keyfile = aKeyfile+        keyfile = aKeyfile,+        keepOldFiles = optKeepOldFiles,+        keepGoing = optKeepGoing       }  logChangesToFile :: FilePath -> Action ()@@ -121,12 +123,12 @@   changes <- getVersionChanges   writeFile' fp $ unlines $ show <$> changes -commitChanges :: Action ()-commitChanges = do+commitChanges :: String -> Action ()+commitChanges commitSummary = do   changes <- getVersionChanges   let commitMsg = case changes of         [x] -> Just $ show x-        xs@(_ : _) -> Just $ "Update\n" <> unlines (show <$> xs)+        xs@(_ : _) -> Just $ commitSummary <> "\n" <> unlines (show <$> xs)         [] -> Nothing   whenJust commitMsg $ \msg -> do     putInfo "Commiting changes"@@ -155,8 +157,8 @@   pkgs <- Map.map pinIfUnmatch <$> runPackageSet packageSet   lastVersions <- parseLastVersions $ buildDir </> generatedJsonFileName   shakeDir <- getDataDir-  -- Set shakeFiles-  let shakeOptions1 = shakeConfig {shakeFiles = shakeDir}+  -- Set shakeFiles and shakeVersion+  let shakeOptions1 = shakeConfig {shakeFiles = shakeDir, shakeVersion = "2"}   -- shakeConfig in Config will be shakeOptions1 (not including shake extra)   shakeExtras <- initShakeExtras (config {shakeConfig = shakeOptions1}) pkgs $ fromMaybe mempty lastVersions   -- Set shakeExtra@@ -167,14 +169,14 @@     -- Don't touch already pinned packages     pinIfUnmatch x@Package {..}       | Just regex <- filterRegex =-        x-          { _ppinned = case _ppinned of-              PermanentStale -> PermanentStale-              _ ->-                if _pname =~ regex-                  then NoStale-                  else TemporaryStale-          }+          x+            { _ppinned = case _ppinned of+                PermanentStale -> PermanentStale+                _ ->+                  if _pname =~ regex+                    then NoStale+                    else TemporaryStale+            }       | otherwise = x  --------------------------------------------------------------------------------@@ -185,12 +187,25 @@     getBuildDir >>= flip removeFilesAfter ["//*"]     actionAfterClean +  "purge" ~> do+    shakeDir <- shakeFiles <$> getShakeOptions+    removeFilesAfter shakeDir ["//*"]+   "build" ~> do+    -- remove all files in build dir except generated nix and json+    -- since core rule has always rerun, any file not generated in this run will be removed+    unless keepOldFiles $+      whenM (liftIO $ D.doesDirectoryExist buildDir) $ do+        oldFiles <- (\\ [generatedJsonFileName, generatedNixFileName]) <$> liftIO (D.listDirectory buildDir)+        putVerbose $ "Removing old files: " <> show oldFiles+        liftIO $ removeFiles buildDir oldFiles     allKeys <- getAllPackageKeys-    results <- parallel $ runPackage <$> allKeys+    results <- fmap (zip allKeys) $ parallel $ runPackage <$> allKeys+    let (fmap (fromJust . snd) -> successResults, fmap fst -> failureKeys) = partition (isJust . snd) results     -- Record removed packages to version changes+    -- Failure keys are also considered as removed in this run     getAllOnDiskVersions-      >>= \oldPkgs -> forM_ (Map.keys oldPkgs \\ allKeys) $+      >>= \oldPkgs -> forM_ (Map.keys oldPkgs \\ (allKeys \\ failureKeys)) $         \pkg -> recordVersionChange (coerce pkg) (oldPkgs Map.!? pkg) "∅"     getVersionChanges >>= \changes ->       if null changes@@ -202,9 +217,9 @@     let generatedNixPath = buildDir </> generatedNixFileName         generatedJSONPath = buildDir </> generatedJsonFileName     putVerbose $ "Generating " <> generatedNixPath-    writeFileChanged generatedNixPath $ T.unpack $ srouces (T.unlines $ toNixExpr <$> results) <> "\n"+    writeFileChanged generatedNixPath $ T.unpack $ srouces (T.unlines $ toNixExpr <$> successResults) <> "\n"     putVerbose $ "Generating " <> generatedJSONPath-    writeFileChanged generatedJSONPath $ LBS.unpack $ A.encodePretty $ A.object [aesonKey (_prname r) A..= r | r <- results]+    writeFileChanged generatedJSONPath $ LBS.unpack $ A.encodePretty $ A.object [aesonKey (_prname r) A..= r | r <- successResults]     actionAfterBuild    customRules

@@ -18,8 +18,11 @@     retry :: Int,     filterRegex :: Maybe String,     cacheNvchecker :: Bool,+    keepOldFiles :: Bool,     -- | Absolute path-    keyfile :: Maybe FilePath+    keyfile :: Maybe FilePath,+    -- | When set to 'True', nvfetcher will keep going even if some packages failed to /fetch/+    keepGoing :: Bool   }  instance Default Config where@@ -37,5 +40,7 @@         retry = 3,         filterRegex = Nothing,         cacheNvchecker = True,-        keyfile = Nothing+        keepOldFiles = False,+        keyfile = Nothing,+        keepGoing = False       }

@@ -43,7 +43,7 @@     -- since the package definition is not tracked at all     alwaysRerun     lookupPackage pkg >>= \case-      Nothing -> fail $ "Unkown package key: " <> show pkg+      Nothing -> fail $ "Unknown package key: " <> show pkg       Just         Package           { _pversion = CheckVersion versionSource options,@@ -52,71 +52,75 @@           } -> do           _prversion@(NvcheckerResult version _mOldV _isStale) <- checkVersion versionSource options pkg           _prfetched <- prefetch (_pfetcher version) _pforcefetch-          buildDir <- getBuildDir-          -- extract src-          _prextract <--            case _pextract of-              Just (PackageExtractSrc extract) -> do-                result <- HMap.toList <$> extractSrcs _prfetched extract-                Just . HMap.fromList-                  <$> sequence-                    [ do-                        -- write extracted files to build dir-                        -- and read them in nix using 'builtins.readFile'-                        writeFile' (buildDir </> path) (T.unpack v)-                        pure (k, T.pack path)-                      | (k, v) <- result,-                        let path =-                              "./"-                                <> T.unpack _pname+          -- If we fail to prefetch, we should fail on this package+          case _prfetched of+            Just _prfetched -> do+              buildDir <- getBuildDir+              -- extract src+              _prextract <-+                case _pextract of+                  Just (PackageExtractSrc extract) -> do+                    result <- HMap.toList <$> extractSrcs _prfetched extract+                    Just . HMap.fromList+                      <$> sequence+                        [ do+                            -- write extracted files to build dir+                            -- and read them in nix using 'builtins.readFile'+                            writeFile' (buildDir </> path) (T.unpack v)+                            pure (k, T.pack path)+                          | (k, v) <- result,+                            let path =+                                  "./"+                                    <> T.unpack _pname+                                    <> "-"+                                    <> T.unpack (coerce version)+                                    </> k+                        ]+                  _ -> pure Nothing+              -- cargo locks+              _prcargolock <-+                case _pcargo of+                  Just (PackageCargoLockFiles lockPath) -> do+                    lockFiles <- HMap.toList <$> extractSrcs _prfetched lockPath+                    result <- parallel $+                      flip fmap lockFiles $ \(lockPath, lockData) -> do+                        result <- fetchRustGitDeps _prfetched lockPath+                        let lockPath' =+                              T.unpack _pname                                 <> "-"                                 <> T.unpack (coerce version)-                                </> k-                    ]-              _ -> pure Nothing-          -- cargo locks-          _prcargolock <--            case _pcargo of-              Just (PackageCargoLockFiles lockPath) -> do-                lockFiles <- HMap.toList <$> extractSrcs _prfetched lockPath-                result <- parallel $-                  flip fmap lockFiles $ \(lockPath, lockData) -> do-                    result <- fetchRustGitDeps _prfetched lockPath-                    let lockPath' =-                          T.unpack _pname-                            <> "-"-                            <> T.unpack (coerce version)-                            </> lockPath-                        lockPathNix = "./" <> T.pack lockPath'-                    -- similar to extract src, write lock file to build dir-                    writeFile' (buildDir </> lockPath') $ T.unpack lockData-                    pure (lockPath, (lockPathNix, result))-                pure . Just $ HMap.fromList result-              _ -> pure Nothing+                                </> lockPath+                            lockPathNix = "./" <> T.pack lockPath'+                        -- similar to extract src, write lock file to build dir+                        writeFile' (buildDir </> lockPath') $ T.unpack lockData+                        pure (lockPath, (lockPathNix, result))+                    pure . Just $ HMap.fromList result+                  _ -> pure Nothing -          -- Only git version source supports git commit date-          _prgitdate <- case versionSource of-            Git {..} -> Just <$> getGitCommitDate _vurl (coerce version) _pgitdateformat-            _ -> pure Nothing+              -- Only git version source supports git commit date+              _prgitdate <- case versionSource of+                Git {..} -> Just <$> getGitCommitDate _vurl (coerce version) _pgitdateformat+                _ -> pure Nothing -          -- update changelog-          -- always use on disk version-          mOldV <- getLastVersionOnDisk pkg-          case mOldV of-            Nothing ->-              recordVersionChange _pname Nothing version-            Just old-              | old /= version ->-                recordVersionChange _pname (Just old) version-            _ -> pure ()+              -- update changelog+              -- always use on disk version+              mOldV <- getLastVersionOnDisk pkg+              case mOldV of+                Nothing ->+                  recordVersionChange _pname Nothing version+                Just old+                  | old /= version ->+                      recordVersionChange _pname (Just old) version+                _ -> pure () -          let _prpassthru = if HMap.null passthru then Nothing else Just passthru-              _prname = _pname-              _prpinned = _ppinned-          -- Since we don't save the previous result, we are not able to know if the result changes-          -- Depending on this rule leads to RunDependenciesChanged-          pure $ RunResult ChangedRecomputeDiff mempty PackageResult {..}+              let _prpassthru = if HMap.null passthru then Nothing else Just passthru+                  _prname = _pname+                  _prpinned = _ppinned+              -- Since we don't save the previous result, we are not able to know if the result changes+              -- Depending on this rule leads to RunDependenciesChanged+              pure $ RunResult ChangedRecomputeDiff mempty $ Just PackageResult {..}+            _ -> pure $ RunResult ChangedRecomputeDiff mempty Nothing  -- | 'Core' rule take a 'PackageKey', find the corresponding 'Package', and run all needed rules to get 'PackageResult'-runPackage :: PackageKey -> Action PackageResult+runPackage :: PackageKey -> Action (Maybe PackageResult) runPackage k = apply1 $ WithPackageKey (Core, k)

@@ -24,6 +24,7 @@ where  import Control.Monad (void)+import Control.Monad.Extra (fromMaybeM) import Data.Binary.Instances () import Data.Coerce (coerce) import Data.HashMap.Strict (HashMap)@@ -54,7 +55,7 @@       parallel         [ case parse gitSrcParser (T.unpack rname) src of             Right ParsedGitSrc {..} -> do-              (_sha256 -> sha256) <- prefetch (gitFetcher pgurl pgsha) NoForceFetch+              (_sha256 -> sha256) <- fromMaybeM (fail $ "Prefetch failed for " <> T.unpack pgurl) $ prefetch (gitFetcher pgurl pgsha) NoForceFetch               -- @${name}-${version}@ -> sha256               pure (rname <> "-" <> coerce rversion, sha256)             Left err -> fail $ "Failed to parse git source in Cargo.lock: " <> show err

@@ -67,6 +67,7 @@       _fetchSubmodules = toNixExpr -> fetchSubmodules,       _deepClone = toNixExpr -> deepClone,       _leaveDotGit = toNixExpr -> leaveDotGit,+      _sparseCheckout = toNixExpr . map quote -> sparseCheckout,       _furl = quote -> url,       _name = nameField -> n     } ->@@ -76,7 +77,8 @@             rev = $rev;             fetchSubmodules = $fetchSubmodules;             deepClone = $deepClone;-            leaveDotGit = $leaveDotGit;$n+            leaveDotGit = $leaveDotGit;+            sparseCheckout = $sparseCheckout;$n             sha256 = $sha256;           }     |]@@ -86,13 +88,14 @@       _fetchSubmodules = toNixExpr -> fetchSubmodules,       _deepClone = toNixExpr -> deepClone,       _leaveDotGit = toNixExpr -> leaveDotGit,+      _sparseCheckout = toNixExpr . map quote -> sparseCheckout,       _fowner = quote -> owner,       _frepo = quote -> repo,       _name = nameField -> n     } ->-      -- TODO: fix fetchFromGitHub in Nixpkgs so that deepClone and-      -- leaveDotGit won't get passed to fetchzip-      if (deepClone == "true") || (leaveDotGit == "true")+      -- TODO: fix fetchFromGitHub in Nixpkgs so that deepClone, leaveDotGit+      -- and sparseCheckout won't get passed to fetchzip+      if (deepClone == "true") || (leaveDotGit == "true") || (sparseCheckout /= "[ ]")         then           [trimming|                fetchFromGitHub {@@ -101,7 +104,8 @@                  rev = $rev;                  fetchSubmodules = $fetchSubmodules;                  deepClone = $deepClone;-                 leaveDotGit = $leaveDotGit;$n+                 leaveDotGit = $leaveDotGit;+                 sparseCheckout = $sparseCheckout;$n                  sha256 = $sha256;                }          |]

@@ -48,12 +48,14 @@     gitHubReleaseFetcher',     gitFetcher,     urlFetcher,+    urlFetcher',     openVsxFetcher,     vscodeMarketplaceFetcher,     tarballFetcher,   ) where +import Control.Exception (ErrorCall) import Control.Monad (void, when) import qualified Data.Aeson as A import Data.Coerce (coerce)@@ -79,12 +81,14 @@     [x] -> pure $ coerce x     _ -> fail $ "Failed to parse output from nix hash to-sri: " <> T.unpack out -runNixPrefetchUrl :: Text -> Bool -> Action Checksum-runNixPrefetchUrl url unpack = do+runNixPrefetchUrl :: Text -> Bool -> Maybe Text -> Action Checksum+runNixPrefetchUrl url unpack name = do   (CmdTime t, Stdout (T.decodeUtf8 -> out), CmdLine c) <-     quietly $       command [EchoStderr False] "nix-prefetch-url" $-        [T.unpack url] <> ["--unpack" | unpack]+        [T.unpack url]+          <> ["--unpack" | unpack]+          <> concat [["--name", T.unpack name] | Just name <- [name]]   putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"   case takeWhile (not . T.null) $ reverse $ T.lines out of     [x] -> sha256ToSri x@@ -93,8 +97,8 @@ newtype FetchedGit = FetchedGit {sha256 :: Text}   deriving (Show, Generic, A.FromJSON) -runNixPrefetchGit :: Text -> Text -> Bool -> Bool -> Bool -> Action Checksum-runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit = do+runNixPrefetchGit :: Text -> Text -> Bool -> Bool -> Bool -> [Text] -> Action Checksum+runNixPrefetchGit url rev fetchSubmodules deepClone leaveDotGit sparseCheckout = do   (CmdTime t, Stdout out, CmdLine c) <-     quietly $       command [EchoStderr False] "nix-prefetch-git" $@@ -103,6 +107,7 @@           <> ["--fetch-submodules" | fetchSubmodules]           <> ["--deepClone" | deepClone]           <> ["--leave-dotGit" | leaveDotGit]+          <> if null sparseCheckout then [] else ["--sparse-checkout", T.unpack $ T.intercalate "\n" sparseCheckout]   putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"   case A.eitherDecode out of     Right (FetchedGit x) -> sha256ToSri x@@ -113,21 +118,21 @@ runFetcher :: NixFetcher Fresh -> Action (NixFetcher Fetched) runFetcher = \case   FetchGit {..} -> do-    result <- runNixPrefetchGit _furl (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit+    result <- runNixPrefetchGit _furl (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit _sparseCheckout     pure FetchGit {_sha256 = coerce result, ..}   FetchGitHub {..} -> do-    let useFetchGit = _fetchSubmodules || _leaveDotGit || _deepClone+    let useFetchGit = _fetchSubmodules || _leaveDotGit || _deepClone || not (null _sparseCheckout)         ver = coerce _rev     result <-       if useFetchGit-        then runNixPrefetchGit [trimming|https://github.com/$_fowner/$_frepo|] (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit-        else runNixPrefetchUrl [trimming|https://github.com/$_fowner/$_frepo/archive/$ver.tar.gz|] True+        then runNixPrefetchGit [trimming|https://github.com/$_fowner/$_frepo|] (coerce _rev) _fetchSubmodules _deepClone _leaveDotGit _sparseCheckout+        else runNixPrefetchUrl [trimming|https://github.com/$_fowner/$_frepo/archive/$ver.tar.gz|] True mempty     pure FetchGitHub {_sha256 = result, ..}   FetchUrl {..} -> do-    result <- runNixPrefetchUrl _furl False+    result <- runNixPrefetchUrl _furl False _name     pure FetchUrl {_sha256 = result, ..}   FetchTarball {..} -> do-    result <- runNixPrefetchUrl _furl True+    result <- runNixPrefetchUrl _furl True mempty     pure FetchTarball {_sha256 = result, ..}   FetchDocker {..} -> do     (CmdTime t, Stdout out, CmdLine c) <-@@ -165,24 +170,31 @@   addOracleCache $ \(RunFetch force f) -> do     when (force == ForceFetch) alwaysRerun     putInfo . show $ "#" <+> pretty f-    withRetry $ runFetcher f+    keepGoing <- nvcheckerKeepGoing+    if keepGoing+      then -- If fetch failed, always rerun and return Nothing+      actionCatch (fmap Just <$> withRetry $ runFetcher f) $ \(e :: ErrorCall) -> do+        alwaysRerun+        putError $ show e <> "\nKeep going..."+        pure Nothing+      else fmap Just <$> withRetry $ runFetcher f  -- | Run nix fetcher-prefetch :: NixFetcher Fresh -> ForceFetch -> Action (NixFetcher Fetched)+prefetch :: NixFetcher Fresh -> ForceFetch -> Action (Maybe (NixFetcher Fetched)) prefetch f force = askOracle $ RunFetch force f  --------------------------------------------------------------------------------  -- | Create a fetcher from git url gitFetcher :: Text -> PackageFetcher-gitFetcher furl rev = FetchGit furl rev False True False Nothing ()+gitFetcher furl rev = FetchGit furl rev False True False [] Nothing ()  -- | Create a fetcher from github repo gitHubFetcher ::   -- | owner and repo   (Text, Text) ->   PackageFetcher-gitHubFetcher (owner, repo) rev = FetchGitHub owner repo rev False False False Nothing ()+gitHubFetcher (owner, repo) rev = FetchGitHub owner repo rev False False False [] Nothing ()  -- | Create a fetcher from pypi pypiFetcher :: Text -> PackageFetcher@@ -212,6 +224,10 @@ -- | Create a fetcher from url urlFetcher :: Text -> NixFetcher Fresh urlFetcher url = FetchUrl url Nothing ()++-- | Create a fetcher from url specifying the file name+urlFetcher' :: Text -> Maybe Text -> NixFetcher Fresh+urlFetcher' url name = FetchUrl url name ()  -- | Create a fetcher from openvsx openVsxFetcher ::

@@ -131,10 +131,10 @@           genOptions options       )   where-    key =: x = tell [key <> " = " <> T.pack (show $ T.unpack x)]+    key =: x = tell [key <> " = " <> quote x]     key =:? (Just x) = key =: x     _ =:? _ = pure ()-    table t m = tell ["[" <> t <> "]"] >> m >> tell [""]+    table t m = tell ["[" <> quote t <> "]"] >> m >> tell [""]     genVersionSource = \case       GitHubRelease {..} -> do         "source" =: "github"

@@ -19,23 +19,26 @@ import Options.Applicative.Simple import qualified Paths_nvfetcher as Paths -data Target = Build | Clean+data Target = Build | Clean | Purge   deriving (Eq)  instance Show Target where   show Build = "build"   show Clean = "clean"+  show Purge = "purge"  targetParser :: ReadM Target targetParser = maybeReader $ \case   "build" -> Just Build   "clean" -> Just Clean+  "purge" -> Just Purge   _ -> Nothing  -- | Options for nvfetcher CLI data CLIOptions = CLIOptions   { optBuildDir :: FilePath,     optCommit :: Bool,+    optCommitSummary :: Maybe String,     optLogPath :: Maybe FilePath,     optThreads :: Int,     optRetry :: Int,@@ -43,6 +46,8 @@     optVerbose :: Bool,     optPkgNameFilter :: Maybe String,     optKeyfile :: Maybe FilePath,+    optKeepOldFiles :: Bool,+    optKeepGoing :: Bool,     optTarget :: Target   }   deriving (Show)@@ -65,6 +70,13 @@       )     <*> optional       ( strOption+          ( long "commit-summary"+              <> metavar "SUMMARY"+              <> help "Summary to use when committing changes"+          )+      )+    <*> optional+      ( strOption           ( long "changelog"               <> short 'l'               <> metavar "FILE"@@ -108,12 +120,14 @@               <> completer (bashCompleter "file")           )       )+    <*> switch (long "keep-old" <> help "Don't remove old files other than generated json and nix before build")+    <*> switch (long "keep-going" <> help "Don't stop if some packages failed to be fetched")     <*> argument       targetParser       ( metavar "TARGET"-          <> help "Two targets are available: 1.build  2.clean"+          <> help "Three targets are available: 1.build  2.clean (remove all generated files) 3.purge (remove shake db)"           <> value Build-          <> completer (listCompleter [show Build, show Clean])+          <> completer (listCompleter [show Build, show Clean, show Purge])           <> showDefault       ) 

@@ -75,6 +75,7 @@     fetchGit,     fetchGit',     fetchUrl,+    fetchUrl',     fetchOpenVsx,     fetchVscodeMarketplace,     fetchTarball,@@ -194,23 +195,23 @@ class Member (a :: Type) (r :: [Type]) where   proj :: Prod r -> a -instance {-# OVERLAPPING #-} NotElem x xs => Member x (x ': xs) where+instance {-# OVERLAPPING #-} (NotElem x xs) => Member x (x ': xs) where   proj (Cons x _) = x -instance Member x xs => Member x (_y ': xs) where+instance (Member x xs) => Member x (_y ': xs) where   proj (Cons _ r) = proj r -instance TypeError (ShowType x :<>: 'Text " is undefined") => Member x '[] where+instance (TypeError (ShowType x :<>: 'Text " is undefined")) => Member x '[] where   proj = undefined  -- | Project optional elements from 'Prod' class OptionalMember (a :: Type) (r :: [Type]) where   projMaybe :: Prod r -> Maybe a -instance {-# OVERLAPPING #-} NotElem x xs => OptionalMember x (x ': xs) where+instance {-# OVERLAPPING #-} (NotElem x xs) => OptionalMember x (x ': xs) where   projMaybe (Cons x _) = Just x -instance OptionalMember x xs => OptionalMember x (_y ': xs) where+instance (OptionalMember x xs) => OptionalMember x (_y ': xs) where   projMaybe (Cons _ r) = projMaybe r  instance OptionalMember x '[] where@@ -514,6 +515,12 @@ -- Arg is a function which constructs the url from a version fetchUrl :: Attach PackageFetcher (Version -> Text) fetchUrl e f = fetch e (urlFetcher . f)++-- | This package is fetched from url+--+-- Args are a function which constructs the url from a version and a file name+fetchUrl' :: Attach PackageFetcher (Text, Version -> Text)+fetchUrl' e (name, f) = fetch e (\v -> FetchUrl (f v) (Just name) ())  -- | This package is fetched from Open VSX --

@@ -383,7 +383,8 @@ data RunFetch = RunFetch ForceFetch (NixFetcher Fresh)   deriving (Show, Eq, Ord, Hashable, NFData, Binary, Typeable, Generic) -type instance RuleResult RunFetch = NixFetcher Fetched+-- Prefetch rule never throws exceptions+type instance RuleResult RunFetch = Maybe (NixFetcher Fetched)  -- | If the package is prefetched, then we can obtain the SHA256 data NixFetcher (k :: FetchStatus)@@ -393,6 +394,7 @@         _deepClone :: Bool,         _fetchSubmodules :: Bool,         _leaveDotGit :: Bool,+        _sparseCheckout :: [Text],         _name :: Maybe Text,         _sha256 :: FetchResult Checksum k       }@@ -403,6 +405,7 @@         _deepClone :: Bool,         _fetchSubmodules :: Bool,         _leaveDotGit :: Bool,+        _sparseCheckout :: [Text],         _name :: Maybe Text,         _sha256 :: FetchResult Checksum k       }@@ -460,6 +463,7 @@         "deepClone" A..= _deepClone,         "fetchSubmodules" A..= _fetchSubmodules,         "leaveDotGit" A..= _leaveDotGit,+        "sparseCheckout" A..= _sparseCheckout,         "name" A..= _name,         "sha256" A..= _sha256,         "type" A..= A.String "git"@@ -472,6 +476,7 @@         "deepClone" A..= _deepClone,         "fetchSubmodules" A..= _fetchSubmodules,         "leaveDotGit" A..= _leaveDotGit,+        "sparseCheckout" A..= _sparseCheckout,         "name" A..= _name,         "sha256" A..= _sha256,         "type" A..= A.String "github"@@ -513,7 +518,8 @@               "rev" <> colon <+> pretty _rev,               "deepClone" <> colon <+> pretty _deepClone,               "fetchSubmodules" <> colon <+> pretty _fetchSubmodules,-              "leaveDotGit" <> colon <+> pretty _leaveDotGit+              "leaveDotGit" <> colon <+> pretty _leaveDotGit,+              "sparseCheckout" <> colon <+> pretty _sparseCheckout             ]               <> ppField "name" _name         )@@ -528,7 +534,8 @@               "rev" <> colon <+> pretty _rev,               "deepClone" <> colon <+> pretty _deepClone,               "fetchSubmodules" <> colon <+> pretty _fetchSubmodules,-              "leaveDotGit" <> colon <+> pretty _leaveDotGit+              "leaveDotGit" <> colon <+> pretty _leaveDotGit,+              "sparseCheckout" <> colon <+> pretty _sparseCheckout             ]               <> ppField "name" _name         )@@ -702,7 +709,8 @@ data Core = Core   deriving (Eq, Show, Ord, Typeable, Generic, Hashable, Binary, NFData) -type instance RuleResult Core = PackageResult+-- If prefetch fails, we don't want to fail the whole build+type instance RuleResult Core = Maybe PackageResult  -- | Decorate a rule's key with 'PackageKey' newtype WithPackageKey k = WithPackageKey (k, PackageKey)

@@ -40,8 +40,9 @@     getAllOnDiskVersions,     getLastVersionUpdated, -    -- * Cache nvchecker+    -- * Booleans     nvcheckerCacheEnabled,+    nvcheckerKeepGoing,   ) where @@ -175,3 +176,7 @@ -- | Get if 'cacheNvchecker' is enabled nvcheckerCacheEnabled :: Action Bool nvcheckerCacheEnabled = cacheNvchecker . config <$> getShakeExtras++-- | Get if 'keepGoing' is enabled+nvcheckerKeepGoing :: Action Bool+nvcheckerKeepGoing = keepGoing . config <$> getShakeExtras

@@ -3,6 +3,7 @@  module FetchRustGitDepsSpec where +import Control.Monad (join) import Control.Monad.Trans.Reader import Data.HashMap.Strict (HashMap) import qualified Data.HashMap.Strict as HMap@@ -30,7 +31,7 @@           ]  runPrefetchRule :: NixFetcher Fresh -> ReaderT ActionQueue IO (Maybe (NixFetcher Fetched))-runPrefetchRule fetcher = runActionChan $ prefetch fetcher NoForceFetch+runPrefetchRule fetcher = fmap join $ runActionChan $ prefetch fetcher NoForceFetch  runFetchRustGitDepsRule :: NixFetcher Fetched -> FilePath -> ReaderT ActionQueue IO (Maybe (HashMap Text Checksum)) runFetchRustGitDepsRule fetcher lockPath = runActionChan $ fetchRustGitDeps fetcher lockPath

@@ -34,6 +34,7 @@         fetchSubmodules = true;         deepClone = false;         leaveDotGit = false;+        sparseCheckout = [ ];         sha256 = "0000000000000000000000000000000000000000000000000000000000000000";       }     |]

@@ -4,6 +4,7 @@ module PrefetchSpec where  import Control.Arrow ((&&&))+import Control.Monad (join) import Control.Monad.Trans.Reader import NvFetcher.NixFetcher import NvFetcher.Types@@ -39,11 +40,18 @@      specifyChan "docker" $       runPrefetchRule' (_sha256 &&& _imageDigest) testDockerFetcher-        `shouldReturnJust`-          ( Checksum "sha256-uaJxeiRm94tWDBTe51/KwUBKR2vj9i4i3rhotsYPxtM=",-            ContainerDigest "sha256:65a2763f593ae85fab3b5406dc9e80f744ec5b449f269b699b5efd37a07ad32e"-          )+        `shouldReturnJust` ( Checksum "sha256-uaJxeiRm94tWDBTe51/KwUBKR2vj9i4i3rhotsYPxtM=",+                             ContainerDigest "sha256:65a2763f593ae85fab3b5406dc9e80f744ec5b449f269b699b5efd37a07ad32e"+                           ) +    specifyChan "url with name" $+      runPrefetchRule+        ( urlFetcher'+            "https://files.yande.re/image/3fc51f6a2fb10c96b73dd0fce6ddb9c8/yande.re%201048717%20dress%20garter%20lolita_fashion%20ruo_gan_zhua.jpg"+            (Just "foo.jpg")+        )+        `shouldReturnJust` Checksum "sha256-wkiXDN6vPFtx88krcQ4szK6dJNjtrDxrsNa3ZvHlfMQ="+ testDockerFetcher :: NixFetcher Fresh testDockerFetcher =   FetchDocker@@ -65,4 +73,4 @@ runPrefetchRule = runPrefetchRule' _sha256  runPrefetchRule' :: (NixFetcher Fetched -> a) -> NixFetcher Fresh -> ReaderT ActionQueue IO (Maybe a)-runPrefetchRule' g f = runActionChan $ g <$> prefetch f NoForceFetch+runPrefetchRule' g f = fmap join $ runActionChan $ prefetch f NoForceFetch >>= \m -> pure $ g <$> m