nvfetcher 0.7.0.0 → 0.8.0.0

raw patch · 26 files changed

+509/−220 lines, 26 filesdep +Globdep +timePVP ok

version bump matches the API change (PVP)

Dependencies added: Glob, time

API changes (from Hackage documentation)

Files

• CHANGELOG.md +16/−0
• Main_example.hs +2/−2
• README.md +92/−44
• app/Config.hs +9/−5
• app/Config/VersionSource.hs +1/−0
• nvfetcher.cabal +9/−6
• src/NvFetcher.hs +16/−9
• src/NvFetcher/Config.hs +1/−1
• src/NvFetcher/Core.hs +9/−34
• src/NvFetcher/ExtractSrc.hs +50/−28
• src/NvFetcher/FetchRustGitDeps.hs +17/−5
• src/NvFetcher/GetGitCommitDate.hs +18/−6
• src/NvFetcher/NixExpr.hs +20/−11
• src/NvFetcher/NixFetcher.hs +11/−4
• src/NvFetcher/Nvchecker.hs +3/−3
• src/NvFetcher/Options.hs +1/−1
• src/NvFetcher/PackageSet.hs +22/−13
• src/NvFetcher/Types.hs +77/−31
• src/NvFetcher/Types/Lens.hs +1/−1
• src/NvFetcher/Types/ShakeExtras.hs +1/−1
• src/NvFetcher/Utils.hs +18/−5
• test/CheckVersionSpec.hs +3/−3
• test/ExtractSpec.hs +73/−0
• test/GetGitCommitDateSpec.hs +31/−3
• test/PrefetchSpec.hs +3/−3
• test/Utils.hs +5/−1

@@ -1,5 +1,21 @@ # Revision history for nvfetcher +## 0.8.0.0++There is a major rework on Source File Extract, which is a *breaking change* and requires users to update their configuration files and the usage of generated Nix expressions.++1. Now both `extract` and `cargo_lock` (renamed from `cargo_locks`) options accept a list of zsh style globs.+2. `ExtractSrc` rule now copies all matched files to the output directory and generates a mapping between relative paths in the output directory and original paths in the source repository, rather than loading them into memory as text files. This makes extracting binary files possible.+3. There are several structural changes in generated Nix expressions. See https://github.com/berberman/nvfetcher/pull/139 for details.++Other changes:+* Add option `git.date_tz` to specify the time zone for `GetGitCommitDate` rule.+* Add `awesomeversion` sort_version_key for nvchecker. Thanks to @auscyber.+* Generate `hash` for Docker fetcher. Thanks to @amesgen.+* Fix binary packaging issue on aarch64-darwin. Thanks to @truelecter.+* Make generated Nix file satisfy nixfmt-rfc-style. Thanks to @zzzsyyy.++ ## 0.7.0.0  Now nvfetcher removes all files *except* `generated.json` and `generated.nix` in `_sources` before each run. If you do want to keep those files, you can use the new CLI option `--keep-old`. In addition, a new target `purge` is introduced for resetting the state of nvfetcher by deleting the shake database saved in XDG directory. 

@@ -36,7 +36,7 @@     package "rust-git-dependency-example"       `sourceManual` "8a5f37a8f80a3b05290707febf57e88661cee442"       `fetchGit` "https://gist.github.com/NickCao/6c4dbc4e15db5da107de6cdb89578375"-      `hasCargoLocks` ["Cargo.lock"]+      `hasCargoLock` ["Cargo.lock"]    define $ package "vscode-LiveServer" `fromOpenVsx` ("ritwickdey", "LiveServer") @@ -44,4 +44,4 @@     package "revda"       `sourceGit` "https://github.com/THMonster/Revda"       `fetchGitHub'` ("THMonster", "Revda", fetchSubmodules .~ True)-      `hasCargoLocks` ["dmlive/Cargo.lock", "dmlive/tars-stream/Cargo.lock"]+      `hasCargoLock` ["dmlive/Cargo.lock", "dmlive/tars-stream/Cargo.lock"]

@@ -4,71 +4,106 @@ [![MIT license](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE) [![nix](https://github.com/berberman/nvfetcher/actions/workflows/nix.yml/badge.svg)](https://github.com/berberman/nvfetcher/actions/workflows/nix.yml) -nvfetcher is a tool to automate nix package updates. It's built on top of [shake](https://www.shakebuild.com/),+`nvfetcher` is a tool to automate nix package updates. It's built on top of [shake](https://www.shakebuild.com/), integrating [nvchecker](https://github.com/lilydjwg/nvchecker).-nvfetcher cli program accepts a TOML file as config, which defines a set of package sources to run.+`nvfetcher` CLI program accepts a TOML file as config, which defines a set of package sources to run.  ## Overview -For example, feeding the following configuration to`nvfetcher`:+For example, feeding the following configuration to `nvfetcher`:  ```toml # nvfetcher.toml-[feeluown]-src.pypi = "feeluown"-fetch.pypi = "feeluown"+[rustdeck]+src.github = "rustdesk/rustdesk"+fetch.github = "rustdesk/rustdesk"+# Compute outputHashes for git dependencies in cargo lock+cargo_lock = ["Cargo.lock"] ``` -it will create `_sources/generated.nix`:+You would get `_sources/generated.nix`:  ```nix-{ fetchgit, fetchurl, fetchFromGitHub }:+# This file was generated by nvfetcher, please do not modify it manually. {-  feeluown = {-    pname = "feeluown";-    version = "3.8.2";-    src = fetchurl {-      url = "https://pypi.io/packages/source/f/feeluown/feeluown-3.8.2.tar.gz";-      sha256 = "sha256-V2yzpkmjRkipZOvQGB2mYRhiiEly6QPrTOMJ7BmyWBQ=";+  fetchgit,+  fetchurl,+  fetchFromGitHub,+  dockerTools,+}:+{+  rustdesk = {+    pname = "rustdesk";+    version = "1.4.2";+    src = fetchFromGitHub {+      owner = "rustdesk";+      repo = "rustdesk";+      rev = "1.4.2";+      fetchSubmodules = false;+      sha256 = "sha256-b/KuWECsmEcz9nPiuPqCcGQ1p+GdbaELScaIXnUTGv8=";     };+    cargoLock."Cargo.lock" = {+      lockFile = ./. + "/sha256-b_KuWECsmEcz9nPiuPqCcGQ1p+GdbaELScaIXnUTGv8=/Cargo.lock";+      outputHashes = {+        "filedescriptor-0.8.2" = "sha256-zXTt9eHAVdCPipWAMFeiqk1JteX+7IZcXZ7uvxkzDjQ=";+         # ...+      };+    };   }; }++ ```  and `_sources/generated.json`:  ```json {-  "feeluown": {-    "pinned": false,-    "cargoLocks": null,-    "name": "feeluown-core",-    "version": "3.8.2",-    "passthru": null,-    "src": {-      "url": "https://pypi.io/packages/source/f/feeluown/feeluown-3.8.2.tar.gz",-      "name": null,-      "type": "url",-      "sha256": "sha256-V2yzpkmjRkipZOvQGB2mYRhiiEly6QPrTOMJ7BmyWBQ="-    },-    "extract": null,-    "rustGitDeps": null-  }+    "rustdesk": {+        "cargoLock": {+            "Cargo.lock": [+                "sha256-b_KuWECsmEcz9nPiuPqCcGQ1p+GdbaELScaIXnUTGv8=/Cargo.lock",+                {+                    "android-wakelock-0.1.0": "sha256-09EH/U1BBs3l4galQOrTKmPUYBgryUjfc/rqPZhdYc4=",+                    // ...+                }+            ]+        },+        "date": null,+        "extract": null,+        "name": "rustdesk",+        "passthru": null,+        "pinned": false,+        "src": {+            "deepClone": false,+            "fetchSubmodules": false,+            "leaveDotGit": false,+            "name": null,+            "owner": "rustdesk",+            "repo": "rustdesk",+            "rev": "1.4.2",+            "sha256": "sha256-b/KuWECsmEcz9nPiuPqCcGQ1p+GdbaELScaIXnUTGv8=",+            "sparseCheckout": [],+            "type": "github"+        },+        "version": "1.4.2"+    } } ``` -We tell nvfetcher how to get the latest version number of packages and how to fetch their sources given version numbers,-and nvfetcher will help us keep their version and prefetched SHA256 sums up-to-date, producing ready-to-use nix expressions in `_sources/generated.nix`.-Nvfetcher reads `generated.json` to produce version change message, such as `feeluown: 3.8.1 → 3.8.2`.-We always check versions of packages during each run, but only do prefetch and further operations when needed.+We tell `nvfetcher` how to get the latest version number of packages and how to fetch their sources given version numbers,+and `nvfetcher` will help us keep their version and prefetched SHA256 sums up-to-date, producing ready-to-use nix expressions in `_sources/generated.nix`. +`nvfetcher` reads `generated.json` to produce version change messages, such as `feeluown: 3.8.1 → 3.8.2`.+Versions of packages are always checked during each run, but only do prefetch and further operations when needed.+ ### Live examples  How to use the generated sources file? Here are several examples:  - My [flakes repo](https://github.com/berberman/flakes) -- Nick Cao's [flakes repo](https://gitlab.com/NickCao/flakes/-/tree/master/pkgs)+**Add your own flakes repo here if you want to share it with others!**  ## Installation @@ -80,14 +115,14 @@  This repo also has flakes support: -```+```sh $ nix run github:berberman/nvfetcher ```  To use it as a Haskell library, the package is available on [Hackage](https://hackage.haskell.org/package/nvfetcher). If you want to use the Haskell library from flakes, there is also a shell `ghcWithNvfetcher`: -```+```sh $ nix develop github:berberman/nvfetcher#ghcWithNvfetcher $ runghc Main.hs ```@@ -96,11 +131,11 @@  ## Usage -Basically, there are two ways to use nvfetcher, where the difference is how we provide package sources definitions to it.+Basically, there are two ways to use `nvfetcher`, where the difference is how we provide package sources definitions to it.  ### CLI -To run nvfetcher as a CLI program, you'll need to provide package sources defined in TOML.+To run `nvfetcher` as a CLI program, you'll need to provide package sources defined in TOML.  ``` Usage: nvfetcher [--version] [--help] [-o|--build-dir DIR] [--commit-changes] @@ -136,13 +171,13 @@                            (default: "nvfetcher.toml") ``` -Each _package_ corresponds to a TOML table, whose name is encoded as table key, with+Each package corresponds to a TOML table: name is encoded as table key, with two required fields and three optional fields in each table. You can find an example of the configuration file, see [`nvfetcher_example.toml`](nvfetcher_example.toml).  ### Keyfile -You can specify nvchecker keyfile via command line option.+You can specify `nvchecker` keyfile via command line option. For the format of this file, please refer to [nvchecker documentation](https://nvchecker.readthedocs.io/en/latest/usage.html#configuration-table).  #### Nvchecker@@ -212,12 +247,20 @@ - `docker.finalImageTag` - `docker.tlsVerify` -#### Extract src+#### Extract source files -Optional _extract src_ config, files are extracted into build directory, and then read by `readFile` in generated nix expr.+Optional source extracting config, files are extracted into build directory. -- `extract = [ "file_1", "file_2", ...]` - file paths are relative to the source root+- `extract = [ "glob_1", "glob_2", ...]` - globs are relative to the source root +Each glob will be resolved to a list of files matching the pattern.+For each matching file, `nvfetcher` copies it to the build directory, and generates `extract.<file_name> = ./. + "<path>"` entry in the nix expr.++Recursive globs are supported, e.g. one can write `extract = [ "**/*" ]` to pull all files in a package to build directory. +See [Glob](https://hackage.haskell.org/package/Glob-0.10.2/docs/System-FilePath-Glob.html#v:compile) for details.++Note: Directories are ignored.+ #### Rust support  `rustPlatform.buildRustPackage` now accepts an attribute [`cargoLock`](https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/rust.section.md#importing-a-cargolock-file) to vendor dependencies from `Cargo.lock`,@@ -225,8 +268,10 @@ extracting the lock file to build and calculating `cargoLock.outputHashes`, as long as you set the config. There can be many lock files in one source. -- `cargo_locks = [ "cargo_lock_path_1", "cargo_lock_path_2", ...]` - relative to the source root+- `cargo_lock = [ "cargo_lock_glob_1", "cargo_lock_glob_2", ...]` - globs are relative to the source root +Similar to extract, cargo locks will be pulled into build directory.+ #### Passthru  _passthru_ config, an additional set of attrs to be generated.@@ -247,6 +292,9 @@ in the format of `%Y-%m-%d` by default. You can provide your own [`strftime`](https://strftime.org/) format:  - `git.date_format = "strftime_format"`+- `git.date_tz = "timezone"`++You can use the `git.date_tz` option to specify the time zone for the commit date. Accepts IANA time zone names, e.g. `git.date_tz = "America/New_York"`. Plus, you can use `git.date_tz = "local"` to refer to the local time zone.  #### Force fetching 

@@ -11,6 +11,7 @@ import Control.Monad.Trans.Except import qualified Data.HashMap.Strict as HMap import Data.List (foldl', intersect)+import qualified Data.List.NonEmpty as NE import qualified Data.Map.Strict as Map import Data.Text (Text) import qualified Data.Text as T@@ -110,16 +111,16 @@     <*> cargoLockPathDecoder     <*> passthruDecoder     <*> pinnedDecoder-    <*> gitDateFormatDecoder+    <*> ((,) <$> gitDateFormatDecoder <*> gitTimeZoneDecoder)     <*> forceFetchDecoder  --------------------------------------------------------------------------------  extractFilesDecoder :: Decoder (Maybe PackageExtractSrc)-extractFilesDecoder = fmap PackageExtractSrc <$> getFieldOpt "extract"+extractFilesDecoder = fmap (PackageExtractSrc . fmap Glob) <$> getFieldOpt @(NE.NonEmpty String) "extract"  cargoLockPathDecoder :: Decoder (Maybe PackageCargoLockFiles)-cargoLockPathDecoder = fmap PackageCargoLockFiles <$> getFieldOpt "cargo_locks"+cargoLockPathDecoder = fmap (PackageCargoLockFiles . fmap Glob) <$> getFieldOpt @(NE.NonEmpty String) "cargo_lock"  nvcheckerOptionsDecoder :: Decoder NvcheckerOptions nvcheckerOptionsDecoder =@@ -149,8 +150,11 @@   maybe NoStale (\x -> if x then PermanentStale else NoStale)     <$> getFieldOpt "pinned" -gitDateFormatDecoder :: Decoder DateFormat-gitDateFormatDecoder = DateFormat <$> getFieldsOpt ["git", "date_format"]+gitDateFormatDecoder :: Decoder GitDateFormat+gitDateFormatDecoder = GitDateFormat <$> getFieldsOpt ["git", "date_format"]++gitTimeZoneDecoder :: Decoder GitTimeZone+gitTimeZoneDecoder = GitTimeZone <$> getFieldsOpt ["git", "date_tz"]  forceFetchDecoder :: Decoder ForceFetch forceFetchDecoder =

@@ -67,6 +67,7 @@     <$> getFieldOpt "include_regex"     <*> getFieldOpt "exclude_regex"     <*> ( getFieldOpt @Text "sort_version_key" >>= \case+            Just "awesomeversion" -> pure $ Just AwesomeVersion             Just "parse_version" -> pure $ Just ParseVersion             Just "vercmp" -> pure $ Just Vercmp             Just _ -> makeDecoder $ invalidValue "unexpected sort_version_key: it should be either parse_version or vercmp"

@@ -1,6 +1,6 @@ cabal-version:   2.4 name:            nvfetcher-version:         0.7.0.0+version:         0.8.0.0 synopsis:   Generate nix sources expr for the latest version of packages @@ -13,7 +13,7 @@ license-file:    LICENSE author:          berberman maintainer:      berberman <berberman.yandex.com>-copyright:       2021-2023 berberman+copyright:       2021-2025 berberman category:        Nix build-type:      Simple extra-doc-files:@@ -33,19 +33,20 @@     , binary-instances      ^>=1.0     , bytestring     , containers-    , data-default          ^>=0.7.1-    , extra                 ^>=1.7+    , data-default          ^>=0.7   || ^>=0.8+    , extra                 ^>=1.7   || ^>=1.8     , free                  >=5.1    && <5.3+    , Glob                  ^>=0.10     , microlens     , microlens-th     , neat-interpolation    ^>=0.5.1     , optparse-simple       ^>=0.1.1     , parsec     , prettyprinter-    , regex-tdfa            ^>=1.3.1+    , regex-tdfa            ^>=1.3     , shake                 ^>=0.19     , text-    , toml-reader           ^>=0.2+    , toml-reader           ^>=0.2   || ^>=0.3     , transformers     , unordered-containers @@ -114,6 +115,7 @@   main-is:            Spec.hs   other-modules:     CheckVersionSpec+    ExtractSpec     FetchRustGitDepsSpec     GetGitCommitDateSpec     NixExprSpec@@ -127,6 +129,7 @@     , hspec     , nvfetcher     , stm+    , time     , unliftio    build-tool-depends: hspec-discover:hspec-discover

@@ -6,7 +6,7 @@ {-# LANGUAGE TypeApplications #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -164,6 +164,15 @@   -- Set shakeExtra   let shakeOptions2 = shakeOptions1 {shakeExtra = addShakeExtra shakeExtras (shakeExtra shakeConfig)}       rules = mainRules config++  -- Remove all files in build dir except generated nix and json+  -- Since core rule has always rerun, any file not generated in this run will be removed+  -- Put it here to avoid removing files generated in this run+  unless (keepOldFiles || (target /= Build)) $+    whenM (D.doesDirectoryExist buildDir) $ do+      oldFiles <- (\\ [generatedJsonFileName, generatedNixFileName]) <$> D.listDirectory buildDir+      putStrLn $ "Removing old files: " <> show oldFiles+      liftIO $ removeFiles buildDir oldFiles   shake shakeOptions2 $ want [show target] >> rules   where     -- Don't touch already pinned packages@@ -192,13 +201,6 @@     removeFilesAfter shakeDir ["//*"]    "build" ~> do-    -- remove all files in build dir except generated nix and json-    -- since core rule has always rerun, any file not generated in this run will be removed-    unless keepOldFiles $-      whenM (liftIO $ D.doesDirectoryExist buildDir) $ do-        oldFiles <- (\\ [generatedJsonFileName, generatedNixFileName]) <$> liftIO (D.listDirectory buildDir)-        putVerbose $ "Removing old files: " <> show oldFiles-        liftIO $ removeFiles buildDir oldFiles     allKeys <- getAllPackageKeys     results <- fmap (zip allKeys) $ parallel $ runPackage <$> allKeys     let (fmap (fromJust . snd) -> successResults, fmap fst -> failureKeys) = partition (isJust . snd) results@@ -229,7 +231,12 @@ srouces body =   [trimming|     # This file was generated by nvfetcher, please do not modify it manually.-    { fetchgit, fetchurl, fetchFromGitHub, dockerTools }:+    {+      fetchgit,+      fetchurl,+      fetchFromGitHub,+      dockerTools,+    }:     {       $body     }

@@ -1,4 +1,4 @@--- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental

@@ -3,7 +3,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -17,9 +17,7 @@  import Data.Coerce (coerce) import qualified Data.HashMap.Strict as HMap-import qualified Data.Text as T import Development.Shake-import Development.Shake.FilePath import Development.Shake.Rule import NvFetcher.ExtractSrc import NvFetcher.FetchRustGitDeps@@ -39,8 +37,8 @@   fetchRustGitDepsRule   getGitCommitDateRule   addBuiltinRule noLint noIdentity $ \(WithPackageKey (Core, pkg)) _ _ -> do-    -- it's important to always rerun-    -- since the package definition is not tracked at all+    -- It's important to always rerun since the package definition is not tracked at all+    -- Also we generate new files in the build directory     alwaysRerun     lookupPackage pkg >>= \case       Nothing -> fail $ "Unknown package key: " <> show pkg@@ -55,27 +53,12 @@           -- If we fail to prefetch, we should fail on this package           case _prfetched of             Just _prfetched -> do-              buildDir <- getBuildDir               -- extract src               _prextract <-                 case _pextract of                   Just (PackageExtractSrc extract) -> do-                    result <- HMap.toList <$> extractSrcs _prfetched extract-                    Just . HMap.fromList-                      <$> sequence-                        [ do-                            -- write extracted files to build dir-                            -- and read them in nix using 'builtins.readFile'-                            writeFile' (buildDir </> path) (T.unpack v)-                            pure (k, T.pack path)-                          | (k, v) <- result,-                            let path =-                                  "./"-                                    <> T.unpack _pname-                                    <> "-"-                                    <> T.unpack (coerce version)-                                    </> k-                        ]+                    result <- extractSrcs _prfetched extract+                    pure $ Just result                   _ -> pure Nothing               -- cargo locks               _prcargolock <-@@ -83,23 +66,15 @@                   Just (PackageCargoLockFiles lockPath) -> do                     lockFiles <- HMap.toList <$> extractSrcs _prfetched lockPath                     result <- parallel $-                      flip fmap lockFiles $ \(lockPath, lockData) -> do-                        result <- fetchRustGitDeps _prfetched lockPath-                        let lockPath' =-                              T.unpack _pname-                                <> "-"-                                <> T.unpack (coerce version)-                                </> lockPath-                            lockPathNix = "./" <> T.pack lockPath'-                        -- similar to extract src, write lock file to build dir-                        writeFile' (buildDir </> lockPath') $ T.unpack lockData-                        pure (lockPath, (lockPathNix, result))+                      flip fmap lockFiles $ \(srcLockPath, dstLockPath) -> do+                        result <- fetchRustGitDeps _prfetched srcLockPath+                        pure (srcLockPath, (dstLockPath, result))                     pure . Just $ HMap.fromList result                   _ -> pure Nothing                -- Only git version source supports git commit date               _prgitdate <- case versionSource of-                Git {..} -> Just <$> getGitCommitDate _vurl (coerce version) _pgitdateformat+                Git {..} -> Just <$> getGitCommitDate _vurl (coerce version) _pgitdate                 _ -> pure Nothing                -- update changelog

@@ -1,21 +1,19 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ScopedTypeVariables #-}-{-# LANGUAGE TupleSections #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental -- Portability: portable ----- This module provides function that extracs files contents from package sources.--- It uses [IFD](https://nixos.wiki/wiki/Import_From_Derivation) under the hood,--- pulling /textual/ files from source drv.+-- This module provides function that extracts files contents from package sources. -- Because we use @nix-instantiate@ to build drv, so @<nixpkgs>@ (@NIX_PATH@) is required. module NvFetcher.ExtractSrc   ( -- * Types     ExtractSrcQ (..),+    Glob (..),      -- * Rules     extractSrcRule,@@ -26,52 +24,76 @@   ) where -import Control.Monad (void)+import Control.Monad (filterM, forM, join, void, when) import Control.Monad.Extra (unlessM) import Data.Binary.Instances ()+import Data.Coerce (coerce) import Data.HashMap.Strict (HashMap) import qualified Data.HashMap.Strict as HM+import Data.List (intercalate) import qualified Data.List.NonEmpty as NE-import Data.Text (Text) import qualified Data.Text as T-import qualified Data.Text.IO as T import Development.Shake-import Development.Shake.FilePath ((</>))+import Development.Shake.FilePath (makeRelative, (</>)) import NvFetcher.NixExpr import NvFetcher.Types import NvFetcher.Types.ShakeExtras+import NvFetcher.Utils (compileGlob) import Prettyprinter (pretty, (<+>))+import qualified System.Directory.Extra as IO+import System.FilePath.Glob (globDir1)  -- | Rules of extract source extractSrcRule :: Rules () extractSrcRule = void $-  addOracle $ \q@(ExtractSrcQ fetcher files) -> withTempFile $ \fp -> withRetry $ do-    putInfo . show $ "#" <+> pretty q-    let nixExpr = T.unpack $ fetcherToDrv fetcher "nvfetcher-extract"-    putVerbose $ "Generated nix expr:\n" <> nixExpr-    writeFile' fp nixExpr-    (CmdTime t, StdoutTrim out, CmdLine c, Stdouterr err) <- quietly $ cmd $ "nix-build --no-out-link " <> fp-    putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"-    putVerbose $ "Output from stdout: " <> out-    putVerbose $ "Output from stderr: " <> err-    unlessM (doesDirectoryExist out) $-      fail $ "nix-build output is not a directory: " <> out-    HM.fromList <$> sequence [(f,) <$> liftIO (T.readFile $ out </> f) | f <- NE.toList files]+  addOracle $ \q@(ExtractSrcQ fetcher files) -> do+    withTempFile $ \fp -> withRetry $ do+      putInfo . show $ "#" <+> pretty q+      let nixExpr = T.unpack $ fetcherToDrv fetcher "nvfetcher-extract"+      putVerbose $ "Generated nix expr:\n" <> nixExpr+      writeFile' fp nixExpr+      (CmdTime t, StdoutTrim out, CmdLine c, Stdouterr err) <- quietly $ cmd $ "nix-build --no-out-link " <> fp+      putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"+      putVerbose $ "Output from stdout: " <> out+      putVerbose $ "Output from stderr: " <> err+      unlessM (liftIO $ IO.doesDirectoryExist out) $+        fail $+          "nix-build output is not a directory: " <> out+      buildDir <- getBuildDir+      HM.fromList+        <$> fmap+          join+          ( sequence+              [ do+                  let compiled = compileGlob glob+                  -- Find all matching files+                  paths <- liftIO $ fmap (makeRelative out) <$> (liftIO (globDir1 compiled out) >>= filterM IO.doesFileExist)+                  putVerbose $ "From glob: " <> coerce glob <> ", found file(s): " <> intercalate ", " paths+                  when (null paths) $ fail $ "No files matched glob: " <> coerce glob+                  forM paths $ \file -> do+                    -- Copy the file to the build directory under the hash of the fetcher+                    -- Also replace slashes in the hash with underscores+                    let dst = (T.unpack . T.replace "/" "_" . coerce $ _sha256 fetcher) </> file+                    copyFile' (out </> file) (buildDir </> dst)+                    pure (file, dst)+                | glob <- NE.toList files+              ]+          )  -- | Run extract source with many sources extractSrcs ::   -- | prefetched source   NixFetcher Fetched ->-  -- | relative file paths to extract-  NE.NonEmpty FilePath ->-  Action (HashMap FilePath Text)+  -- | glob patterns+  NE.NonEmpty Glob ->+  Action (HashMap FilePath FilePath) extractSrcs fetcher xs = askOracle (ExtractSrcQ fetcher xs)  -- | Run extract source extractSrc ::   -- | prefetched source   NixFetcher Fetched ->-  -- | relative file path to extract-  FilePath ->-  Action (HashMap FilePath Text)-extractSrc fetcher fp = extractSrcs fetcher $ NE.fromList [fp]+  -- | glob pattern+  Glob ->+  Action (HashMap FilePath FilePath)+extractSrc fetcher glob = extractSrcs fetcher $ NE.fromList [glob]

@@ -4,7 +4,7 @@ {-# LANGUAGE TypeApplications #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -33,32 +33,44 @@ import Data.Maybe (maybeToList) import Data.Text (Text) import qualified Data.Text as T+import qualified Data.Text.IO as T import Development.Shake+import Development.Shake.FilePath ((</>)) import NvFetcher.ExtractSrc import NvFetcher.NixFetcher import NvFetcher.Types+import NvFetcher.Types.ShakeExtras (getBuildDir) import Prettyprinter (pretty, (<+>)) import qualified TOML as Toml import Text.Parsec import Text.Parsec.Text +-- | Read extracted cargo lock content+getExtractedLock :: HashMap FilePath FilePath -> Action (FilePath, Text)+getExtractedLock result = case HMap.toList result of+  [(s, fp)] -> do+    buildDir <- getBuildDir+    content <- liftIO $ T.readFile (buildDir </> fp)+    pure (s, content)+  _ -> fail "Failed to extract cargo lock content. The size of extracted file is not 1."+ -- | Rules of fetch rust git dependencies fetchRustGitDepsRule :: Rules () fetchRustGitDepsRule = void $   addOracleCache $ \key@(FetchRustGitDepsQ fetcher lockPath) -> do     putInfo . show $ "#" <+> pretty key-    cargoLock <- head . HMap.elems <$> extractSrc fetcher lockPath+    (s, cargoLock) <- extractSrc fetcher (Glob lockPath) >>= getExtractedLock     deps <- case Toml.decodeWith (Toml.getFieldWith (Toml.getArrayOf rustDepDecoder) "package") cargoLock of       Right r -> pure $ nubOrdOn rrawSrc r-      Left err -> fail $ "Failed to parse Cargo.lock: " <> T.unpack (Toml.renderTOMLError err)+      Left err -> fail $ "Failed to parse Cargo lock file " <> s <> ": " <> T.unpack (Toml.renderTOMLError err)     r <-       parallel-        [ case parse gitSrcParser (T.unpack rname) src of+        [ case parse gitSrcParser s src of             Right ParsedGitSrc {..} -> do               (_sha256 -> sha256) <- fromMaybeM (fail $ "Prefetch failed for " <> T.unpack pgurl) $ prefetch (gitFetcher pgurl pgsha) NoForceFetch               -- @${name}-${version}@ -> sha256               pure (rname <> "-" <> coerce rversion, sha256)-            Left err -> fail $ "Failed to parse git source in Cargo.lock: " <> show err+            Left err -> fail $ "Failed to parse git source in Cargo lock file: " <> show err           | RustDep {..} <- deps,             -- it's a dependency             src <- maybeToList rrawSrc,

@@ -1,7 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -11,8 +11,9 @@ -- The cloned repo will not be preserved. module NvFetcher.GetGitCommitDate   ( -- * Types-    DateFormat (..),+    GitDateFormat (..),     GetGitCommitDate (..),+    GitTimeZone (..),      -- * Rules     getGitCommitDateRule,@@ -33,15 +34,26 @@  getGitCommitDateRule :: Rules () getGitCommitDateRule = void $ do-  addOracleCache $ \q@(GetGitCommitDate (T.unpack -> url) (T.unpack -> rev) format) -> withTempDir $ \repo -> do+  addOracleCache $ \q@(GetGitCommitDate (T.unpack -> url) (T.unpack -> rev) (coerce -> format, coerce -> tz)) -> withTempDir $ \repo -> do     putInfo . show $ "#" <+> pretty q     (StdoutTrim out) <- quietly $ do       cmd_ [Cwd repo, EchoStderr False, EchoStdout False] ("git init" :: String)       cmd_ [Cwd repo, EchoStderr False] $ "git remote add origin " <> url       cmd_ [Cwd repo, EchoStderr False] $ "git fetch --depth 1 origin " <> rev       cmd_ [Cwd repo, EchoStderr False] ("git checkout FETCH_HEAD" :: String)-      cmd [Cwd repo, Shell] $ "git --no-pager log -1 --format=%cd --date=format:\"" <> T.unpack (fromMaybe "%Y-%m-%d" $ coerce format) <> "\""+      cmd+        ( [Cwd repo, Shell]+            -- If the time zone is not local, set it in the environment+            <> [AddEnv "TZ" tz' | Just (T.unpack -> tz') <- [tz], tz' /= "local"]+        )+        $ "git --no-pager log -1 "+          <> "--format=%cd --date=format"+          -- Use --date-local instead of --date if a time zone is specified+          <> maybe "" (const "-local") tz+          <> ":\""+          <> T.unpack (fromMaybe "%Y-%m-%d" format)+          <> "\""     pure $ T.pack out -getGitCommitDate :: Text -> Text -> DateFormat -> Action Text-getGitCommitDate url rev format = askOracle $ GetGitCommitDate url rev format+getGitCommitDate :: Text -> Text -> (GitDateFormat, GitTimeZone) -> Action Text+getGitCommitDate url rev = askOracle . GetGitCommitDate url rev

@@ -7,7 +7,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -44,10 +44,10 @@   toNixExpr True = "true"   toNixExpr False = "false" -instance ToNixExpr a => ToNixExpr [a] where+instance (ToNixExpr a) => ToNixExpr [a] where   toNixExpr xs = foldl (\acc x -> acc <> " " <> toNixExpr x) "[" xs <> " ]" -instance ToNixExpr a => ToNixExpr (NE.NonEmpty a) where+instance (ToNixExpr a) => ToNixExpr (NE.NonEmpty a) where   toNixExpr = toNixExpr . NE.toList  instance {-# OVERLAPS #-} ToNixExpr String where@@ -207,14 +207,23 @@           ""           ( \ex ->               T.unlines-                [ quoteIfNeeds (T.pack name)-                    <> " = builtins.readFile "-                    <> fp+                [ quoteIfNeeds (T.pack src)+                    <> " = ./. + "+                    <> quote ("/" <> T.pack dst)                     <> ";"-                  | (name, fp) <- HMap.toList ex+                  | (src, dst) <- HMap.toList ex                 ]           )           _prextract+      extracted' =+        if T.null extract+          then ""+          else+            [trimming|+              extract = {+                $extract+              };+            |]       cargo = fromMaybe "" $ do         cargoLocks <- _prcargolock         let depsSnippet (deps :: HashMap Text Checksum) =@@ -225,11 +234,11 @@                     <> ";"                   | (name, sum) <- HMap.toList deps                 ]-            lockSnippet ((T.pack -> fp) :: FilePath, (nixFP :: NixExpr, deps :: HashMap Text Checksum)) =+            lockSnippet ((T.pack -> src) :: FilePath, ((T.pack -> dst) :: FilePath, deps :: HashMap Text Checksum)) =               let hashes = depsSnippet deps                in [trimming|-                    cargoLock."$fp" = {-                      lockFile = $nixFP;+                    cargoLock."$src" = {+                      lockFile = ./. + "/$dst";                       outputHashes = {                         $hashes                       };@@ -250,5 +259,5 @@           )           _prpassthru       date = maybe "" (\d -> "date = " <> quote d <> ";") _prgitdate-      joined = extract <> cargo <> passthru <> date+      joined = extracted' <> cargo <> passthru <> date       appending = if T.null joined then "" else "\n" <> joined

@@ -11,7 +11,7 @@ {-# LANGUAGE TypeApplications #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -55,6 +55,7 @@   ) where +import Control.Applicative ((<|>)) import Control.Exception (ErrorCall) import Control.Monad (void, when) import qualified Data.Aeson as A@@ -147,15 +148,21 @@     putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"     case A.eitherDecode out of       Right FetchedContainer {..} -> do-        sri <- sha256ToSri sha256+        sri <- sha256ToSri hash         pure FetchDocker {_sha256 = sri, _imageDigest = imageDigest, ..}       Left e -> fail $ "Failed to parse output from nix-prefetch-docker as JSON: " <> e  data FetchedContainer = FetchedContainer   { imageDigest :: ContainerDigest,-    sha256 :: Text+    hash :: Text   }-  deriving (Show, Generic, A.FromJSON)+  deriving (Show, Generic)++instance A.FromJSON FetchedContainer where+  parseJSON = A.withObject "FetchedContainer" $ \o -> do+    imageDigest <- o A..: "imageDigest"+    hash <- o A..: "sha256" <|> o A..: "hash"+    pure FetchedContainer {..}  pypiUrl :: Text -> Version -> Text pypiUrl pypi (coerce -> ver) =

@@ -3,7 +3,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -105,7 +105,7 @@ runNvchecker pkg options versionSource = withTempFile $ \config -> withRetry $ do   mKeyfile <- getKeyfilePath   let nvcheckerConfig = T.unpack $ T.unlines $ execWriter $ genNvConfig pkg options mKeyfile versionSource-  putVerbose $ "Generated nvchecker config for " <> show pkg <> ":" <> nvcheckerConfig+  putVerbose $ "Generated nvchecker config for " <> show pkg <> ":\n" <> nvcheckerConfig   writeFile' config nvcheckerConfig   (CmdTime t, Stdout out, CmdLine c) <- quietly . cmd $ "nvchecker --logger json -c " <> config   putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"@@ -131,7 +131,7 @@           genOptions options       )   where-    key =: x = tell [key <> " = " <> quote x]+    key =: x = tell [key <> " = " <> quoteAndEscape x]     key =:? (Just x) = key =: x     _ =:? _ = pure ()     table t m = tell ["[" <> quote t <> "]"] >> m >> tell [""]

@@ -1,7 +1,7 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE TemplateHaskell #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental

@@ -13,7 +13,7 @@ {-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -82,11 +82,12 @@      -- * Addons     extractSource,-    hasCargoLocks,+    hasCargoLock,     tweakVersion,     passthru,     pinned,     gitDateFormat,+    gitTimeZone,     forceFetch,      -- ** Miscellaneous@@ -160,11 +161,11 @@   Maybe PackageCargoLockFiles ->   PackagePassthru ->   UseStaleVersion ->-  DateFormat ->+  (GitDateFormat, GitTimeZone) ->   ForceFetch ->   PackageSet ()-newPackage name source fetcher extract cargo pasthru useStale format force =-  liftF $ NewPackage (Package name source fetcher extract cargo pasthru useStale format force) ()+newPackage name source fetcher extract cargo pasthru useStale (format, tz) force =+  liftF $ NewPackage (Package name source fetcher extract cargo pasthru useStale (format, tz) force) ()  -- | Add a list of packages into package set purePackageSet :: [Package] -> PackageSet ()@@ -263,7 +264,8 @@            NvcheckerOptions,            PackagePassthru,            UseStaleVersion,-           DateFormat,+           GitDateFormat,+           GitTimeZone,            ForceFetch          ]         r@@ -289,7 +291,7 @@       (projMaybe p)       (fromMaybe mempty (projMaybe p))       (fromMaybe NoStale (projMaybe p))-      (fromMaybe (DateFormat Nothing) (projMaybe p))+      (fromMaybe (GitDateFormat Nothing) (projMaybe p), fromMaybe (GitTimeZone Nothing) (projMaybe p))       (fromMaybe NoForceFetch (projMaybe p))  -- | 'PkgDSL' version of 'newPackage'@@ -312,7 +314,8 @@          PackagePassthru,          NvcheckerOptions,          UseStaleVersion,-         DateFormat,+         GitDateFormat,+         GitTimeZone,          ForceFetch        ]       r@@ -543,14 +546,14 @@ --------------------------------------------------------------------------------  -- | Extract files from fetched package source-extractSource :: Attach PackageExtractSrc [FilePath]+extractSource :: Attach PackageExtractSrc [Glob] extractSource = (. pure . PackageExtractSrc . NE.fromList) . andThen  -- | Run 'FetchRustGitDependencies' given the path to @Cargo.lock@ files -- -- The lock files will be extracted as well.-hasCargoLocks :: Attach PackageCargoLockFiles [FilePath]-hasCargoLocks = (. pure . PackageCargoLockFiles . NE.fromList) . andThen+hasCargoLock :: Attach PackageCargoLockFiles [Glob]+hasCargoLock = (. pure . PackageCargoLockFiles . NE.fromList) . andThen  -- | Set 'NvcheckerOptions' for a package, which can tweak the version number we obtain tweakVersion :: Attach NvcheckerOptions (NvcheckerOptions -> NvcheckerOptions)@@ -571,8 +574,14 @@ -- | Specify the date format for getting git commit date -- -- Available only for git version source-gitDateFormat :: Attach DateFormat (Maybe Text)-gitDateFormat = (. pure . DateFormat) . andThen+gitDateFormat :: Attach GitDateFormat (Maybe Text)+gitDateFormat = (. pure . GitDateFormat) . andThen++-- | Specify the time zone for getting git commit date+--+-- Available only for git version source+gitTimeZone :: Attach GitTimeZone (Maybe Text)+gitTimeZone = (. pure . GitTimeZone) . andThen  -- | Set always fetching regardless of the version changing forceFetch :: PackageSet (Prod r) -> PackageSet (Prod (ForceFetch : r))

@@ -2,21 +2,23 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE DeriveAnyClass #-} {-# LANGUAGE DeriveGeneric #-}-{-# LANGUAGE DerivingStrategies #-}+{-# LANGUAGE DerivingVia #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE PolyKinds #-} {-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE StandaloneDeriving #-} {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE TypeOperators #-} {-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE UndecidableSuperClasses #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -32,6 +34,7 @@     NixExpr,     VersionChange (..),     WithPackageKey (..),+    Glob (..),      -- * Nvchecker types     VersionSortMethod (..),@@ -57,8 +60,9 @@     FetchRustGitDepsQ (..),      -- * GetGitCommitDate types-    DateFormat (..),+    GitDateFormat (..),     GetGitCommitDate (..),+    GitTimeZone (..),      -- * Core types     Core (..),@@ -81,16 +85,30 @@ import Data.HashMap.Strict (HashMap) import qualified Data.List.NonEmpty as NE import Data.Maybe (fromMaybe, isNothing)+import Data.Proxy (Proxy (..)) import Data.String (IsString) import Data.Text (Text) import qualified Data.Text as T import Development.Shake import Development.Shake.Classes import GHC.Generics (Generic)+import GHC.TypeLits (KnownSymbol, Symbol, symbolVal) import Prettyprinter  -------------------------------------------------------------------------------- +-- | Helper type for generating 'Show' and 'Pretty' instances+-- Type level string @d@ denotes the string used for the default value 'Nothing'+newtype DefaultableText (d :: Symbol) = DefaultableText (Maybe Text)++instance (KnownSymbol d) => Show (DefaultableText d) where+  show (DefaultableText Nothing) = "default (" <> symbolVal (Proxy :: Proxy d) <> ")"+  show (DefaultableText x) = show x++instance (KnownSymbol d) => Pretty (DefaultableText d) where+  pretty (DefaultableText Nothing) = pretty $ "default (" <> symbolVal (Proxy :: Proxy d) <> ")"+  pretty (DefaultableText (Just x)) = pretty x+ -- | Package version newtype Version = Version Text   deriving newtype (Eq, Show, Ord, IsString, Semigroup, Monoid, A.FromJSON, A.ToJSON, Pretty)@@ -111,9 +129,10 @@  -- | Git branch ('Nothing': master) newtype Branch = Branch (Maybe Text)-  deriving newtype (Show, Eq, Ord, Default, Pretty)+  deriving newtype (Eq, Ord, Default)   deriving stock (Typeable, Generic)   deriving anyclass (Hashable, Binary, NFData)+  deriving (Pretty, Show) via DefaultableText "master"  -- | Version change of a package --@@ -138,17 +157,19 @@  -------------------------------------------------------------------------------- -data VersionSortMethod = ParseVersion | Vercmp+data VersionSortMethod = ParseVersion | Vercmp | AwesomeVersion   deriving (Typeable, Eq, Ord, Enum, Generic, Hashable, Binary, NFData)  instance Show VersionSortMethod where   show = \case     ParseVersion -> "parse_version"     Vercmp -> "vercmp"+    AwesomeVersion -> "awesomeversion"  instance Pretty VersionSortMethod where   pretty ParseVersion = "ParseVersion"   pretty Vercmp = "Vercmp"+  pretty AwesomeVersion = "AwesomeVersion"  instance Default VersionSortMethod where   def = ParseVersion@@ -213,7 +234,7 @@               ]         ) -ppField :: Pretty a => Doc ann -> Maybe a -> [Doc ann]+ppField :: (Pretty a) => Doc ann -> Maybe a -> [Doc ann] ppField _ Nothing = [] ppField s (Just x) = [s <> colon <+> pretty x] @@ -435,17 +456,17 @@  instance (c (FetchResult Checksum k), c (FetchResult ContainerDigest k)) => ForFetchResult c k -deriving instance Show `ForFetchResult` k => Show (NixFetcher k)+deriving instance (Show `ForFetchResult` k) => Show (NixFetcher k) -deriving instance Eq `ForFetchResult` k => Eq (NixFetcher k)+deriving instance (Eq `ForFetchResult` k) => Eq (NixFetcher k) -deriving instance Ord `ForFetchResult` k => Ord (NixFetcher k)+deriving instance (Ord `ForFetchResult` k) => Ord (NixFetcher k) -deriving instance Hashable `ForFetchResult` k => Hashable (NixFetcher k)+deriving instance (Hashable `ForFetchResult` k) => Hashable (NixFetcher k) -deriving instance Binary `ForFetchResult` k => Binary (NixFetcher k)+deriving instance (Binary `ForFetchResult` k) => Binary (NixFetcher k) -deriving instance NFData `ForFetchResult` k => NFData (NixFetcher k)+deriving instance (NFData `ForFetchResult` k) => NFData (NixFetcher k)  -- | Fetch status data FetchStatus = Fresh | Fetched@@ -568,12 +589,24 @@  -------------------------------------------------------------------------------- +-- | Zsh style glob pattern+-- Notably, recursive wildcards like @**/@ are supported.+newtype Glob = Glob FilePath+  deriving newtype (Show, Eq, Ord, IsString, Pretty)+  deriving stock (Typeable, Generic)+  deriving anyclass (Hashable, Binary, NFData)+ -- | Extract file contents from package source--- e.g. @Cargo.lock@-data ExtractSrcQ = ExtractSrcQ (NixFetcher Fetched) (NE.NonEmpty FilePath)+-- Matched files will be copied to build dir.+-- All matched directories are ignored. Only files will be processed.+data ExtractSrcQ = ExtractSrcQ (NixFetcher Fetched) (NE.NonEmpty Glob)   deriving (Show, Eq, Ord, Hashable, NFData, Binary, Typeable, Generic) -type instance RuleResult ExtractSrcQ = HashMap FilePath Text+-- | Rule result for extracting source files. 'Text' is the file contents,+-- only available if the second element of the tuple in 'ExtractSrcQ' is @True@.+-- The key of the result map is the file path relative to the package source,+-- and the value is the file path relative to the build directory.+type instance RuleResult ExtractSrcQ = HashMap FilePath FilePath  instance Pretty ExtractSrcQ where   pretty (ExtractSrcQ f n) =@@ -613,18 +646,31 @@ --------------------------------------------------------------------------------  -- | @strftime@ format------ Nothing defaults to @%Y-%m-%d@-newtype DateFormat = DateFormat (Maybe Text)-  deriving newtype (Show, Eq, Ord, Default, Pretty)+-- Defaults to @%Y-%m-%d@+newtype GitDateFormat = GitDateFormat (Maybe Text)+  deriving newtype (Eq, Ord, Default)   deriving stock (Typeable, Generic)   deriving anyclass (Hashable, Binary, NFData)+  deriving (Pretty, Show) via DefaultableText "%Y-%m-%d" +-- | Defaults to commit's time zone.+-- When set to @local@, current local time zone is used.+-- Only used in 'GetGitCommitDate'.+newtype GitTimeZone = GitTimeZone (Maybe Text)+  deriving newtype (Eq, Ord, Default)+  deriving stock (Typeable, Generic)+  deriving anyclass (Hashable, Binary, NFData)+  deriving (Pretty, Show) via DefaultableText "commit's time zone"+ -- | Get the commit date by using shallow clone -- -- @_gformat@ is in. -- Note: Requires git >= 2.5-data GetGitCommitDate = GetGitCommitDate {_gurl :: Text, _grev :: Text, _gformat :: DateFormat}+data GetGitCommitDate = GetGitCommitDate+  { _gurl :: Text,+    _grev :: Text,+    _gformat :: (GitDateFormat, GitTimeZone)+  }   deriving (Show, Eq, Ord, Hashable, NFData, Binary, Typeable, Generic)  type instance RuleResult GetGitCommitDate = Text@@ -650,9 +696,9 @@ -- | How to create package source fetcher given a version type PackageFetcher = Version -> NixFetcher Fresh -newtype PackageExtractSrc = PackageExtractSrc (NE.NonEmpty FilePath)+newtype PackageExtractSrc = PackageExtractSrc (NE.NonEmpty Glob) -newtype PackageCargoLockFiles = PackageCargoLockFiles (NE.NonEmpty FilePath)+newtype PackageCargoLockFiles = PackageCargoLockFiles (NE.NonEmpty Glob)  newtype PackagePassthru = PackagePassthru (HashMap Text Text)   deriving newtype (Semigroup, Monoid)@@ -676,10 +722,10 @@ -- 2. how to track its version -- 3. how to fetch it as we have the version -- 4. optional file paths to extract (dump to build dir)--- 5. optional @Cargo.lock@ path (if it's a rust package)+-- 5. optional @Cargo.lock@ paths (if it's a rust package) -- 6. an optional pass through map -- 7. if the package version was pinned--- 8. optional git date format (if the version source is git)+-- 8. optional git date format with time zone (if the version source is git) -- 9. whether to always fetch a package regardless of the version changing -- /INVARIANT: 'Version' passed to 'PackageFetcher' MUST be used textually,/ -- /i.e. can only be concatenated with other strings,/@@ -692,7 +738,7 @@     _pcargo :: Maybe PackageCargoLockFiles,     _ppassthru :: PackagePassthru,     _ppinned :: UseStaleVersion,-    _pgitdateformat :: DateFormat,+    _pgitdate :: (GitDateFormat, GitTimeZone),     _pforcefetch :: ForceFetch   } @@ -716,7 +762,7 @@ newtype WithPackageKey k = WithPackageKey (k, PackageKey)   deriving newtype (Eq, Hashable, Binary, NFData) -instance Show k => Show (WithPackageKey k) where+instance (Show k) => Show (WithPackageKey k) where   show (WithPackageKey (k, n)) = show k <> " (" <> show n <> ")"  type instance RuleResult (WithPackageKey k) = RuleResult k@@ -727,10 +773,10 @@     _prversion :: NvcheckerResult,     _prfetched :: NixFetcher 'Fetched,     _prpassthru :: Maybe (HashMap Text Text),-    -- | extracted file name -> file path in build dir-    _prextract :: Maybe (HashMap FilePath NixExpr),-    -- | cargo lock file path in build dir -> (file path in nix, git dependencies)-    _prcargolock :: Maybe (HashMap FilePath (NixExpr, HashMap Text Checksum)),+    -- | file path relative to package source -> file path relative to build dir+    _prextract :: Maybe (HashMap FilePath FilePath),+    -- | cargo lock file path relative to package source -> (lock file path relative to build dir, git dependencies)+    _prcargolock :: Maybe (HashMap FilePath (FilePath, HashMap Text Checksum)),     _prpinned :: UseStaleVersion,     _prgitdate :: Maybe Text   }@@ -744,7 +790,7 @@         "src" A..= _prfetched,         "extract" A..= _prextract,         "passthru" A..= _prpassthru,-        "cargoLocks" A..= _prcargolock,+        "cargoLock" A..= _prcargolock,         "pinned" A..= case _prpinned of           PermanentStale -> True           _ -> False,

@@ -1,7 +1,7 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE TemplateHaskell #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental

@@ -2,7 +2,7 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE TypeApplications #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental

@@ -1,6 +1,6 @@ {-# LANGUAGE CPP #-} --- | Copyright: (c) 2021-2022 berberman+-- | Copyright: (c) 2021-2025 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -12,21 +12,29 @@ import qualified Data.ByteString.Lazy as LBS import Data.Text (Text) import qualified Data.Text as T-import System.Directory.Extra (XdgDirectory (XdgData), getXdgDirectory)+import NvFetcher.Types+import System.Directory.Extra (XdgDirectory (..), getXdgDirectory)+import System.FilePath.Glob (Pattern, compile) import Text.Regex.TDFA ((=~))+ #if MIN_VERSION_aeson(2,0,0) import qualified Data.Aeson.Key as A #endif -encode' :: Binary a => a -> BS.ByteString+encode' :: (Binary a) => a -> BS.ByteString encode' = BS.concat . LBS.toChunks . encode -decode' :: Binary a => BS.ByteString -> a+decode' :: (Binary a) => BS.ByteString -> a decode' = decode . LBS.fromChunks . pure  quote :: Text -> Text-quote = T.pack . show+quote x = T.concat [q, x, q]+  where+    q = T.pack "\"" +quoteAndEscape :: Text -> Text+quoteAndEscape = T.pack . show+ isLegalNixId :: Text -> Bool isLegalNixId x = x =~ "^[a-zA-Z_][a-zA-Z0-9_'-]*$" @@ -37,6 +45,11 @@  getDataDir :: IO FilePath getDataDir = getXdgDirectory XdgData "nvfetcher"++-- | Compile a 'Glob' pattern into a 'Pattern'.+-- See <https://hackage.haskell.org/package/Glob/docs/System-FilePath-Glob.html#v:compile> for details.+compileGlob :: Glob -> Pattern+compileGlob (Glob txt) = compile txt  #if MIN_VERSION_aeson(2,0,0) aesonKey :: Text -> A.Key

@@ -30,7 +30,7 @@       runNvcheckerRule (Pypi "example") `shouldReturnJust` Version "0.1.0"      specifyChan "archpkg" $-      runNvcheckerRule (ArchLinux "lib32-libva-vdpau-driver") `shouldReturnJust` Version "0.7.4"+      runNvcheckerRule (ArchLinux "xfce4-artwork") `shouldReturnJust` Version "0.1.1a_git20110420"      specifyChan "aur" $       runNvcheckerRule (Aur "ssed") `shouldReturnJust` Version "3.62"@@ -76,7 +76,7 @@       runNvcheckerRule (Cmd "echo Meow") `shouldReturnJust` Version "Meow"      specifyChan "container" $-      runNvcheckerRule (Container "testcontainers/helloworld" def) `shouldReturnJust` Version "1.1.0"+      runNvcheckerRule (Container "testcontainers/helloworld" def) `shouldReturnJust` Version "1.3.0"  -------------------------------------------------------------------------------- @@ -136,7 +136,7 @@       _pextract = undefined,       _ppassthru = undefined,       _ppinned = NoStale,-      _pgitdateformat = undefined,+      _pgitdate = undefined,       _pforcefetch = undefined     } 

@@ -0,0 +1,73 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ViewPatterns #-}++module ExtractSpec where++import Control.Monad.Trans.Reader+import qualified Data.HashMap.Strict as HMap+import qualified Data.List.NonEmpty as NE+import Data.Maybe (fromJust)+import Development.Shake (liftIO)+import Development.Shake.FilePath ((</>))+import FetchRustGitDepsSpec (runPrefetchRule)+import NvFetcher (getBuildDir)+import NvFetcher.ExtractSrc+import NvFetcher.NixFetcher+import NvFetcher.Types+import qualified System.Directory.Extra as IO+import Test.Hspec+import Utils++spec :: Spec+spec = aroundShake $ do+  describe "extract" $ do+    let fetcher = gitHubFetcher ("berberman", "nvfetcher") $ Version "0.7.0"+        srcSha256 = "sha256-GVjVivAz2AW7WNqpWTA5Uy9vN8Wh_3pjuZRFhsGVk24="+        mkExpectedMap files = HMap.fromList [(f, srcSha256 </> f) | f <- files]+        mkExpectedPaths buildDir files = [buildDir </> srcSha256 </> f | f <- files]+        allExist = mapM_ shouldExist+    specifyChan "extracts single file" $ do+      prefetched <- runPrefetchRule fetcher+      shouldBeJust prefetched+      let expectedFiles = ["Setup.hs"]+      runExtractSrcsRule (fromJust prefetched) (NE.singleton "Setup.hs")+        `shouldReturnJust` mkExpectedMap expectedFiles+      (fromJust -> buildDir) <- runActionChan getBuildDir+      allExist $ mkExpectedPaths buildDir expectedFiles+    specifyChan "extracts files" $ do+      prefetched <- runPrefetchRule fetcher+      shouldBeJust prefetched+      let expectedFiles =+            [ "default.nix",+              "CHANGELOG.md",+              "README.md",+              "flake.nix",+              "nix/default.nix"+            ]+      runExtractSrcsRule (fromJust prefetched) (NE.fromList ["**/*.nix", "*.md"])+        `shouldReturnJust` mkExpectedMap expectedFiles+      (fromJust -> buildDir) <- runActionChan getBuildDir+      allExist $ mkExpectedPaths buildDir expectedFiles+    specifyChan "extracts rerun" $ do+      prefetched <- runPrefetchRule fetcher+      shouldBeJust prefetched+      let expectedFiles = ["Setup.hs"]+      runExtractSrcsRule (fromJust prefetched) (NE.singleton "Setup.hs")+        `shouldReturnJust` mkExpectedMap expectedFiles+      (fromJust -> buildDir) <- runActionChan getBuildDir+      let path = case mkExpectedPaths buildDir expectedFiles of+            [] -> error "Impossible"+            (p : _) -> p+      allExist [path]+      -- Delete this generated file and rerun the rule+      -- We should see the file being regenerated+      liftIO $ IO.removeFile path+      -- Run again+      runExtractSrcsRule (fromJust prefetched) (NE.singleton "Setup.hs")+        `shouldReturnJust` mkExpectedMap expectedFiles+      -- Check that the file has been regenerated+      allExist [path]++runExtractSrcsRule :: NixFetcher Fetched -> NE.NonEmpty Glob -> ReaderT ActionQueue IO (Maybe (HMap.HashMap FilePath FilePath))+runExtractSrcsRule fetcher globs = runActionChan $ extractSrcs fetcher globs

@@ -1,10 +1,15 @@ {-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE ScopedTypeVariables #-}  module GetGitCommitDateSpec where +import Control.Monad.IO.Class (liftIO) import Control.Monad.Trans.Reader import Data.Default (def)+import Data.Maybe (fromJust) import Data.Text (Text)+import qualified Data.Text as T+import Data.Time (UTCTime, defaultTimeLocale, formatTime, getCurrentTimeZone, parseTimeM, utcToLocalTime) import NvFetcher.GetGitCommitDate import Test.Hspec import Utils@@ -17,14 +22,37 @@         runGetGitCommitDateRule repo rev def           `shouldReturnJust` "2021-05-31"       specifyChan "custom format" $-        runGetGitCommitDateRule repo rev (DateFormat $ Just "%Y-%m-%d %H:%M:%S")+        runGetGitCommitDateRule repo rev (GitDateFormat $ Just "%Y-%m-%d %H:%M:%S", def)           `shouldReturnJust` "2021-05-31 18:43:48"+      specifyChan "UTC time zone" $+        runGetGitCommitDateRule repo rev (GitDateFormat $ Just "%H:%M:%S", GitTimeZone $ Just "UTC")+          -- The commit was made at UTC+8; hence we expect the time to be 8 hours earlier+          `shouldReturnJust` "10:43:48" +      -- TODO+      -- Test commented out as this logic does not work:+      -- The git command with --date=local returns DST time zone if the commit time falls within DST, +      -- but our conversion always uses current local time zone, which may not match git's output.+      -- It's not clear how to check whether DST applies for a given time zone at a specific date/time in Haskell.+      --+      -- specifyChan "local time zone" $ do+      --   -- Get the current time zone+      --   tz <- liftIO getCurrentTimeZone+      --   -- Get commit time at UTC+      --   rawCommitTime <- runGetGitCommitDateRule repo rev (GitDateFormat $ Just "%Y-%m-%d %H:%M:%S", GitTimeZone $ Just "UTC")+      --   shouldBeJust rawCommitTime+      --   parsedCommitTime :: UTCTime <- parseTimeM True defaultTimeLocale "%Y-%m-%d %H:%M:%S" . T.unpack . fromJust $ rawCommitTime+      --   -- Convert UTC time to local time+      --   let localTimeStr = T.pack $ formatTime defaultTimeLocale "%H:%M:%S" $ utcToLocalTime tz parsedCommitTime+      --   -- Rule should return the same result as we computed+      --   runGetGitCommitDateRule repo rev (GitDateFormat $ Just "%H:%M:%S", GitTimeZone $ Just "local")+      --     `shouldReturnJust` localTimeStr+ repo :: Text repo = "https://gist.github.com/NickCao/6c4dbc4e15db5da107de6cdb89578375"  rev :: Text rev = "8a5f37a8f80a3b05290707febf57e88661cee442" -runGetGitCommitDateRule :: Text -> Text -> DateFormat -> ReaderT ActionQueue IO (Maybe Text)-runGetGitCommitDateRule url rev format = runActionChan $ getGitCommitDate url rev format+runGetGitCommitDateRule :: Text -> Text -> (GitDateFormat, GitTimeZone) -> ReaderT ActionQueue IO (Maybe Text)+runGetGitCommitDateRule url rev (format, tz) = runActionChan $ getGitCommitDate url rev (format, tz)

@@ -47,10 +47,10 @@     specifyChan "url with name" $       runPrefetchRule         ( urlFetcher'-            "https://files.yande.re/image/3fc51f6a2fb10c96b73dd0fce6ddb9c8/yande.re%201048717%20dress%20garter%20lolita_fashion%20ruo_gan_zhua.jpg"-            (Just "foo.jpg")+            "https://whatthecommit.com/6f7a23353b95ca5981bf7b685c278d47.txt"+            (Just "foo.txt")         )-        `shouldReturnJust` Checksum "sha256-wkiXDN6vPFtx88krcQ4szK6dJNjtrDxrsNa3ZvHlfMQ="+        `shouldReturnJust` Checksum "sha256-jFxm3Zts3ZkgwEJmpHk3cocszzVm2umfhRJCFNam2GA="  testDockerFetcher :: NixFetcher Fresh testDockerFetcher =

@@ -21,6 +21,7 @@ import NvFetcher.Core (coreRules) import NvFetcher.Types import NvFetcher.Types.ShakeExtras+import qualified System.Directory.Extra as Extra import qualified System.IO.Extra as Extra import System.Time.Extra import Test.Hspec@@ -82,7 +83,10 @@ shouldBeJust :: (MonadIO m, Show a) => Maybe a -> m () shouldBeJust x = liftIO $ x `shouldSatisfy` isJust -specifyChan :: HasCallStack => String -> ReaderT ActionQueue IO () -> SpecWith ActionQueue+shouldExist :: (MonadIO m) => FilePath -> m ()+shouldExist path = liftIO $ Extra.doesFileExist path `shouldReturn` True++specifyChan :: (HasCallStack) => String -> ReaderT ActionQueue IO () -> SpecWith ActionQueue specifyChan s m = specify s $ \r -> runReaderT m r  runActionChan :: Action a -> ReaderT ActionQueue IO (Maybe a)