nvfetcher 0.2.0.0 → 0.3.0.0
raw patch · 22 files changed
+1406/−556 lines, 22 filesdep +binary-instancesdep +data-defaultdep +parsecdep ~optparse-simple
Dependencies added: binary-instances, data-default, parsec, unordered-containers
Dependency ranges changed: optparse-simple
Files
- CHANGELOG.md +23/−0
- Main_example.hs +17/−9
- README.md +76/−24
- app/Config.hs +23/−149
- app/Config/PackageFetcher.hs +102/−0
- app/Config/VersionSource.hs +177/−0
- app/Main.hs +11/−92
- nvfetcher.cabal +30/−17
- src/NvFetcher.hs +61/−11
- src/NvFetcher/Core.hs +79/−18
- src/NvFetcher/ExtractSrc.hs +69/−0
- src/NvFetcher/FetchRustGitDeps.hs +109/−0
- src/NvFetcher/NixExpr.hs +102/−0
- src/NvFetcher/NixFetcher.hs +8/−56
- src/NvFetcher/Nvchecker.hs +73/−59
- src/NvFetcher/Options.hs +98/−0
- src/NvFetcher/PackageSet.hs +125/−7
- src/NvFetcher/ShakeExtras.hs +0/−80
- src/NvFetcher/Types.hs +121/−34
- src/NvFetcher/Types/Lens.hs +5/−0
- src/NvFetcher/Types/ShakeExtras.hs +88/−0
- src/NvFetcher/Utils.hs +9/−0
CHANGELOG.md view
@@ -1,5 +1,28 @@ # Revision history for nvfetcher +## 0.3.0.0++There are massive enhancements since the last release:++* Add support for nvchecker [list options](https://nvchecker.readthedocs.io/en/latest/usage.html#list-options)+* Refactor TOML config parsing+* Remove version specification in fetcher config (`fetch.url = url:version` -> `fetch.url = url`)+* Add support for calculating [`cargoLock`](https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/rust.section.md#importing-a-cargolock-file) for `rustPlatform.buildRustPackage`+* Add support for nvchecker [global options](https://nvchecker.readthedocs.io/en/latest/usage.html#global-options)+* Remove ambiguous branch specification (`git.branch`) from git fetcher+* Enable parallelism by default+* Add a global retry option+* Rename `.shake` to `_build`+* Generate nix output file in `_build`, and symlink it to `../sources.nix` (You have to keep `_build` as the `nvfetcher` run result)+* Support extracting arbitrary files from fetched package source+* Add nvchecker upstream sources [`src.webpage`](https://nvchecker.readthedocs.io/en/latest/usage.html#search-in-a-webpage) and [`src.httpheader`](https://nvchecker.readthedocs.io/en/latest/usage.html#search-in-an-http-header)+* Add nvchecker upstream source `src.github_tag`+* Share CLI between `runNvfetcher` (use `nvfetcher` in the DSL way) and `nvfetcher` executable program +* Nix related improvements:+ * Add a development shell `ghcWithNvfetcher` for people who want to use `nvfetcher` as a Haskell library+ * Generate command line completion for the executable+ + ## 0.2.0.0 * Generated package sources will be sorted alphabetically.
Main_example.hs view
@@ -5,23 +5,19 @@ import NvFetcher main :: IO ()-main = runNvFetcher defaultArgs packageSet+main = runNvFetcher packageSet packageSet :: PackageSet () packageSet = do+ define $ package "fd" `fromGitHub` ("sharkdp", "fd") `extractSource` ["Cargo.lock"]++ define $ package "gcc-10" `fromGitHubTag` ("gcc-mirror", "gcc", includeRegex ?~ "releases/gcc-10.*")+ define $ package "feeluown-core" `fromPypi` "feeluown" define $ package "qliveplayer" `fromGitHub'` ("IsoaSFlus", "QLivePlayer", fetchSubmodules .~ True) define $- package "fcitx5-pinyin-zhwiki"- `sourceAur` "fcitx5-pinyin-zhwiki"- `fetchUrl` \v ->- "https://github.com/felixonmars/fcitx5-pinyin-zhwiki/releases/download/0.2.2/zhwiki-"- <> coerce v- <> ".dict"-- define $ package "apple-emoji" `sourceManual` "0.0.0.20200413" `fetchUrl` const@@ -31,3 +27,15 @@ package "nvfetcher-git" `sourceGit` "https://github.com/berberman/nvfetcher" `fetchGitHub` ("berberman", "nvfetcher")++ define $+ package "vim"+ `sourceWebpage` ("http://ftp.vim.org/pub/vim/patches/7.3/", "7\\.3\\.\\d+", id)+ `fetchGitHub` ("vim", "vim")+ `tweakVersion` (\v -> v & fromPattern ?~ "(.+)" & toPattern ?~ "v\\1")++ define $+ package "rust-git-dependency-example"+ `sourceManual` "8a5f37a8f80a3b05290707febf57e88661cee442"+ `fetchGit` "https://gist.github.com/NickCao/6c4dbc4e15db5da107de6cdb89578375"+ `hasCargoLock` "Cargo.lock"
README.md view
@@ -6,8 +6,6 @@ nvfetcher is a tool to automate packages updates in flakes repos. It's built on top of [shake](https://www.shakebuild.com/), integrating [nvchecker](https://github.com/lilydjwg/nvchecker).-It's very simple -- most complicated works are done by nvchecker, nvfetcher just wires it with prefetch tools,-producing only one artifact as the result of build. nvfetcher cli program accepts a TOML file as config, which defines a set of package sources to run. ## Overview@@ -59,6 +57,10 @@ and nvfetcher will help us keep their version and prefetched SHA256 sums up-to-date, stored in `sources.nix`. Shake will help us handle necessary rebuilds -- we check versions of packages during each run, but only prefetch them when needed. ++`sources.nix` is a symlink of `_build/generated.nix`, which means that the "build" results produced by `nvfetcher` are under `_build`,+so you have to keep this directory for further use.+ ### Live examples How to use the generated sources file? Here are some examples:@@ -82,7 +84,15 @@ ``` To use it as a Haskell library, the package is available on [Hackage](https://hackage.haskell.org/package/nvfetcher).+If you want to use the Haskell library from flakes, there is also a shell `ghcWithNvfetcher`: +```+$ nix develop github:berberman/nvfetcher#ghcWithNvfetcher+$ runghc Main.hs+```++where you can define packages in `Main.hs`. See [Haskell library](#Haskell-library) for details.+ ## Usage Basically, there are two ways to use nvfetcher, where the difference is how we provide package sources definitions to it.@@ -101,35 +111,77 @@ -l,--changelog FILE Dump version changes to a file -j NUM Number of threads (0: detected number of processors) (default: 0)+ -r,--retry NUM Times to retry of some rules (nvchecker, prefetch,+ nix-instantiate, etc.) (default: 3) -t,--timing Show build time -v,--verbose Verbose mode- TARGET Two targets are available: build and clean+ TARGET Two targets are available: 1.build 2.clean+ (default: "build") ``` -Each *package* corresponds to a TOML table, whose name is encoded as table key;-there are two fields and four optional git prefetch configuration in each table:-* a nvchecker configuration, how to track version updates- * `src.github = owner/repo` - the latest gituhb release- * `src.pypi = pypi_name` - the latest pypi release- * `src.git = git_url` - the latest commit of a repo- * `src.archpkg = archlinux_pkg_name` -- the latest version of an archlinux package- * `src.aur = aur_pkg_name` -- the latest version of an aur package- * `src.manual = v` -- a fixed version, which never updates- * `src.repology = project:repo` -- the latest version from repology-* a nix fetcher function, how to fetch the package given the version number. `$ver` is available, which will be set to the result of nvchecker.- * `fetch.github = owner/repo` or `owner/repo:rev` (default to `$ver` if no `rev` specified)- * `fetch.pypi = pypi_name` or `pypi_name:ver` (default to `$ver` if no `ver` specified)- * `fetch.git = git_url` or `git_url:rev` (default to `$ver` if no `rev` specified)- * `fetch.url = url`+Each *package* corresponds to a TOML table, whose name is encoded as table key, with+two required fields and three optional fields in each table.+You can find an example of the configuration file, see [`nvfetcher_example.toml`](nvfetcher_example.toml). -* optional git prefetch configuration, which makes sense only when the fetcher equals to `fetch.github` or `fetch.git`.+#### Nvchecker++Version source -- how do we track upstream version updates?+* `src.github = owner/repo` - the latest gituhb release+* `src.github_tag = owner/repo` - the max github tag, usually used with list options (see below)+* `src.pypi = pypi_name` - the latest pypi release+* `src.git = git_url` (and an optional `src.branch = git_branch`) - **the latest commit** of a repo+* `src.archpkg = archlinux_pkg_name` -- the latest version of an archlinux package+* `src.aur = aur_pkg_name` -- the latest version of an aur package+* `src.manual = v` -- a fixed version, which never updates+* `src.repology = project:repo` -- the latest version from repology+* `src.webpage = web_url` and `src.regex` -- a string in webpage that matches with regex+* `src.httpheader = request_url` and `src.regex` -- a string in http header that matches with regex+++Optional list options for some version sources (`src.github_tag`, `src.webpage`, and `src.httpheader`),+see the corresponding [nvchecker documentation](https://nvchecker.readthedocs.io/en/latest/usage.html#list-options) for details.+* `src.include_regex`+* `src.exclude_regex`+* `src.sort_version_key`+* `src.ignored`+++Optional global options for all kinds of version sources,+see the corresponding [nvchecker documentation](https://nvchecker.readthedocs.io/en/latest/usage.html#global-options) for details. You can tweak obtained version number using this option, e.g. stripping the prefix `v` or transforming the result by regex.+* `src.prefix`+* `src.from_pattern`+* `src.to_pattern`++#### Nix fetcher++How do we fetch the package source if we have the target version number?+`$ver` is available in string, which will be set to the result of nvchecker.++* `fetch.github = owner/repo`+* `fetch.pypi = pypi_name`+* `fetch.git = git_url`+* `fetch.url = url`+++Optional `nix-prefetch-git` config, which make sense only when the fetcher equals to `fetch.github` or `fetch.git`. They can exist simultanesouly.- * `git.branch = branch_name` - branch to fetch- * `git.deepClone` - a bool value to control deep clone- * `git.fetchSubmodules` - a bool value to control fetching submodules- * `git.leaveDotGit` - a bool value to control leaving dot git+ * `git.deepClone`+ * `git.fetchSubmodules`+ * `git.leaveDotGit` -You can find an example of the configuration file, see [`nvfetcher_example.toml`](nvfetcher_example.toml).++#### Extract src++Optional *extract src* config, files are extracted into build directory, and then read by `readFile` in generated nix expr.+ * `extract = [ "file_1", "file_2", ...]` - file paths are relative to the source root++#### Rust support++`rustPlatform.buildRustPackage` now accepts an attribute [`cargoLock`](https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/rust.section.md#importing-a-cargolock-file) to vendor dependencies from `Cargo.lock`,+so we can use this instead TOFU `cargoSha256` for Rust packageing. `nvfetcher` supports automating this process,+extracting the lock file to build and calculating `cargoLock.outputHashes`, as long as you set the config+* `cargo_lock = cargo_lock_path` - relative to the source root+ ### Haskell library
app/Config.hs view
@@ -1,4 +1,6 @@ {-# LANGUAGE DataKinds #-}+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-}@@ -6,18 +8,14 @@ module Config where -import Control.Applicative ((<|>))-import Data.Coerce (coerce)+import Config.PackageFetcher+import Config.VersionSource import Data.List.NonEmpty (NonEmpty ((:|))) import Data.Maybe (fromMaybe)-import qualified Data.Text as T import Lens.Micro import Lens.Micro.Extras (view)-import NvFetcher.NixFetcher import NvFetcher.Types-import NvFetcher.Types.Lens-import Toml (TOML, TomlCodec, (.=))-import qualified Toml+import Toml import Validation (validationToEither) parseConfig :: TOML -> Either [Toml.TomlDecodeError] [Package]@@ -28,148 +26,24 @@ go [] [] sp = Right sp go [] se _ = Left se tables = [fmap (toPackage k) $ validationToEither $ Toml.runTomlCodec iCodec v | (Toml.unKey -> (Toml.unPiece -> k) :| _, v) <- Toml.toList $ Toml.tomlTables toml]- toPackage k (v, f, g@GitOptions {..}) =- let f' v = case f v of- x@FetchGit {} ->- x- & branch .~ goBranch- & deepClone .~ fromMaybe False goDeepClone- & fetchSubmodules .~ fromMaybe False goFetchSubmodules- & leaveDotGit .~ fromMaybe False goLeaveDotGit- x ->- if not $ isGitOptionsDefault g- then error $ "Try to set git-prefetch configuration for a url fetcher: " <> show x- else x- in Package k v f'- iCodec = (,,) <$> versionSourceCodec .= (view _1) <*> fetcherCodec .= (view _2) <*> gitOptionsCodec .= (view _3)--versionSourceCodec :: TomlCodec VersionSource-versionSourceCodec =- Toml.dimatch- ( \case- GitHubRelease {..} -> Just GitHubRelease {..}- _ -> Nothing- )- id- ( Toml.textBy- ( \case- GitHubRelease {..} -> _owner <> "/" <> _repo- _ -> error "impossible"- )- ( \x -> case T.split (== '/') x of- [_owner, _repo] -> Right GitHubRelease {..}- _ -> Left "unexpected github srouce: it should be something like [owner]/[repo]"- )- "src.github"- )- <|> Toml.dimatch- ( \case- Git {..} -> Just _vurl- _ -> Nothing- )- Git- (Toml.text "src.git")- <|> Toml.dimatch- ( \case- Pypi {..} -> Just _pypi- _ -> Nothing- )- Pypi- (Toml.text "src.pypi")- <|> Toml.dimatch- ( \case- ArchLinux {..} -> Just _archpkg- _ -> Nothing- )- ArchLinux- (Toml.text "src.archpkg")- <|> Toml.dimatch- ( \case- Aur {..} -> Just _aur- _ -> Nothing- )- Aur- (Toml.text "src.aur")- <|> Toml.dimatch- ( \case- Manual {..} -> Just _manual- _ -> Nothing- )- Manual- (Toml.text "src.manual")- <|> Toml.dimatch- ( \case- Repology {..} -> Just Repology {..}- _ -> Nothing- )- id- ( Toml.textBy- ( \case- Repology {..} -> _repology <> ":" <> _repo- _ -> error "impossible"- )- ( \t -> case T.split (== ':') t of- [_repology, _repo] -> Right Repology {..}- _ -> Left "unexpected repology source: it should be something like [project]:[repo]"- )- "src.repology"- )--unsupportError :: a-unsupportError = error "serialization is unsupported"---- | Use it only for deserialization!!!-fetcherCodec :: TomlCodec PackageFetcher-fetcherCodec =- Toml.textBy- unsupportError- ( \t -> case T.split (== '/') t of- [owner, rest] -> case T.split (== ':') rest of- [repo, rawV] ->- Right $ \(coerce -> realV) -> gitHubFetcher (owner, repo) $ coerce $ T.replace "$ver" realV rawV- [repo] -> Right $ gitHubFetcher (owner, repo)- _ -> Left "unexpected github fetcher: it should be something like [owner]/[repo] or [owner]/[repo]:[ver]"- _ -> Left "unexpected github fetcher: it should be something like [owner]/[repo] or [owner]/[repo]:[ver]"- )- "fetch.github"- <|> Toml.textBy- unsupportError- ( \t -> case T.split (== ':') t of- [fpypi, rawV] ->- Right $ \(coerce -> realV) -> pypiFetcher fpypi $ coerce $ T.replace "$ver" realV rawV- [fpypi] -> Right $ pypiFetcher fpypi- _ -> Left "unexpected pypi fetcher: it should be something like [pypi] or [pypi]:[ver]"- )- "fetch.pypi"- <|> Toml.textBy- unsupportError- ( \t -> case T.split (== ':') t of- [furl, rawV] ->- Right $ \(coerce -> realV) -> gitFetcher furl $ coerce $ T.replace "$ver" realV rawV- [furl] -> Right $ gitFetcher furl- _ -> Left "unexpected git fetcher: it should be something like [git_url] or [git_url]:[ver]"- )- "fetch.git"- <|> Toml.textBy- unsupportError- (\t -> Right $ \(coerce -> v) -> urlFetcher $ T.replace "$ver" v t)- "fetch.url"+ toPackage k (versionSource, f, e, c, options) = Package k (NvcheckerQ versionSource options) f e c+ iCodec =+ (,,,,)+ <$> versionSourceCodec .= view _1+ <*> fetcherCodec .= view _2+ <*> extractFilesCodec .= view _3+ <*> cargoLockPathCodec .= view _4+ <*> nvcheckerOptionsCodec .= view _5 -data GitOptions = GitOptions- { goBranch :: Maybe T.Text,- goDeepClone :: Maybe Bool,- goFetchSubmodules :: Maybe Bool,- goLeaveDotGit :: Maybe Bool- }- deriving (Eq)+extractFilesCodec :: TomlCodec PackageExtractSrc+extractFilesCodec = diwrap $ dimap Just (fromMaybe []) $ dioptional $ arrayOf _String "extract" -isGitOptionsDefault :: GitOptions -> Bool-isGitOptionsDefault = (== GitOptions Nothing Nothing Nothing Nothing)+cargoLockPathCodec :: TomlCodec (Maybe PackageCargoFilePath)+cargoLockPathCodec = dioptional $ diwrap (string "cargo_lock") -gitOptionsCodec :: TomlCodec GitOptions-gitOptionsCodec =- GitOptions- <$> Toml.dioptional (Toml.text "git.branch") .= goBranch- <*> Toml.dioptional (Toml.bool "git.deepClone") .= goDeepClone- <*> Toml.dioptional (Toml.bool "git.fetchSubmodules") .= goFetchSubmodules- <*> Toml.dioptional (Toml.bool "git.leaveDotGit") .= goLeaveDotGit+nvcheckerOptionsCodec :: TomlCodec NvcheckerOptions+nvcheckerOptionsCodec =+ NvcheckerOptions+ <$> dioptional (text "src.prefix") .= _stripPrefix+ <*> dioptional (text "src.from_pattern") .= _fromPattern+ <*> dioptional (text "src.to_pattern") .= _toPattern
+ app/Config/PackageFetcher.hs view
@@ -0,0 +1,102 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE DeriveAnyClass #-}+{-# LANGUAGE DeriveGeneric #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE ViewPatterns #-}++module Config.PackageFetcher (fetcherCodec) where++import Control.Applicative ((<|>))+import Data.Coerce (coerce)+import Data.Default (Default, def)+import Data.Maybe (fromMaybe)+import qualified Data.Text as T+import GHC.Generics (Generic)+import Lens.Micro+import Lens.Micro.Extras (view)+import NvFetcher.NixFetcher+import NvFetcher.Types+import NvFetcher.Types.Lens+import Toml++unsupportError :: a+unsupportError = error "serialization is unsupported"++--------------------------------------------------------------------------------++data GitOptions = GitOptions+ { goDeepClone :: Maybe Bool,+ goFetchSubmodules :: Maybe Bool,+ goLeaveDotGit :: Maybe Bool+ }+ deriving (Eq, Generic, Default)++gitOptionsCodec :: TomlCodec GitOptions+gitOptionsCodec =+ GitOptions+ <$> dioptional (bool "git.deepClone") .= goDeepClone+ <*> dioptional (bool "git.fetchSubmodules") .= goFetchSubmodules+ <*> dioptional (bool "git.leaveDotGit") .= goLeaveDotGit++_GitOptions :: Traversal' (NixFetcher f) GitOptions+_GitOptions f x@FetchGit {..} =+ ( \GitOptions {..} ->+ x+ & deepClone .~ fromMaybe False goDeepClone+ & fetchSubmodules .~ fromMaybe False goFetchSubmodules+ & leaveDotGit .~ fromMaybe False goLeaveDotGit+ )+ <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit))+_GitOptions _ x@FetchUrl {} = pure x++--------------------------------------------------------------------------------++gitHubICodec :: TomlCodec (Version -> NixFetcher 'Fresh)+gitHubICodec =+ textBy+ unsupportError+ ( \t -> case T.split (== '/') t of+ [owner, repo] -> Right $ gitHubFetcher (owner, repo)+ _ -> Left "unexpected github fetcher: it should be something like [owner]/[repo]"+ )+ "fetch.github"++gitHubCodec :: TomlCodec (Version -> NixFetcher Fresh)+gitHubCodec =+ dimap+ ( \f -> let fake = f "$ver" in (f, fromMaybe def $ fake ^? _GitOptions)+ )+ (\(f, g) v -> f v & _GitOptions .~ g)+ $ (,) <$> gitHubICodec .= view _1 <*> gitOptionsCodec .= view _2++--------------------------------------------------------------------------------+gitICodec :: TomlCodec (Version -> NixFetcher 'Fresh)+gitICodec =+ textBy+ unsupportError+ (Right . gitFetcher)+ "fetch.git"++gitCodec :: TomlCodec (Version -> NixFetcher Fresh)+gitCodec =+ dimap+ ( \f -> let fake = f "$ver" in (f, fromMaybe def $ fake ^? _GitOptions)+ )+ (\(f, g) v -> f v & _GitOptions .~ g)+ $ (,) <$> gitICodec .= view _1 <*> gitOptionsCodec .= view _2++--------------------------------------------------------------------------------++fetcherCodec :: TomlCodec PackageFetcher+fetcherCodec =+ gitHubCodec+ <|> Toml.textBy+ unsupportError+ (Right . pypiFetcher)+ "fetch.pypi"+ <|> gitCodec+ <|> Toml.textBy+ unsupportError+ (\t -> Right $ \(coerce -> v) -> urlFetcher $ T.replace "$ver" v t)+ "fetch.url"
+ app/Config/VersionSource.hs view
@@ -0,0 +1,177 @@+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RecordWildCards #-}++module Config.VersionSource (versionSourceCodec) where++import Data.Foldable (asum)+import Data.Text (Text)+import qualified Data.Text as T+import Data.Tuple.Extra (uncurry3)+import Lens.Micro+import Lens.Micro.Extras (view)+import NvFetcher.Types+import NvFetcher.Types.Lens+import Toml++versionSourceCodec :: TomlCodec VersionSource+versionSourceCodec =+ asum+ [ gitHubReleaseCodec,+ gitHubTagCodec,+ gitCodec,+ pypiCodec,+ archLinuxCodec,+ aurCodec,+ manualCodec,+ repologyCodec,+ webpageCodec,+ httpHeaderCodec+ ]++--------------------------------------------------------------------------------++githubICodec :: Key -> TomlCodec (Text, Text)+githubICodec =+ textBy+ (\(owner, repo) -> owner <> "/" <> repo)+ ( \t ->+ case T.split (== '/') t of+ [owner, repo] -> Right (owner, repo)+ _ -> Left "unexpected github source format: it should be something like [owner]/[repo]"+ )++listOptionsCodec :: TomlCodec ListOptions+listOptionsCodec =+ ListOptions+ <$> dioptional (text "src.include_regex") .= _includeRegex+ <*> dioptional (text "src.exclude_regex") .= _excludeRegex+ <*> dioptional+ ( textBy+ (T.pack . show)+ ( \case+ "parse_version" -> Right ParseVersion+ "vercmp" -> Right Vercmp+ _ -> Left "unexpected sort_version_key: it should be either parse_version or vercmp"+ )+ "src.sort_version_key"+ )+ .= _sortVersionKey+ <*> dioptional (text "src.ignored") .= _ignored++--------------------------------------------------------------------------------++matchGitHubRelease :: VersionSource -> Maybe (Text, Text)+matchGitHubRelease x = (,) <$> x ^? owner <*> x ^? repo++gitHubReleaseCodec :: TomlCodec VersionSource+gitHubReleaseCodec = dimatch matchGitHubRelease (uncurry GitHubRelease) $ githubICodec "src.github"++--------------------------------------------------------------------------------++matchGitHubTag :: VersionSource -> Maybe ((Text, Text), ListOptions)+matchGitHubTag x = do+ o <- x ^? owner+ r <- x ^? repo+ l <- x ^? listOptions+ pure ((o, r), l)++gitHubTagICodec :: TomlCodec ((Text, Text), ListOptions)+gitHubTagICodec =+ (,) <$> githubICodec "src.github_tag" .= view _1+ <*> listOptionsCodec .= view _2++gitHubTagCodec :: TomlCodec VersionSource+gitHubTagCodec = dimatch matchGitHubTag (\((_owner, _repo), _listOptions) -> GitHubTag {..}) gitHubTagICodec++--------------------------------------------------------------------------------++matchGit :: VersionSource -> Maybe (Text, Branch)+matchGit x = (,) <$> x ^? vurl <*> x ^? vbranch++gitICodec :: TomlCodec (Text, Branch)+gitICodec = (,) <$> text "src.git" .= view _1 <*> diwrap (dioptional (text "src.branch")) .= view _2++gitCodec :: TomlCodec VersionSource+gitCodec = dimatch matchGit (uncurry Git) gitICodec++--------------------------------------------------------------------------------++matchPypi :: VersionSource -> Maybe Text+matchPypi x = x ^? pypi++pypiCodec :: TomlCodec VersionSource+pypiCodec = dimatch matchPypi Pypi (text "src.pypi")++--------------------------------------------------------------------------------++matchArchLinux :: VersionSource -> Maybe Text+matchArchLinux x = x ^? archpkg++archLinuxCodec :: TomlCodec VersionSource+archLinuxCodec = dimatch matchArchLinux ArchLinux (text "src.archpkg")++--------------------------------------------------------------------------------++matchAur :: VersionSource -> Maybe Text+matchAur x = x ^? aur++aurCodec :: TomlCodec VersionSource+aurCodec = dimatch matchAur Aur (text "src.aur")++--------------------------------------------------------------------------------++matchManual :: VersionSource -> Maybe Text+matchManual x = x ^? manual++manualCodec :: TomlCodec VersionSource+manualCodec = dimatch matchManual Manual (text "src.manual")++--------------------------------------------------------------------------------++matchRepology :: VersionSource -> Maybe (Text, Text)+matchRepology x = (,) <$> x ^? repology <*> x ^? repo++repologyICodec :: TomlCodec (Text, Text)+repologyICodec =+ textBy+ (\(repology, repo) -> repology <> ":" <> repo)+ ( \t ->+ case T.split (== ':') t of+ [owner, repo] -> Right (owner, repo)+ _ -> Left "unexpected repology source format: it should be something like [repology]:[repo]"+ )+ "src.repology"++repologyCodec :: TomlCodec VersionSource+repologyCodec = dimatch matchRepology (uncurry Repology) repologyICodec++------------------------------------------------------------++matchWebpage :: VersionSource -> Maybe (Text, Text, ListOptions)+matchWebpage x = (,,) <$> x ^? vurl <*> x ^? regex <*> x ^? listOptions++webpageICodec :: TomlCodec (Text, Text, ListOptions)+webpageICodec =+ (,,) <$> text "src.webpage" .= view _1+ <*> text "src.regex" .= view _2+ <*> listOptionsCodec .= view _3++webpageCodec :: TomlCodec VersionSource+webpageCodec = dimatch matchWebpage (uncurry3 Webpage) webpageICodec++--------------------------------------------------------------------------------++matchHttpHeader :: VersionSource -> Maybe (Text, Text, ListOptions)+matchHttpHeader x = (,,) <$> x ^? vurl <*> x ^? regex <*> x ^? listOptions++httpHeaderICodec :: TomlCodec (Text, Text, ListOptions)+httpHeaderICodec =+ (,,) <$> text "src.httpheader" .= view _1+ <*> text "src.regex" .= view _2+ <*> listOptionsCodec .= view _3++httpHeaderCodec :: TomlCodec VersionSource+httpHeaderCodec = dimatch matchHttpHeader (uncurry3 HttpHeader) httpHeaderICodec++--------------------------------------------------------------------------------
app/Main.hs view
@@ -8,112 +8,31 @@ import Config import qualified Data.Text as T import qualified Data.Text.IO as T-import Development.Shake import NvFetcher+import NvFetcher.Options import Options.Applicative.Simple-import qualified Paths_nvfetcher as Paths import qualified Toml -data CLIOptions = CLIOptions- { configPath :: FilePath,- outputPath :: FilePath,- logPath :: Maybe FilePath,- threads :: Int,- timing :: Bool,- verbose :: Bool,- target :: String- }- deriving (Show)--cliOptionsParser :: Parser CLIOptions-cliOptionsParser =- CLIOptions- <$> strOption- ( long "config"- <> short 'c'- <> metavar "FILE"- <> help "Path to nvfetcher TOML config"- <> value "nvfetcher.toml"- <> showDefault- <> completer (bashCompleter "file")- )+getCLIOptionsWithConfig :: IO (CLIOptions, FilePath)+getCLIOptionsWithConfig =+ getCLIOptions $+ (,) <$> cliOptionsParser <*> strOption- ( long "output"- <> short 'o'+ ( long "config"+ <> short 'c' <> metavar "FILE"- <> help "Path to output nix file"+ <> help "Path to nvfetcher TOML config"+ <> value "nvfetcher.toml" <> showDefault- <> value "sources.nix" <> completer (bashCompleter "file") )- <*> optional- ( strOption- ( long "changelog"- <> short 'l'- <> metavar "FILE"- <> help "Dump version changes to a file"- <> completer (bashCompleter "file")- )- )- <*> ( option- auto- ( short 'j'- <> metavar "NUM"- <> help "Number of threads (0: detected number of processors)"- <> value 0- <> showDefault- )- )- <*> switch (long "timing" <> short 't' <> help "Show build time")- <*> switch (long "verbose" <> short 'v' <> help "Verbose mode")- <*> strArgument- ( metavar "TARGET"- <> help "Two targets are available: build and clean"- <> value "build"- <> completer (listCompleter ["build", "clean"])- ) -version :: String-version = $(simpleVersion Paths.version)--getCLIOptions :: IO CLIOptions-getCLIOptions = do- (opts, ()) <-- simpleOptions- version- "nvfetcher - generate nix sources expr for the latest version of packages"- ( unlines- [ "It's important to keep .shake dir to make caches work properly.",- "If you change the version source or fetcher of an existing package, please run target \"clean\" to rebuild everything."- ]- )- cliOptionsParser- empty- pure opts- main :: IO () main = do- CLIOptions {..} <- getCLIOptions- let args =- defaultArgs- { argOutputFilePath = outputPath,- argActionAfterBuild = maybe (pure ()) logChangesToFile logPath,- argTarget = target,- argShakeOptions =- (argShakeOptions defaultArgs)- { shakeTimings = timing,- shakeVerbosity = if verbose then Verbose else Info,- shakeThreads = threads- }- }+ (opt, configPath) <- getCLIOptionsWithConfig toml <- Toml.parse <$> T.readFile configPath case toml of Left e -> error $ T.unpack $ Toml.prettyTomlDecodeError $ Toml.ParseError e Right x -> case parseConfig x of Left e -> error $ T.unpack $ Toml.prettyTomlDecodeErrors e- Right pkgs -> runNvFetcher args $ purePackageSet pkgs--logChangesToFile :: FilePath -> Action ()-logChangesToFile fp = do- changes <- getVersionChanges- writeFile' fp $ unlines $ show <$> changes+ Right pkgs -> runNvFetcherNoCLI (cliOptionsToArgs opt) $ purePackageSet pkgs
nvfetcher.cabal view
@@ -1,10 +1,12 @@ cabal-version: 2.4 name: nvfetcher-version: 0.2.0.0+version: 0.3.0.0 synopsis: Generate nix sources expr for the latest version of packages -description: Please see README+description:+ Please see [README](https://github.com/berberman/nvfetcher/blob/master/README.md)+ homepage: https://github.com/berberman/nvfetcher bug-reports: https://github.com/berberman/nvfetcher/issues license: MIT@@ -24,19 +26,25 @@ common common-options build-depends:- , aeson ^>=1.5.6- , base >=4.8 && <5+ , aeson ^>=1.5.6+ , base >=4.8 && <5 , binary+ , binary-instances ^>=1.0.1 , bytestring , containers- , extra ^>=1.7.9- , free ^>=5.1.5+ , data-default ^>=0.7.1+ , extra ^>=1.7.9+ , free ^>=5.1.5 , microlens , microlens-th- , neat-interpolation ^>=0.5.1- , shake ^>=0.19.4+ , neat-interpolation ^>=0.5.1+ , optparse-simple ^>=0.1.1+ , parsec+ , shake ^>=0.19.4 , text+ , tomland ^>=1.3.2 , transformers+ , unordered-containers ghc-options: -Wall -Wcompat -Widentities -Wincomplete-uni-patterns@@ -53,29 +61,34 @@ exposed-modules: NvFetcher NvFetcher.Core+ NvFetcher.ExtractSrc+ NvFetcher.FetchRustGitDeps+ NvFetcher.NixExpr NvFetcher.NixFetcher NvFetcher.Nvchecker+ NvFetcher.Options NvFetcher.PackageSet- NvFetcher.ShakeExtras NvFetcher.Types NvFetcher.Types.Lens+ NvFetcher.Types.ShakeExtras + other-modules: Paths_nvfetcher+ autogen-modules: Paths_nvfetcher+ executable nvfetcher- import: common-options- hs-source-dirs: app- main-is: Main.hs+ import: common-options+ hs-source-dirs: app+ main-is: Main.hs other-modules: Config- Paths_nvfetcher+ Config.PackageFetcher+ Config.VersionSource - autogen-modules: Paths_nvfetcher build-depends: , nvfetcher- , tomland ^>=1.3.2 , validation-selective- , optparse-simple - ghc-options: -threaded -rtsopts -with-rtsopts=-N+ ghc-options: -threaded -rtsopts -with-rtsopts=-N flag build-example description: Build example executable
src/NvFetcher.hs view
@@ -42,22 +42,29 @@ ( Args (..), defaultArgs, runNvFetcher,+ runNvFetcherNoCLI,+ cliOptionsToArgs, module NvFetcher.PackageSet, module NvFetcher.Types,- module NvFetcher.ShakeExtras,+ module NvFetcher.Types.ShakeExtras, ) where +import qualified Control.Exception as CE import Data.Text (Text) import qualified Data.Text as T import Development.Shake+import Development.Shake.FilePath import NeatInterpolation (trimming) import NvFetcher.Core import NvFetcher.NixFetcher import NvFetcher.Nvchecker+import NvFetcher.Options import NvFetcher.PackageSet-import NvFetcher.ShakeExtras import NvFetcher.Types+import NvFetcher.Types.ShakeExtras+import NvFetcher.Utils (getShakeDir)+import System.Directory.Extra (createDirectoryIfMissing, createFileLink, removeFile) -- | Arguments for running nvfetcher data Args = Args@@ -72,7 +79,9 @@ -- | Action run after build rule argActionAfterBuild :: Action (), -- | Action run after clean rule- argActionAfterClean :: Action ()+ argActionAfterClean :: Action (),+ -- | Retry times+ argRetries :: Int } -- | Default arguments of 'defaultMain'@@ -82,7 +91,8 @@ defaultArgs = Args ( shakeOptions- { shakeProgress = progressSimple+ { shakeProgress = progressSimple,+ shakeThreads = 0 } ) "build"@@ -90,23 +100,55 @@ (pure ()) (pure ()) (pure ())+ 3 +-- | Run nvfetcher with CLI options+--+-- This function calls 'runNvFetcherNoCLI', using 'Args' from 'CLIOptions'.+-- Use this function to create your own Haskell executable program.+runNvFetcher :: PackageSet () -> IO ()+runNvFetcher packageSet =+ getCLIOptions cliOptionsParser >>= flip runNvFetcherNoCLI packageSet . cliOptionsToArgs++-- | Apply 'CLIOptions' to 'defaultArgs'+cliOptionsToArgs :: CLIOptions -> Args+cliOptionsToArgs CLIOptions {..} =+ defaultArgs+ { argOutputFilePath = outputPath,+ argActionAfterBuild = maybe (pure ()) logChangesToFile logPath,+ argTarget = target,+ argShakeOptions =+ (argShakeOptions defaultArgs)+ { shakeTimings = timing,+ shakeVerbosity = if verbose then Verbose else Info,+ shakeThreads = threads+ }+ }++logChangesToFile :: FilePath -> Action ()+logChangesToFile fp = do+ changes <- getVersionChanges+ writeFile' fp $ unlines $ show <$> changes+ -- | Entry point of nvfetcher-runNvFetcher :: Args -> PackageSet () -> IO ()-runNvFetcher args@Args {..} packageSet = do+runNvFetcherNoCLI :: Args -> PackageSet () -> IO ()+runNvFetcherNoCLI args@Args {..} packageSet = do pkgs <- runPackageSet packageSet- shakeExtras <- initShakeExtras pkgs+ shakeExtras <- initShakeExtras pkgs argRetries let opts = argShakeOptions- { shakeExtra = addShakeExtra shakeExtras (shakeExtra argShakeOptions)+ { shakeExtra = addShakeExtra shakeExtras (shakeExtra argShakeOptions),+ shakeFiles = "_build" } rules = mainRules args shake opts $ want [argTarget] >> rules +--------------------------------------------------------------------------------+ mainRules :: Args -> Rules () mainRules Args {..} = do "clean" ~> do- removeFilesAfter ".shake" ["//*"]+ removeFilesAfter "_build" ["//*"] removeFilesAfter "." [argOutputFilePath] argActionAfterClean @@ -119,8 +161,16 @@ else do putInfo "Changes:" putInfo $ unlines $ show <$> changes- writeFileChanged argOutputFilePath $ T.unpack $ srouces (T.unlines body) <> "\n"- putInfo $ "Generate " <> argOutputFilePath+ shakeDir <- getShakeDir+ let genPath = shakeDir </> "generated.nix"+ putVerbose $ "Generating " <> genPath+ writeFileChanged genPath $ T.unpack $ srouces (T.unlines body) <> "\n"+ need [genPath]+ let outDir = takeDirectory argOutputFilePath+ liftIO $ createDirectoryIfMissing True outDir+ liftIO $ CE.catch @IOError (removeFile argOutputFilePath) (const $ pure ())+ putVerbose $ "Symlinking " <> argOutputFilePath+ liftIO $ createFileLink genPath argOutputFilePath argActionAfterBuild argRules
src/NvFetcher/Core.hs view
@@ -1,5 +1,6 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE LambdaCase #-}+{-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE TypeApplications #-}@@ -11,28 +12,37 @@ -- Stability: experimental -- Portability: portable module NvFetcher.Core- ( coreRules,+ ( Core (..),+ coreRules, generateNixSourceExpr, ) where import Data.Coerce (coerce)+import qualified Data.HashMap.Strict as HMap import Data.Text (Text)+import qualified Data.Text as T import Development.Shake+import Development.Shake.FilePath import Development.Shake.Rule import NeatInterpolation (trimming)+import NvFetcher.ExtractSrc+import NvFetcher.FetchRustGitDeps+import NvFetcher.NixExpr import NvFetcher.NixFetcher import NvFetcher.Nvchecker-import NvFetcher.ShakeExtras import NvFetcher.Types+import NvFetcher.Types.ShakeExtras import NvFetcher.Utils -- | The core rule of nvchecker.--- nvchecker rule and prefetch rule are wired here.+-- all rules are wired here. coreRules :: Rules () coreRules = do nvcheckerRule prefetchRule+ extractSrcRule+ fetchRustGitDepsRule addBuiltinRule noLint noIdentity $ \(WithPackageKey (Core, pkg)) mOld mode -> case mode of RunDependenciesSame | Just old <- mOld,@@ -41,21 +51,71 @@ _ -> lookupPackage pkg >>= \case Nothing -> fail $ "Unkown package key: " <> show pkg- Just Package {..} -> do- (NvcheckerResult version mOldV) <- checkVersion _pversion pkg- prefetched <- prefetch $ _pfetcher version- case mOldV of- Nothing ->- recordVersionChange _pname Nothing version- Just old- | old /= version ->- recordVersionChange _pname (Just old) version- _ -> pure ()- let result = gen _pname version prefetched- pure $ RunResult ChangedRecomputeDiff (encode' (result, version)) result+ Just+ Package+ { _pversion = NvcheckerQ versionSource options,+ _pextract = PackageExtractSrc extract,+ ..+ } -> do+ (NvcheckerA version mOldV) <- checkVersion versionSource options pkg+ prefetched <- prefetch $ _pfetcher version+ shakeDir <- getShakeDir+ appending1 <-+ if null extract+ then pure ""+ else do+ result <- HMap.toList <$> extractSrc prefetched extract+ T.unlines+ <$> sequence+ [ do+ -- write extracted files to shake dir+ -- and read them in nix using 'builtins.readFile'+ writeFile' (shakeDir </> path) (T.unpack v)+ pure $ toNixExpr k <> " = builtins.readFile ./" <> T.pack path+ | (k, v) <- result,+ let path =+ T.unpack _pname+ <> "-"+ <> T.unpack (coerce version)+ </> k+ ]+ appending2 <-+ case _pcargo of+ Just (PackageCargoFilePath lockPath) -> do+ (_, lockData) <- head . HMap.toList <$> extractSrc prefetched [lockPath]+ result <- fetchRustGitDeps prefetched lockPath+ let body = T.unlines [asString k <> " = " <> coerce (asString $ coerce v) <> ";" | (k, v) <- HMap.toList result]+ lockPath' =+ T.unpack _pname+ <> "-"+ <> T.unpack (coerce version)+ </> lockPath+ lockPathNix = "./" <> T.pack lockPath'+ -- similar to extract src, write lock file to shake dir+ writeFile' (shakeDir </> lockPath') $ T.unpack lockData+ pure+ [trimming|+ cargoLock = {+ lockFile = $lockPathNix;+ outputHashes = {+ $body+ };+ };+ |]+ _ -> pure ""+ case mOldV of+ Nothing ->+ recordVersionChange _pname Nothing version+ Just old+ | old /= version ->+ recordVersionChange _pname (Just old) version+ _ -> pure ()+ let result = gen _pname version prefetched $ appending1 <> appending2+ pure $ RunResult ChangedRecomputeDiff (encode' (result, version)) result -- | Run the core rule.--- Given a package key, run nvchecker and then prefetch it,+-- Given a 'PackageKey', run "NvFetcher.Nvchecker", "NvFetcher.NixFetcher"+-- (may also run "NvFetcher.ExtractSrc" or "FetchRustGitDeps") -- resulting a nix source snippet like: -- -- @@@ -71,12 +131,13 @@ generateNixSourceExpr :: PackageKey -> Action NixExpr generateNixSourceExpr k = apply1 $ WithPackageKey (Core, k) -gen :: PackageName -> Version -> NixFetcher Prefetched -> Text-gen name (coerce -> ver) (toNixExpr -> srcP) =+gen :: PackageName -> Version -> NixFetcher Fetched -> Text -> Text+gen name (coerce -> ver) (toNixExpr -> srcP) appending = [trimming| $name = { pname = "$name"; version = "$ver"; src = $srcP;+ $appending }; |]
+ src/NvFetcher/ExtractSrc.hs view
@@ -0,0 +1,69 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE ScopedTypeVariables #-}+{-# LANGUAGE ViewPatterns #-}++-- | Copyright: (c) 2021 berberman+-- SPDX-License-Identifier: MIT+-- Maintainer: berberman <berberman@yandex.com>+-- Stability: experimental+-- Portability: portable+--+-- This module provides function that extracs files contents from package sources.+-- It uses [IFD](https://nixos.wiki/wiki/Import_From_Derivation) under the hood,+-- pulling /textual/ files from source drv.+-- Because we use @nix-instantiate@ to build drv, so @<nixpkgs>@ (@NIX_PATH@) is required.+module NvFetcher.ExtractSrc+ ( -- * Types+ ExtractSrcQ (..),++ -- * Rules+ extractSrcRule,+ extractSrc,+ )+where++import Control.Monad (void)+import qualified Data.Aeson as A+import Data.Binary.Instances ()+import Data.HashMap.Strict (HashMap)+import Data.Text (Text)+import qualified Data.Text as T+import qualified Data.Text.Encoding as T+import Development.Shake+import NeatInterpolation (trimming)+import NvFetcher.NixExpr+import NvFetcher.Types+import NvFetcher.Types.ShakeExtras++-- | Rules of extract source+extractSrcRule :: Rules ()+extractSrcRule = void $+ addOracleCache $ \(q :: ExtractSrcQ) -> withTempFile $ \fp -> withRetries $ do+ writeFile' fp $ T.unpack $ wrap $ toNixExpr q+ need [fp]+ -- TODO: Avoid using NIX_PATH+ (CmdTime t, StdoutTrim out, CmdLine c) <- cmd Shell $ "nix-instantiate --eval --strict --json --read-write-mode -E 'let pkgs = import <nixpkgs> { }; in ((import " <> fp <> ") pkgs)'"+ putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"+ case A.decodeStrict out of+ Just x -> pure x+ _ -> fail $ "Failed to parse output of nix-instantiate: " <> T.unpack (T.decodeUtf8 out)++-- | Run extract source+extractSrc ::+ -- | prefetched source+ NixFetcher Fetched ->+ -- | relative file paths to extract+ [FilePath] ->+ Action (HashMap FilePath Text)+extractSrc fetcher fp = askOracle (ExtractSrcQ fetcher fp)++--------------------------------------------------------------------------------++wrap :: NixExpr -> NixExpr+wrap expr =+ [trimming|+ { pkgs, ... }:+ $expr+ |]
+ src/NvFetcher/FetchRustGitDeps.hs view
@@ -0,0 +1,109 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE ViewPatterns #-}++-- | Copyright: (c) 2021 berberman+-- SPDX-License-Identifier: MIT+-- Maintainer: berberman <berberman@yandex.com>+-- Stability: experimental+-- Portability: portable+--+-- This module provides function to calculate @cargoLock@ used in @rustPlatform.buildRustPackage@.+module NvFetcher.FetchRustGitDeps+ ( -- * Types+ FetchRustGitDepsQ (..),++ -- * Rules+ fetchRustGitDepsRule,+ fetchRustGitDeps,+ )+where++import Control.Monad (void)+import Data.Binary.Instances ()+import Data.Coerce (coerce)+import Data.HashMap.Strict (HashMap)+import qualified Data.HashMap.Strict as HMap+import Data.List.Extra (nubOrdOn)+import Data.Maybe (maybeToList)+import Data.Text (Text)+import qualified Data.Text as T+import Development.Shake+import NvFetcher.ExtractSrc+import NvFetcher.NixFetcher+import NvFetcher.Types+import Text.Parsec+import Text.Parsec.Text+import Toml (TomlCodec, (.=))+import qualified Toml++-- | Rules of fetch rust git dependencies+fetchRustGitDepsRule :: Rules ()+fetchRustGitDepsRule = void $+ addOracleCache $ \(FetchRustGitDepsQ fetcher lockPath) -> do+ cargoLock <- head . HMap.elems <$> extractSrc fetcher [lockPath]+ deps <- case Toml.decode (Toml.list rustDepCodec "package") cargoLock of+ Right r -> pure $ nubOrdOn rrawSrc r+ Left err -> fail $ "Failed to parse Cargo.lock: " <> T.unpack (Toml.prettyTomlDecodeErrors err)+ r <-+ parallel+ [ case parse gitSrcParser (T.unpack rname) src of+ Right ParsedGitSrc {..} -> do+ (_sha256 -> sha256) <- prefetch $ gitFetcher pgurl pgsha+ -- @${name}-${version}@ -> sha256+ pure (rname <> "-" <> coerce rversion, sha256)+ Left err -> fail $ "Failed to parse git source in Cargo.lock: " <> show err+ | RustDep {..} <- deps,+ -- it's a dependency+ src <- maybeToList rrawSrc,+ -- it's a git dependency+ "git+" `T.isPrefixOf` src+ ]+ pure $ HMap.fromList r++-- | Run fetch rust git dependencies+fetchRustGitDeps ::+ -- | prefetched source+ NixFetcher Fetched ->+ -- | relative file path of @Cargo.lock@+ FilePath ->+ Action (HashMap Text Checksum)+fetchRustGitDeps fetcher lockPath = askOracle $ FetchRustGitDepsQ fetcher lockPath++data ParsedGitSrc = ParsedGitSrc+ { -- | git url+ pgurl :: Text,+ pgsha :: Version+ }+ deriving (Show, Eq, Ord)++-- | Parse src in lock: git\+([^?]+)(\?rev=(.*))?#(.*)?+-- >>> parse gitSrcParser "test" "git+https://github.com/rust-random/rand.git?rev=0.8.3#6ecbe2626b2cc6110a25c97b1702b347574febc7"+-- Right (ParsedGitSrc {pgurl = "https://github.com/rust-random/rand.git", pgsha = "6ecbe2626b2cc6110a25c97b1702b347574febc7"})+--+-- >>> parse gitSrcParser "test" "git+https://github.com/rust-random/rand.git#f0e01ee0a7257753cc51b291f62666f4765923ef"+-- Right (ParsedGitSrc {pgurl = "https://github.com/rust-random/rand.git", pgsha = "f0e01ee0a7257753cc51b291f62666f4765923ef"})+gitSrcParser :: Parser ParsedGitSrc+gitSrcParser = do+ _ <- string "git+"+ pgurl <- many1 $ noneOf ['?', '#']+ let revParser = string "?rev=" >> many1 (noneOf ['#'])+ _rev <- optionMaybe revParser+ _ <- char '#'+ pgsha <- manyTill anyChar eof+ pure $ ParsedGitSrc (T.pack pgurl) (coerce $ T.pack pgsha)++data RustDep = RustDep+ { rname :: PackageName,+ rversion :: Version,+ rrawSrc :: Maybe Text+ }+ deriving (Show, Eq, Ord)++rustDepCodec :: TomlCodec RustDep+rustDepCodec =+ RustDep+ <$> Toml.text "name" .= rname+ <*> Toml.diwrap (Toml.text "version") .= rversion+ <*> Toml.dioptional (Toml.text "source") .= rrawSrc
+ src/NvFetcher/NixExpr.hs view
@@ -0,0 +1,102 @@+{-# LANGUAGE DataKinds #-}+{-# LANGUAGE FlexibleInstances #-}+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE OverloadedStrings #-}+{-# LANGUAGE QuasiQuotes #-}+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE ViewPatterns #-}++-- | Copyright: (c) 2021 berberman+-- SPDX-License-Identifier: MIT+-- Maintainer: berberman <berberman@yandex.com>+-- Stability: experimental+-- Portability: portable+--+-- This module contains a type class 'ToNixExpr' and some its instances associated with either Haskell+-- primitive types or our "NvFetcher.Types".+module NvFetcher.NixExpr+ ( NixExpr,+ ToNixExpr (..),+ )+where++import Data.Coerce (coerce)+import Data.Text (Text)+import qualified Data.Text as T+import NeatInterpolation (trimming)+import NvFetcher.Types+import NvFetcher.Utils (asString)++-- | Types can be converted into nix expr+class ToNixExpr a where+ toNixExpr :: a -> NixExpr++instance ToNixExpr (NixFetcher Fresh) where+ toNixExpr = nixFetcher "lib.fakeSha256"++instance ToNixExpr (NixFetcher Fetched) where+ toNixExpr f = nixFetcher (asString $ coerce $ _sha256 f) f++instance ToNixExpr Bool where+ toNixExpr True = "true"+ toNixExpr False = "false"++instance ToNixExpr a => ToNixExpr [a] where+ toNixExpr xs = foldl (\acc x -> acc <> " " <> toNixExpr x) "[" xs <> " ]"++instance {-# OVERLAPS #-} ToNixExpr String where+ toNixExpr = T.pack . show++instance ToNixExpr NixExpr where+ toNixExpr = id++instance ToNixExpr Version where+ toNixExpr = coerce++nixFetcher :: Text -> NixFetcher k -> NixExpr+nixFetcher sha256 = \case+ FetchGit+ { _sha256 = _,+ _rev = asString . toNixExpr -> rev,+ _fetchSubmodules = toNixExpr -> fetchSubmodules,+ _deepClone = toNixExpr -> deepClone,+ _leaveDotGit = toNixExpr -> leaveDotGit,+ _furl = asString -> url+ } ->+ [trimming|+ fetchgit {+ url = $url;+ rev = $rev;+ fetchSubmodules = $fetchSubmodules;+ deepClone = $deepClone;+ leaveDotGit = $leaveDotGit;+ sha256 = $sha256;+ }+ |]+ (FetchUrl (asString -> url) _) ->+ [trimming|+ fetchurl {+ sha256 = $sha256;+ url = $url;+ }+ |]++instance ToNixExpr ExtractSrcQ where+ toNixExpr (ExtractSrcQ fetcher files) = extractFiles fetcher files++extractFiles :: NixFetcher Fetched -> [FilePath] -> NixExpr+extractFiles (toNixExpr -> fetcherExpr) (toNixExpr -> fileNames) =+ [trimming|+ let+ drv = import (pkgs.writeText "src" ''+ pkgs: {+ src = pkgs.$fetcherExpr;+ }+ '');+ fileNames = $fileNames;+ toFile = f: builtins.readFile ((drv pkgs).src + "/" + f);+ in builtins.listToAttrs (builtins.map (x: {+ name = x;+ value = toFile x;+ }) fileNames)+ |]
src/NvFetcher/NixFetcher.hs view
@@ -26,9 +26,8 @@ module NvFetcher.NixFetcher ( -- * Types NixFetcher (..),- Prefetch (..),- ToNixExpr (..),- PrefetchResult,+ FetchStatus (..),+ FetchResult, -- * Rules prefetchRule,@@ -47,44 +46,25 @@ import qualified Data.Aeson as A import qualified Data.Aeson.Types as A import Data.Coerce (coerce)-import Data.Maybe (maybeToList) import Data.Text (Text) import qualified Data.Text as T import qualified Data.Text.Encoding as T import Development.Shake import NeatInterpolation (trimming) import NvFetcher.Types+import NvFetcher.Types.ShakeExtras -------------------------------------------------------------------------------- --- | Types can be converted into nix expr-class ToNixExpr a where- toNixExpr :: a -> NixExpr--instance ToNixExpr (NixFetcher Fresh) where- toNixExpr = buildNixFetcher "lib.fakeSha256"--instance ToNixExpr (NixFetcher Prefetched) where- -- add quotation marks- toNixExpr f = buildNixFetcher (T.pack $ show $ T.unpack $ coerce $ _sha256 f) f--instance ToNixExpr Bool where- toNixExpr True = "true"- toNixExpr False = "false"--instance ToNixExpr Version where- toNixExpr = coerce--runFetcher :: NixFetcher Fresh -> Action SHA256+runFetcher :: NixFetcher Fresh -> Action Checksum runFetcher = \case FetchGit {..} -> do- let parser = A.withObject "nix-prefetch-git" $ \o -> SHA256 <$> o A..: "sha256"+ let parser = A.withObject "nix-prefetch-git" $ \o -> Checksum <$> o A..: "sha256" (CmdTime t, Stdout out, CmdLine c) <- command [EchoStderr False] "nix-prefetch-git" $ [T.unpack _furl] <> ["--rev", T.unpack $ coerce _rev] <> ["--fetch-submodules" | _fetchSubmodules]- <> concat [["--branch-name", T.unpack b] | b <- maybeToList _branch] <> ["--deepClone" | _deepClone] <> ["--leave-dotGit" | _leaveDotGit] putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"@@ -99,34 +79,6 @@ [x] -> pure $ coerce x _ -> fail $ "Failed to parse output from nix-prefetch-url: " <> T.unpack out -buildNixFetcher :: Text -> NixFetcher k -> Text-buildNixFetcher sha256 = \case- FetchGit- { _sha256 = _,- _rev = toNixExpr -> rev,- _fetchSubmodules = toNixExpr -> fetchSubmodules,- _deepClone = toNixExpr -> deepClone,- _leaveDotGit = toNixExpr -> leaveDotGit,- ..- } ->- [trimming|- fetchgit {- url = "$_furl";- rev = "$rev";- fetchSubmodules = $fetchSubmodules;- deepClone = $deepClone;- leaveDotGit = $leaveDotGit;- sha256 = $sha256;- }- |]- (FetchUrl url _) ->- [trimming|- fetchurl {- sha256 = $sha256;- url = "$url";- }- |]- pypiUrl :: Text -> Version -> Text pypiUrl pypi (coerce -> ver) = let h = T.cons (T.head pypi) ""@@ -138,18 +90,18 @@ prefetchRule :: Rules () prefetchRule = void $ addOracleCache $ \(f :: NixFetcher Fresh) -> do- sha256 <- runFetcher f+ sha256 <- withRetries $ runFetcher f pure $ f {_sha256 = sha256} -- | Run nix fetcher-prefetch :: NixFetcher Fresh -> Action (NixFetcher Prefetched)+prefetch :: NixFetcher Fresh -> Action (NixFetcher Fetched) prefetch = askOracle -------------------------------------------------------------------------------- -- | Create a fetcher from git url gitFetcher :: Text -> PackageFetcher-gitFetcher furl rev = FetchGit furl rev Nothing False False False ()+gitFetcher furl rev = FetchGit furl rev False False False () -- | Create a fetcher from github repo gitHubFetcher ::
src/NvFetcher/Nvchecker.hs view
@@ -1,6 +1,5 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ViewPatterns #-} @@ -19,8 +18,12 @@ -- see 'nvcheckerRule' for details. module NvFetcher.Nvchecker ( -- * Types+ VersionSortMethod (..),+ ListOptions (..),+ NvcheckerQ (..),+ NvcheckerOptions (..), VersionSource (..),- NvcheckerResult (..),+ NvcheckerA (..), -- * Rules nvcheckerRule,@@ -31,19 +34,20 @@ import qualified Data.Aeson as A import Data.Coerce (coerce) import Data.Maybe (mapMaybe)-import Data.Text (Text)+import Data.String (fromString) import qualified Data.Text as T import qualified Data.Text.Encoding as T import Development.Shake import Development.Shake.Rule-import NeatInterpolation (trimming)-import NvFetcher.ShakeExtras import NvFetcher.Types+import NvFetcher.Types.ShakeExtras import NvFetcher.Utils+import Toml (Value (Bool, Text), pretty)+import Toml.Type.Edsl -- | Rules of nvchecker nvcheckerRule :: Rules ()-nvcheckerRule = addBuiltinRule noLint noIdentity $ \(WithPackageKey (q, pkg)) old _mode ->+nvcheckerRule = addBuiltinRule noLint noIdentity $ \(WithPackageKey (NvcheckerQ versionSource options, pkg)) old _mode -> -- If the package was removed after the last run, -- shake still runs the nvchecker rule for this package. -- So we record a version change here, indicating that the package has been removed.@@ -54,8 +58,8 @@ recordVersionChange (coerce pkg) oldVer "∅" pure $ RunResult ChangedRecomputeDiff mempty undefined -- skip running, returning a never consumed result _ ->- withTempFile $ \config -> do- writeFile' config $ T.unpack $ genNvConfig "pkg" q+ withTempFile $ \config -> withRetries $ do+ writeFile' config $ T.unpack $ pretty $ mkToml $ genNvConfig pkg options versionSource need [config] (CmdTime t, Stdout out, CmdLine c) <- cmd $ "nvchecker --logger json -c " <> config putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"@@ -73,56 +77,66 @@ else RunResult ChangedRecomputeDiff (encode' $ nvNow now) now {nvOld = Just cachedResult} Nothing -> RunResult ChangedRecomputeDiff (encode' $ nvNow now) now -genNvConfig :: Text -> VersionSource -> Text-genNvConfig srcName = \case- GitHubRelease {..} ->- [trimming|- [$srcName]- source = "github"- github = "$_owner/$_repo"- use_latest_release = true- |]- Git {..} ->- [trimming|- [$srcName]- source = "git"- git = "$_vurl"- use_commit = true- |]- Aur {..} ->- [trimming|- [$srcName]- source = "aur"- aur = "$_aur"- strip_release = true- |]- ArchLinux {..} ->- [trimming|- [$srcName]- source = "archpkg"- archpkg = "$_archpkg"- strip_release = true- |]- Pypi {..} ->- [trimming|- [$srcName]- source = "pypi"- pypi = "$_pypi"- |]- Manual {..} ->- [trimming|- [$srcName]- source = "manual"- manual = "$_manual"- |]- Repology {..} ->- [trimming|- [$srcName]- source = "repology"- repology = "$_repology"- repo = "$_repo"- |]+genNvConfig :: PackageKey -> NvcheckerOptions -> VersionSource -> TDSL+genNvConfig pkg options versionSource = table (fromString $ T.unpack $ coerce pkg) $ do+ genVersionSource versionSource+ genOptions options+ where+ key =:? (Just x) = key =: Text x+ _ =:? _ = pure ()+ genVersionSource = \case+ GitHubRelease {..} -> do+ "source" =: "github"+ "github" =: Text (_owner <> "/" <> _repo)+ "use_latest_release" =: Bool True+ GitHubTag {..} -> do+ "source" =: "github"+ "github" =: Text (_owner <> "/" <> _repo)+ "use_max_tag" =: Bool True+ genListOptions _listOptions+ Git {..} -> do+ "source" =: "git"+ "git" =: Text _vurl+ "branch" =:? coerce _vbranch+ "use_commit" =: Bool True+ Aur {..} -> do+ "source" =: "aur"+ "aur" =: Text _aur+ "strip_release" =: Bool True+ ArchLinux {..} -> do+ "source" =: "archpkg"+ "aur" =: Text _archpkg+ "strip_release" =: Bool True+ Pypi {..} -> do+ "source" =: "pypi"+ "pypi" =: Text _pypi+ Manual {..} -> do+ "source" =: "manual"+ "manual" =: Text _manual+ Repology {..} -> do+ "source" =: "repology"+ "repology" =: Text _repology+ "repo" =: Text _repo+ Webpage {..} -> do+ "source" =: "regex"+ "url" =: Text _vurl+ "regex" =: Text _regex+ genListOptions _listOptions+ HttpHeader {..} -> do+ "source" =: "httpheader"+ "url" =: Text _vurl+ "regex" =: Text _regex+ genListOptions _listOptions+ genListOptions ListOptions {..} = do+ "include_regex" =:? _includeRegex+ "exclude_regex" =:? _excludeRegex+ "sort_version_key" =:? fmap (T.pack . show) _sortVersionKey+ "ignored" =:? _ignored+ genOptions NvcheckerOptions {..} = do+ "prefix" =:? _stripPrefix+ "from_pattern" =:? _fromPattern+ "to_pattern" =:? _toPattern -- | Run nvchecker-checkVersion :: VersionSource -> PackageKey -> Action NvcheckerResult-checkVersion v k = apply1 $ WithPackageKey (v, k)+checkVersion :: VersionSource -> NvcheckerOptions -> PackageKey -> Action NvcheckerA+checkVersion v o k = apply1 $ WithPackageKey (NvcheckerQ v o, k)
+ src/NvFetcher/Options.hs view
@@ -0,0 +1,98 @@+{-# LANGUAGE TemplateHaskell #-}++-- | Copyright: (c) 2021 berberman+-- SPDX-License-Identifier: MIT+-- Maintainer: berberman <berberman@yandex.com>+-- Stability: experimental+-- Portability: portable+--+-- CLI interface of nvfetcher+module NvFetcher.Options+ ( CLIOptions (..),+ cliOptionsParser,+ getCLIOptions,+ )+where++import Options.Applicative.Simple+import qualified Paths_nvfetcher as Paths++-- | Options for nvfetcher CLI+data CLIOptions = CLIOptions+ { outputPath :: FilePath,+ logPath :: Maybe FilePath,+ threads :: Int,+ retries :: Int,+ timing :: Bool,+ verbose :: Bool,+ target :: String+ }+ deriving (Show)++cliOptionsParser :: Parser CLIOptions+cliOptionsParser =+ CLIOptions+ <$> strOption+ ( long "output"+ <> short 'o'+ <> metavar "FILE"+ <> help "Path to output nix file"+ <> showDefault+ <> value "sources.nix"+ <> completer (bashCompleter "file")+ )+ <*> optional+ ( strOption+ ( long "changelog"+ <> short 'l'+ <> metavar "FILE"+ <> help "Dump version changes to a file"+ <> completer (bashCompleter "file")+ )+ )+ <*> option+ auto+ ( short 'j'+ <> metavar "NUM"+ <> help "Number of threads (0: detected number of processors)"+ <> value 0+ <> showDefault+ )+ <*> option+ auto+ ( short 'r'+ <> long "retry"+ <> metavar "NUM"+ <> help "Times to retry of some rules (nvchecker, prefetch, nix-instantiate, etc.)"+ <> value 3+ <> showDefault+ )+ <*> switch (long "timing" <> short 't' <> help "Show build time")+ <*> switch (long "verbose" <> short 'v' <> help "Verbose mode")+ <*> strArgument+ ( metavar "TARGET"+ <> help "Two targets are available: 1.build 2.clean"+ <> value "build"+ <> completer (listCompleter ["build", "clean"])+ <> showDefault+ )++version :: String+version = $(simpleVersion Paths.version)++-- | Parse nvfetcher CLI options+getCLIOptions :: Parser a -> IO a+getCLIOptions parser = do+ (opts, ()) <-+ simpleOptions+ version+ "nvfetcher - generate nix sources expr for the latest version of packages"+ ( unlines+ [ "It's important to keep _build dir.",+ "If you change any field of an existing package, you may have to run target \"clean\" to invalidate the databse,",+ "making sure the consistency of our build system."+ ]+ )+ parser+ empty+ pure opts
src/NvFetcher/PackageSet.hs view
@@ -7,9 +7,11 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE TypeOperators #-} {-# LANGUAGE UndecidableInstances #-}+{-# LANGUAGE ViewPatterns #-} -- | Copyright: (c) 2021 berberman -- SPDX-License-Identifier: MIT@@ -42,16 +44,22 @@ -- ** Two-in-one functions fromGitHub, fromGitHub',+ fromGitHubTag,+ fromGitHubTag', fromPypi, -- ** Version sources sourceGitHub,+ sourceGitHubTag, sourceGit,+ sourceGit', sourcePypi, sourceAur, sourceArchLinux, sourceManual, sourceRepology,+ sourceWebpage,+ sourceHttpHeader, -- ** Fetchers fetchGitHub,@@ -62,6 +70,11 @@ fetchGit', fetchUrl, + -- * Addons+ extractSource,+ hasCargoLock,+ tweakVersion,+ -- ** Miscellaneous Prod, Member,@@ -74,6 +87,7 @@ (.~), (%~), (^.),+ (?~), module NvFetcher.Types.Lens, ) where@@ -81,13 +95,15 @@ import Control.Monad.Free import Control.Monad.IO.Class import Data.Coerce (coerce)+import Data.Default (def) import Data.Kind (Constraint, Type) import Data.Map.Strict as HMap-import Data.Maybe (isJust)+import Data.Maybe (fromMaybe, isJust) import Data.Text (Text) import GHC.TypeLits import Lens.Micro import NvFetcher.NixFetcher+-- import NvFetcher.PostFetcher import NvFetcher.Types import NvFetcher.Types.Lens @@ -117,10 +133,12 @@ -- | Add a package to package set newPackage :: PackageName ->- VersionSource ->+ NvcheckerQ -> PackageFetcher ->+ PackageExtractSrc ->+ Maybe PackageCargoFilePath -> PackageSet ()-newPackage name source fetcher = liftF $ NewPackage (Package name source fetcher) ()+newPackage name source fetcher extract cargo = liftF $ NewPackage (Package name source fetcher extract cargo) () -- | Add a list of packages into package set purePackageSet :: [Package] -> PackageSet ()@@ -160,6 +178,18 @@ instance TypeError (ShowType x :<>: 'Text " is undefined") => Member x '[] where proj = undefined +class OptionalMember (a :: Type) (r :: [Type]) where+ projMaybe :: Prod r -> Maybe a++instance {-# OVERLAPPING #-} NotElem x xs => OptionalMember x (x ': xs) where+ projMaybe (Cons x _) = Just x++instance OptionalMember x xs => OptionalMember x (_y ': xs) where+ projMaybe (Cons _ r) = projMaybe r++instance OptionalMember x '[] where+ projMaybe Nil = Nothing+ -- | Constraint for producing error messages type family NotElem (x :: Type) (xs :: [Type]) :: Constraint where NotElem x (x ': xs) = TypeError (ShowType x :<>: 'Text " is defined more than one times")@@ -172,7 +202,16 @@ class PkgDSL f where new :: f PackageName -> f (Prod '[PackageName]) andThen :: f (Prod r) -> f a -> f (Prod (a ': r))- end :: (Member PackageName r, Member VersionSource r, Member PackageFetcher r) => f (Prod r) -> f ()+ end ::+ ( Member PackageName r,+ Member VersionSource r,+ Member PackageFetcher r,+ OptionalMember PackageExtractSrc r,+ OptionalMember PackageCargoFilePath r,+ OptionalMember NvcheckerOptions r+ ) =>+ f (Prod r) ->+ f () instance PkgDSL PackageSet where new e = do@@ -184,7 +223,12 @@ pure $ Cons x p end e = do p <- e- newPackage (proj p) (proj p) (proj p)+ newPackage+ (proj p)+ (NvcheckerQ (proj p) (fromMaybe def (projMaybe p)))+ (proj p)+ (fromMaybe (PackageExtractSrc []) $ projMaybe p)+ (projMaybe p) -- | 'PkgDSL' version of 'newPackage' --@@ -196,7 +240,10 @@ define :: ( Member PackageName r, Member VersionSource r,- Member PackageFetcher r+ Member PackageFetcher r,+ OptionalMember PackageExtractSrc r,+ OptionalMember PackageCargoFilePath r,+ OptionalMember NvcheckerOptions r ) => PackageSet (Prod r) -> PackageSet ()@@ -235,6 +282,22 @@ (Prod (PackageFetcher : VersionSource : r)) fromGitHub' e p@(owner, repo, _) = fetchGitHub' (sourceGitHub e (owner, repo)) p +-- | A synonym of 'fetchGitHub' and 'sourceGitHubTag'+fromGitHubTag ::+ PackageSet (Prod r) ->+ (Text, Text, ListOptions -> ListOptions) ->+ PackageSet+ (Prod (PackageFetcher : VersionSource : r))+fromGitHubTag e (owner, repo, f) = fromGitHubTag' e (owner, repo, f, id)++-- | A synonym of 'fetchGitHub'' and 'sourceGitHubTag'+fromGitHubTag' ::+ PackageSet (Prod r) ->+ (Text, Text, ListOptions -> ListOptions, NixFetcher Fresh -> NixFetcher Fresh) ->+ PackageSet+ (Prod (PackageFetcher : VersionSource : r))+fromGitHubTag' e (owner, repo, fv, ff) = fetchGitHub' (sourceGitHubTag e (owner, repo, fv)) (owner, repo, ff)+ -- | A synonym of 'fetchPypi' and 'sourcePypi' fromPypi :: PackageSet (Prod r) ->@@ -253,14 +316,30 @@ PackageSet (Prod (VersionSource : r)) sourceGitHub e (owner, repo) = src e $ GitHubRelease owner repo +-- | This package follows the a tag from github+sourceGitHubTag ::+ PackageSet (Prod r) ->+ -- | owner, repo, and nvchecker list options to find the target tag+ (Text, Text, ListOptions -> ListOptions) ->+ PackageSet (Prod (VersionSource : r))+sourceGitHubTag e (owner, repo, f) = src e $ GitHubTag owner repo $ f def+ -- | This package follows the latest git commit sourceGit :: PackageSet (Prod r) -> -- | git url Text -> PackageSet (Prod (VersionSource : r))-sourceGit e _vurl = src e Git {..}+sourceGit e _vurl = src e $ Git _vurl def +-- | Similar to 'sourceGit', but allows to specify branch+sourceGit' ::+ PackageSet (Prod r) ->+ -- | git url and branch+ (Text, Text) ->+ PackageSet (Prod (VersionSource : r))+sourceGit' e (_vurl, coerce . Just -> _vbranch) = src e $ Git {..}+ -- | This package follows the latest pypi release sourcePypi :: PackageSet (Prod r) ->@@ -300,6 +379,22 @@ PackageSet (Prod (VersionSource : r)) sourceRepology e (project, repo) = src e $ Repology project repo +-- | This package follows a version extracted from web page+sourceWebpage ::+ PackageSet (Prod r) ->+ -- | web page url, regex, and list options+ (Text, Text, ListOptions -> ListOptions) ->+ PackageSet (Prod (VersionSource : r))+sourceWebpage e (_vurl, _regex, f) = src e $ Webpage _vurl _regex $ f def++-- | This package follows a version extracted from http header+sourceHttpHeader ::+ PackageSet (Prod r) ->+ -- | url of the http request, regex, and list options+ (Text, Text, ListOptions -> ListOptions) ->+ PackageSet (Prod (VersionSource : r))+sourceHttpHeader e (_vurl, _regex, f) = src e $ HttpHeader _vurl _regex $ f def+ -------------------------------------------------------------------------------- -- | This package is fetched from a github repo@@ -372,3 +467,26 @@ fetchUrl e f = fetch e (urlFetcher . f) --------------------------------------------------------------------------------++-- | Extract files from fetched package source+extractSource ::+ PackageSet (Prod r) ->+ [FilePath] ->+ PackageSet (Prod (PackageExtractSrc : r))+extractSource = (. pure . PackageExtractSrc) . andThen++-- | Run 'FetchRustGitDependencies' given the path to @Cargo.lock@+--+-- The lock file will be extracted as well.+hasCargoLock ::+ PackageSet (Prod r) ->+ FilePath ->+ PackageSet (Prod (PackageCargoFilePath : r))+hasCargoLock = (. pure . PackageCargoFilePath) . andThen++-- | Set 'NvcheckerOptions' for a package, which can tweak the version number we obtain+tweakVersion ::+ PackageSet (Prod r) ->+ (NvcheckerOptions -> NvcheckerOptions) ->+ PackageSet (Prod (NvcheckerOptions : r))+tweakVersion = (. pure . ($ def)) . andThen
− src/NvFetcher/ShakeExtras.hs
@@ -1,80 +0,0 @@-{-# LANGUAGE LambdaCase #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE TypeApplications #-}---- | Copyright: (c) 2021 berberman--- SPDX-License-Identifier: MIT--- Maintainer: berberman <berberman@yandex.com>--- Stability: experimental--- Portability: portable------ This module is about global information we use in rules.-module NvFetcher.ShakeExtras- ( -- * Types- ShakeExtras (..),- initShakeExtras,- getShakeExtras,-- -- * Packages- lookupPackage,- getAllPackageKeys,- isPackageKeyTarget,-- -- * Version changes- recordVersionChange,- getVersionChanges,- )-where--import Control.Concurrent.Extra-import Data.Map.Strict (Map)-import qualified Data.Map.Strict as Map-import Development.Shake-import NvFetcher.Types---- | Values we use during the build. It's stored in 'shakeExtra'-data ShakeExtras = ShakeExtras- { versionChanges :: Var [VersionChange],- targetPackages :: Map PackageKey Package- }---- | Get our values from shake-getShakeExtras :: Action ShakeExtras-getShakeExtras =- getShakeExtra @ShakeExtras >>= \case- Just x -> pure x- _ -> fail "ShakeExtras is missing!"---- | Create an empty 'ShakeExtras' from packages to build-initShakeExtras :: Map PackageKey Package -> IO ShakeExtras-initShakeExtras targetPackages = do- versionChanges <- newVar mempty- pure ShakeExtras {..}---- | Get keys of all packages to build-getAllPackageKeys :: Action [PackageKey]-getAllPackageKeys = do- ShakeExtras {..} <- getShakeExtras- pure $ Map.keys targetPackages---- | Find a package given its key-lookupPackage :: PackageKey -> Action (Maybe Package)-lookupPackage key = do- ShakeExtras {..} <- getShakeExtras- pure $ Map.lookup key targetPackages---- | Check if we need build this package-isPackageKeyTarget :: PackageKey -> Action Bool-isPackageKeyTarget k = Map.member k . targetPackages <$> getShakeExtras---- | Record version change of a package-recordVersionChange :: PackageName -> Maybe Version -> Version -> Action ()-recordVersionChange vcName vcOld vcNew = do- ShakeExtras {..} <- getShakeExtras- liftIO $ modifyVar_ versionChanges (pure . (++ [VersionChange {..}]))---- | Get version changes since the last run-getVersionChanges :: Action [VersionChange]-getVersionChanges = do- ShakeExtras {..} <- getShakeExtras- liftIO $ readVar versionChanges
src/NvFetcher/Types.hs view
@@ -23,24 +23,39 @@ module NvFetcher.Types ( -- * Common types Version (..),- SHA256 (..),+ Checksum (..),+ Branch (..), NixExpr, VersionChange (..), WithPackageKey (..),- Core (..), -- * Nvchecker types+ VersionSortMethod (..),+ ListOptions (..), VersionSource (..),- NvcheckerResult (..),+ NvcheckerA (..),+ NvcheckerQ (..),+ NvcheckerOptions (..), -- * Nix fetcher types NixFetcher (..),- Prefetch (..),- PrefetchResult,+ FetchResult,+ FetchStatus (..), + -- * ExtractSrc Types+ ExtractSrcQ (..),++ -- * FetchRustGitDeps types+ FetchRustGitDepsQ (..),++ -- * Core types+ Core (..),+ -- * Package types PackageName, PackageFetcher,+ PackageExtractSrc (..),+ PackageCargoFilePath (..), Package (..), PackageKey (..), )@@ -48,6 +63,8 @@ import qualified Data.Aeson as A import Data.Coerce (coerce)+import Data.Default+import Data.HashMap.Strict (HashMap) import Data.Maybe (fromMaybe) import Data.String (IsString) import Data.Text (Text)@@ -64,12 +81,18 @@ deriving stock (Typeable, Generic) deriving anyclass (Hashable, Binary, NFData) --- | SHA 256 sum-newtype SHA256 = SHA256 Text- deriving newtype (Show, Eq)+-- | Check sum, sha256, sri or base32, etc.+newtype Checksum = Checksum Text+ deriving newtype (Show, Eq, Ord) deriving stock (Typeable, Generic) deriving anyclass (Hashable, Binary, NFData) +-- | Git branch ('Nothing': master)+newtype Branch = Branch (Maybe Text)+ deriving newtype (Show, Eq, Ord, Default)+ deriving stock (Typeable, Generic)+ deriving anyclass (Hashable, Binary, NFData)+ -- | Version change of a package -- -- >>> VersionChange "foo" Nothing "2.3.3"@@ -93,90 +116,154 @@ -------------------------------------------------------------------------------- --- | The input of nvchecker+data VersionSortMethod = ParseVersion | Vercmp+ deriving (Typeable, Eq, Ord, Enum, Generic, Hashable, Binary, NFData)++instance Show VersionSortMethod where+ show = \case+ ParseVersion -> "parse_version"+ Vercmp -> "vercmp"++instance Default VersionSortMethod where+ def = ParseVersion++-- | Filter-like configuration for some version sources.+-- See <https://nvchecker.readthedocs.io/en/latest/usage.html#list-options> for details.+data ListOptions = ListOptions+ { _includeRegex :: Maybe Text,+ _excludeRegex :: Maybe Text,+ _sortVersionKey :: Maybe VersionSortMethod,+ _ignored :: Maybe Text+ }+ deriving (Show, Typeable, Eq, Ord, Generic, Hashable, Binary, NFData, Default)++-- | Configuration available for evey version sourece.+-- See <https://nvchecker.readthedocs.io/en/latest/usage.html#global-options> for details.+data NvcheckerOptions = NvcheckerOptions+ { _stripPrefix :: Maybe Text,+ _fromPattern :: Maybe Text,+ _toPattern :: Maybe Text+ }+ deriving (Show, Typeable, Eq, Ord, Generic, Hashable, Binary, NFData, Default)++-- | Upstream version source for nvchecker to check data VersionSource = GitHubRelease {_owner :: Text, _repo :: Text}- | Git {_vurl :: Text}+ | GitHubTag {_owner :: Text, _repo :: Text, _listOptions :: ListOptions}+ | Git {_vurl :: Text, _vbranch :: Branch} | Pypi {_pypi :: Text} | ArchLinux {_archpkg :: Text} | Aur {_aur :: Text} | Manual {_manual :: Text} | Repology {_repology :: Text, _repo :: Text}+ | Webpage {_vurl :: Text, _regex :: Text, _listOptions :: ListOptions}+ | HttpHeader {_vurl :: Text, _regex :: Text, _listOptions :: ListOptions} deriving (Show, Typeable, Eq, Ord, Generic, Hashable, Binary, NFData) +-- | The input of nvchecker+data NvcheckerQ = NvcheckerQ VersionSource NvcheckerOptions+ deriving (Show, Typeable, Eq, Ord, Generic, Hashable, Binary, NFData)+ -- | The result of running nvchecker-data NvcheckerResult = NvcheckerResult+data NvcheckerA = NvcheckerA { nvNow :: Version, -- | nvchecker doesn't give this value, but shake restores it from last run nvOld :: Maybe Version } deriving (Show, Typeable, Eq, Generic, Hashable, Binary, NFData) -instance A.FromJSON NvcheckerResult where+instance A.FromJSON NvcheckerA where parseJSON = A.withObject "NvcheckerResult" $ \o ->- NvcheckerResult <$> o A..: "version" <*> pure Nothing+ NvcheckerA <$> o A..: "version" <*> pure Nothing -type instance RuleResult VersionSource = NvcheckerResult+type instance RuleResult NvcheckerQ = NvcheckerA -------------------------------------------------------------------------------- -- | If the package is prefetched, then we can obtain the SHA256-data NixFetcher (k :: Prefetch)+data NixFetcher (k :: FetchStatus) = FetchGit { _furl :: Text, _rev :: Version,- _branch :: Maybe Text, _deepClone :: Bool, _fetchSubmodules :: Bool, _leaveDotGit :: Bool,- _sha256 :: PrefetchResult k+ _sha256 :: FetchResult k }- | FetchUrl {_furl :: Text, _sha256 :: PrefetchResult k}+ | FetchUrl {_furl :: Text, _sha256 :: FetchResult k} deriving (Typeable, Generic) --- | Prefetch status-data Prefetch = Fresh | Prefetched+-- | Fetch status+data FetchStatus = Fresh | Fetched -- | Prefetched fetchers hold hashes-type family PrefetchResult (k :: Prefetch) where- PrefetchResult Fresh = ()- PrefetchResult Prefetched = SHA256+type family FetchResult (k :: FetchStatus) where+ FetchResult Fresh = ()+ FetchResult Fetched = Checksum -type instance RuleResult (NixFetcher Fresh) = NixFetcher Prefetched+type instance RuleResult (NixFetcher Fresh) = NixFetcher Fetched -deriving instance Show (PrefetchResult k) => Show (NixFetcher k)+deriving instance Show (FetchResult k) => Show (NixFetcher k) -deriving instance Eq (PrefetchResult k) => Eq (NixFetcher k)+deriving instance Eq (FetchResult k) => Eq (NixFetcher k) -deriving instance Ord (PrefetchResult k) => Ord (NixFetcher k)+deriving instance Ord (FetchResult k) => Ord (NixFetcher k) -deriving instance Hashable (PrefetchResult k) => Hashable (NixFetcher k)+deriving instance Hashable (FetchResult k) => Hashable (NixFetcher k) -deriving instance Binary (PrefetchResult k) => Binary (NixFetcher k)+deriving instance Binary (FetchResult k) => Binary (NixFetcher k) -deriving instance NFData (PrefetchResult k) => NFData (NixFetcher k)+deriving instance NFData (FetchResult k) => NFData (NixFetcher k) -------------------------------------------------------------------------------- +-- | Extract file contents from package source+-- e.g. @Cargo.lock@+data ExtractSrcQ = ExtractSrcQ (NixFetcher Fetched) [FilePath]+ deriving (Show, Eq, Ord, Hashable, NFData, Binary, Typeable, Generic)++type instance RuleResult ExtractSrcQ = HashMap FilePath Text++--------------------------------------------------------------------------------++-- | Fetch @outputHashes@ for git dependencies in @Cargo.lock@.+-- See <https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/rust.section.md#importing-a-cargolock-file> for details.+-- We need fetched source and the file path to @Cargo.lock@.+data FetchRustGitDepsQ = FetchRustGitDepsQ (NixFetcher Fetched) FilePath+ deriving (Show, Eq, Ord, Hashable, NFData, Binary, Typeable, Generic)++-- | @outputHashes@, a mapping from nameVer -> output hash+type instance RuleResult FetchRustGitDepsQ = HashMap Text Checksum++--------------------------------------------------------------------------------+ -- | Package name, used in generating nix expr type PackageName = Text -- | How to create package source fetcher given a version type PackageFetcher = Version -> NixFetcher Fresh +newtype PackageExtractSrc = PackageExtractSrc [FilePath]++newtype PackageCargoFilePath = PackageCargoFilePath FilePath+ -- | A package is defined with: -- -- 1. its name -- 2. how to track its version -- 3. how to fetch it as we have the version+-- 4. optional file paths to extract (dump to generated nix expr)+-- 5. @Cargo.lock@ path (if it's a rust package) -- -- /INVARIANT: 'Version' passed to 'PackageFetcher' MUST be used textually,/ -- /i.e. can only be concatenated with other strings,/ -- /in case we can't check the equality between fetcher functions./ data Package = Package { _pname :: PackageName,- _pversion :: VersionSource,- _pfetcher :: PackageFetcher+ _pversion :: NvcheckerQ,+ _pfetcher :: PackageFetcher,+ _pextract :: PackageExtractSrc,+ _pcargo :: Maybe PackageCargoFilePath } -- | Package key is the name of a package.@@ -186,13 +273,13 @@ deriving stock (Typeable, Generic) deriving anyclass (Hashable, Binary, NFData) +--------------------------------------------------------------------------------+ -- | The key type of nvfetcher rule. See "NvFetcher.Core" data Core = Core deriving (Eq, Show, Ord, Typeable, Generic, Hashable, Binary, NFData) type instance RuleResult Core = NixExpr---------------------------------------------------------------------------------- -- | Decorate a rule's key with 'PackageKey' newtype WithPackageKey k = WithPackageKey (k, PackageKey)
src/NvFetcher/Types/Lens.hs view
@@ -1,3 +1,4 @@+{-# LANGUAGE DataKinds #-} {-# LANGUAGE TemplateHaskell #-} -- | Copyright: (c) 2021 berberman@@ -10,6 +11,10 @@ import Lens.Micro.TH import NvFetcher.Types++makeLenses ''ListOptions++makeLenses ''NvcheckerOptions makeLenses ''VersionSource
+ src/NvFetcher/Types/ShakeExtras.hs view
@@ -0,0 +1,88 @@+{-# LANGUAGE LambdaCase #-}+{-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE TypeApplications #-}++-- | Copyright: (c) 2021 berberman+-- SPDX-License-Identifier: MIT+-- Maintainer: berberman <berberman@yandex.com>+-- Stability: experimental+-- Portability: portable+--+-- This module is about global information we use in rules.+module NvFetcher.Types.ShakeExtras+ ( -- * Types+ ShakeExtras (..),+ initShakeExtras,+ getShakeExtras,++ -- * Packages+ lookupPackage,+ getAllPackageKeys,+ isPackageKeyTarget,++ -- * Version changes+ recordVersionChange,+ getVersionChanges,++ -- * Retries+ withRetries,+ )+where++import Control.Concurrent.Extra+import Data.Map.Strict (Map)+import qualified Data.Map.Strict as Map+import Development.Shake+import NvFetcher.Types++-- | Values we use during the build. It's stored in 'shakeExtra'+data ShakeExtras = ShakeExtras+ { versionChanges :: Var [VersionChange],+ targetPackages :: Map PackageKey Package,+ retries :: Int+ }++-- | Get our values from shake+getShakeExtras :: Action ShakeExtras+getShakeExtras =+ getShakeExtra @ShakeExtras >>= \case+ Just x -> pure x+ _ -> fail "ShakeExtras is missing!"++-- | Create an empty 'ShakeExtras' from packages to build and times to retry for each rule+initShakeExtras :: Map PackageKey Package -> Int -> IO ShakeExtras+initShakeExtras targetPackages retries = do+ versionChanges <- newVar mempty+ pure ShakeExtras {..}++-- | Get keys of all packages to build+getAllPackageKeys :: Action [PackageKey]+getAllPackageKeys = do+ ShakeExtras {..} <- getShakeExtras+ pure $ Map.keys targetPackages++-- | Find a package given its key+lookupPackage :: PackageKey -> Action (Maybe Package)+lookupPackage key = do+ ShakeExtras {..} <- getShakeExtras+ pure $ Map.lookup key targetPackages++-- | Check if we need build this package+isPackageKeyTarget :: PackageKey -> Action Bool+isPackageKeyTarget k = Map.member k . targetPackages <$> getShakeExtras++-- | Record version change of a package+recordVersionChange :: PackageName -> Maybe Version -> Version -> Action ()+recordVersionChange vcName vcOld vcNew = do+ ShakeExtras {..} <- getShakeExtras+ liftIO $ modifyVar_ versionChanges (pure . (++ [VersionChange {..}]))++-- | Get version changes since the last run+getVersionChanges :: Action [VersionChange]+getVersionChanges = do+ ShakeExtras {..} <- getShakeExtras+ liftIO $ readVar versionChanges++-- | Run an action, retry at most 'retries' times if it throws an exception+withRetries :: Action a -> Action a+withRetries a = getShakeExtras >>= \ShakeExtras {..} -> actionRetry retries a
src/NvFetcher/Utils.hs view
@@ -3,9 +3,18 @@ import Data.Binary import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as LBS+import Data.Text (Text)+import qualified Data.Text as T+import Development.Shake encode' :: Binary a => a -> BS.ByteString encode' = BS.concat . LBS.toChunks . encode decode' :: Binary a => BS.ByteString -> a decode' = decode . LBS.fromChunks . pure++asString :: Text -> Text+asString = T.pack . show++getShakeDir :: Action FilePath+getShakeDir = shakeFiles <$> getShakeOptions