nvfetcher 0.4.0.0 → 0.5.0.0
raw patch · 27 files changed
+1446/−505 lines, 27 filesdep +aeson-prettydep +prettyprinterdep +regex-tdfadep ~aeson
Dependencies added: aeson-pretty, prettyprinter, regex-tdfa
Dependency ranges changed: aeson
Files
- CHANGELOG.md +28/−0
- Main_example.hs +13/−9
- README.md +90/−74
- app/Config.hs +55/−17
- app/Config/PackageFetcher.hs +36/−3
- app/Config/VersionSource.hs +31/−2
- app/Main.hs +3/−5
- nvfetcher.cabal +8/−4
- src/NvFetcher.hs +114/−87
- src/NvFetcher/Config.hs +38/−0
- src/NvFetcher/Core.hs +76/−129
- src/NvFetcher/ExtractSrc.hs +8/−5
- src/NvFetcher/FetchRustGitDeps.hs +4/−2
- src/NvFetcher/NixExpr.hs +133/−7
- src/NvFetcher/NixFetcher.hs +62/−27
- src/NvFetcher/Nvchecker.hs +83/−39
- src/NvFetcher/Options.hs +38/−13
- src/NvFetcher/PackageSet.hs +42/−18
- src/NvFetcher/Types.hs +350/−27
- src/NvFetcher/Types/Lens.hs +1/−1
- src/NvFetcher/Types/ShakeExtras.hs +93/−11
- src/NvFetcher/Utils.hs +32/−5
- test/CheckVersionSpec.hs +65/−5
- test/FetchRustGitDepsSpec.hs +1/−1
- test/NixExprSpec.hs +23/−1
- test/PrefetchSpec.hs +10/−6
- test/Utils.hs +9/−7
CHANGELOG.md view
@@ -1,5 +1,33 @@ # Revision history for nvfetcher +## 0.5.0.0++There have been many significant changes since the last release.+**Starting from this version, nvfetcher no longer requires shake database for each project,+in other words, there is no need to commit the database in git or share it between machines.**+Also, a machine-readable `generated.json` will be generated for convenience.++### Migration++The option in TOML configuration `cargo_lock` (string) was changed to `cargo_locks` (list of strings), since now nvfetcher supports handling multi-cargo locks.++* Use `fetchFromGitHub` as the GitHub fetcher (introduces [`nix-prefetch`](https://github.com/msteen/nix-prefetch))+* Add [cmd](https://nvchecker.readthedocs.io/en/latest/usage.html#find-with-a-command) version source+* Support pinning a package+* Tweak src name to extract .vsx file properly+* Add option --filter to specify packages to update+* Fix missing `argActionAfterBuild`+* Add one shot nvchecker rule independent of package definition+* Don't cache generated nix exprs in shake database+* Add `fetchTarball`+* Produce parser readable `generated.json`+* Internalize shake database+* Validate config before decoding+* Extract `Config` from `Arg` and save it to shake extra+* Breakdown `Args` to provide a more concise API+* Support multi-cargo lock files+* Pretty print rules in command line+ ## 0.4.0.0 * Rename `_build` to `_sources`
Main_example.hs view
@@ -15,29 +15,33 @@ define $ package "feeluown-core" `fromPypi` "feeluown" - define $ package "qliveplayer" `fromGitHub'` ("IsoaSFlus", "QLivePlayer", fetchSubmodules .~ True)- define $ package "apple-emoji" `sourceManual` "0.0.0.20200413" `fetchUrl` const- "https://github.com/samuelngs/apple-emoji-linux/releases/download/latest/AppleColorEmoji.ttf"+ "https://github.com/samuelngs/apple-emoji-linux/releases/download/alpha-release-v1.0.0/AppleColorEmoji.ttf" define $ package "nvfetcher-git" `sourceGit` "https://github.com/berberman/nvfetcher" `fetchGitHub` ("berberman", "nvfetcher") - define $- package "vim"- `sourceWebpage` ("http://ftp.vim.org/pub/vim/patches/7.3/", "7\\.3\\.\\d+", id)- `fetchGitHub` ("vim", "vim")- `tweakVersion` (\v -> v & fromPattern ?~ "(.+)" & toPattern ?~ "v\\1")+ -- define $+ -- package "vim"+ -- `sourceWebpage` ("http://ftp.vim.org/pub/vim/patches/7.3/", "7\\.3\\.\\d+", id)+ -- `fetchGitHub` ("vim", "vim")+ -- `tweakVersion` (\v -> v & fromPattern ?~ "(.+)" & toPattern ?~ "v\\1") define $ package "rust-git-dependency-example" `sourceManual` "8a5f37a8f80a3b05290707febf57e88661cee442" `fetchGit` "https://gist.github.com/NickCao/6c4dbc4e15db5da107de6cdb89578375"- `hasCargoLock` "Cargo.lock"+ `hasCargoLocks` ["Cargo.lock"] define $ package "vscode-LiveServer" `fromOpenVsx` ("ritwickdey", "LiveServer")++ define $+ package "revda"+ `sourceGit` "https://github.com/THMonster/Revda"+ `fetchGitHub'` ("THMonster", "Revda", fetchSubmodules .~ True)+ `hasCargoLocks` ["dmlive/Cargo.lock", "dmlive/tars-stream/Cargo.lock"]
README.md view
@@ -14,58 +14,61 @@ ```toml # nvfetcher.toml-[feeluown-core]+[feeluown] src.pypi = "feeluown" fetch.pypi = "feeluown"--[qliveplayer]-src.github = "IsoaSFlus/QLivePlayer"-fetch.github = "IsoaSFlus/QLivePlayer"-git.fetchSubmodules = true ``` -it can create `sources.nix` like:+it will create `_sources/generated.nix`: ```nix-# sources.nix-{ fetchgit, fetchurl }:+{ fetchgit, fetchurl, fetchFromGitHub }: {- feeluown-core = {- pname = "feeluown-core";- version = "3.7.7";+ feeluown = {+ pname = "feeluown";+ version = "3.8.2"; src = fetchurl {- sha256 = "06d3j39ff9znqxkhp9ly81lcgajkhg30hyqxy2809yn23xixg3x2";- url = "https://pypi.io/packages/source/f/feeluown/feeluown-3.7.7.tar.gz";- };- };- qliveplayer = {- pname = "qliveplayer";- version = "3.22.1";- src = fetchgit {- url = "https://github.com/IsoaSFlus/QLivePlayer";- rev = "3.22.1";- fetchSubmodules = true;- deepClone = false;- leaveDotGit = false;- sha256 = "00zqg28q5xrbgql0kclgkhd15fc02qzsrvi0qg8lg3qf8a53v263";+ url = "https://pypi.io/packages/source/f/feeluown/feeluown-3.8.2.tar.gz";+ sha256 = "sha256-V2yzpkmjRkipZOvQGB2mYRhiiEly6QPrTOMJ7BmyWBQ="; }; }; } ``` +and `_sources/generated.json`:++```json+{+ "feeluown": {+ "pinned": false,+ "cargoLocks": null,+ "name": "feeluown-core",+ "version": "3.8.2",+ "passthru": null,+ "src": {+ "url": "https://pypi.io/packages/source/f/feeluown/feeluown-3.8.2.tar.gz",+ "name": null,+ "type": "url",+ "sha256": "sha256-V2yzpkmjRkipZOvQGB2mYRhiiEly6QPrTOMJ7BmyWBQ="+ },+ "extract": null,+ "rustGitDeps": null+ }+}+```+ We tell nvfetcher how to get the latest version number of packages and how to fetch their sources given version numbers,-and nvfetcher will help us keep their version and prefetched SHA256 sums up-to-date, stored in `_sources/generated.nix`.-Shake will handle necessary rebuilds as long as you keep `_sources` directory -- we check versions of packages during each run, but only prefetch them when needed.+and nvfetcher will help us keep their version and prefetched SHA256 sums up-to-date, producing ready-to-use nix expressions in `_sources/generated.nix`.+Nvfetcher reads `generated.json` to produce version change message, such as `feeluown: 3.8.1 → 3.8.2`.+We always check versions of packages during each run, but only do prefetch and further operations when needed. ### Live examples How to use the generated sources file? Here are several examples: -* [DevOS](https://github.com/divnix/devos/tree/main/pkgs) - Packages are defined in TOML--* My [flakes repo](https://github.com/berberman/flakes) - Packages are defined in eDSL+- My [flakes repo](https://github.com/berberman/flakes) -* Nick Cao's [flakes repo](https://gitlab.com/NickCao/flakes/-/tree/master/pkgs) - Packages are defined in TOML+- Nick Cao's [flakes repo](https://gitlab.com/NickCao/flakes/-/tree/master/pkgs) ## Installation @@ -103,9 +106,10 @@ Available options: --version Show version --help Show this help text- -o,--build-dir FILE Directory that nvfetcher puts artifacts to+ -o,--build-dir DIR Directory that nvfetcher puts artifacts to (default: "_sources")- --commit-changes `git commit` changes in this run (with shake db)+ --commit-changes `git commit` build dir with version changes as commit+ message -l,--changelog FILE Dump version changes to a file -j NUM Number of threads (0: detected number of processors) (default: 0)@@ -113,88 +117,100 @@ nix-instantiate, etc.) (default: 3) -t,--timing Show build time -v,--verbose Verbose mode+ -f,--filter REGEX Regex to filter packages to be updated TARGET Two targets are available: 1.build 2.clean (default: "build") -c,--config FILE Path to nvfetcher TOML config (default: "nvfetcher.toml") ``` -Each *package* corresponds to a TOML table, whose name is encoded as table key, with+Each _package_ corresponds to a TOML table, whose name is encoded as table key, with two required fields and three optional fields in each table. You can find an example of the configuration file, see [`nvfetcher_example.toml`](nvfetcher_example.toml). #### Nvchecker Version source -- how do we track upstream version updates?-* `src.github = owner/repo` - the latest github release-* `src.github_tag = owner/repo` - the max github tag, usually used with list options (see below)-* `src.pypi = pypi_name` - the latest pypi release-* `src.git = git_url` (and an optional `src.branch = git_branch`) - **the latest commit** of a repo-* `src.archpkg = archlinux_pkg_name` -- the latest version of an archlinux package-* `src.aur = aur_pkg_name` -- the latest version of an aur package-* `src.manual = v` -- a fixed version, which never updates-* `src.repology = project:repo` -- the latest version from repology-* `src.webpage = web_url` and `src.regex` -- a string in webpage that matches with regex-* `src.httpheader = request_url` and `src.regex` -- a string in http header that matches with regex-* `src.openvsx = publisher.ext_name` -- the latest version of a vscode extension from open vsx-* `src.vsmarketplace = publisher.ext_name` -- the latest version of a vscode extension from vscode marketplace +- `src.github = owner/repo` - the latest github release+- `src.github_tag = owner/repo` - the max github tag, usually used with list options (see below)+- `src.pypi = pypi_name` - the latest pypi release+- `src.git = git_url` (and an optional `src.branch = git_branch`) - **the latest commit** of a repo+- `src.archpkg = archlinux_pkg_name` -- the latest version of an archlinux package+- `src.aur = aur_pkg_name` -- the latest version of an aur package+- `src.manual = v` -- a fixed version, which never updates+- `src.repology = project:repo` -- the latest version from repology+- `src.webpage = web_url` and `src.regex` -- a string in webpage that matches with regex+- `src.httpheader = request_url` and `src.regex` -- a string in http header that matches with regex+- `src.openvsx = publisher.ext_name` -- the latest version of a vscode extension from open vsx+- `src.vsmarketplace = publisher.ext_name` -- the latest version of a vscode extension from vscode marketplace+- `src.cmd = cmd` -- the version from a shell command (e.g. `echo Meow`) Optional list options for some version sources (`src.github_tag`, `src.webpage`, and `src.httpheader`), see the corresponding [nvchecker documentation](https://nvchecker.readthedocs.io/en/latest/usage.html#list-options) for details.-* `src.include_regex`-* `src.exclude_regex`-* `src.sort_version_key`-* `src.ignored` +- `src.include_regex`+- `src.exclude_regex`+- `src.sort_version_key`+- `src.ignored` Optional global options for all kinds of version sources, see the corresponding [nvchecker documentation](https://nvchecker.readthedocs.io/en/latest/usage.html#global-options) for details. You can tweak obtained version number using this option, e.g. stripping the prefix `v` or transforming the result by regex.-* `src.prefix`-* `src.from_pattern`-* `src.to_pattern` +- `src.prefix`+- `src.from_pattern`+- `src.to_pattern`+ #### Nix fetcher How do we fetch the package source if we have the target version number? `$ver` is available in string, which will be set to the result of nvchecker. -* `fetch.github = owner/repo`-* `fetch.pypi = pypi_name`-* `fetch.git = git_url`-* `fetch.url = url`-* `fetch.openvsx = publisher.ext_name`-* `fetch.vsmarketplace = publisher.ext_name`-+- `fetch.github = owner/repo`+- `fetch.pypi = pypi_name`+- `fetch.git = git_url`+- `fetch.url = url`+- `fetch.openvsx = publisher.ext_name`+- `fetch.vsmarketplace = publisher.ext_name`+- `fetch.tarball = tarball_url` -Optional `nix-prefetch-git` config, which make sense only when the fetcher equals to `fetch.github` or `fetch.git`.-They can exist simultanesouly.- * `git.deepClone`- * `git.fetchSubmodules`- * `git.leaveDotGit`+Optional `nix-prefetch fetchgit` config, which make sense only when the fetcher equals to `fetch.github` or `fetch.git`.+They can exist simultaneously. +- `git.deepClone`+- `git.fetchSubmodules`+- `git.leaveDotGit` #### Extract src -Optional *extract src* config, files are extracted into build directory, and then read by `readFile` in generated nix expr.- * `extract = [ "file_1", "file_2", ...]` - file paths are relative to the source root+Optional _extract src_ config, files are extracted into build directory, and then read by `readFile` in generated nix expr. +- `extract = [ "file_1", "file_2", ...]` - file paths are relative to the source root+ #### Rust support `rustPlatform.buildRustPackage` now accepts an attribute [`cargoLock`](https://github.com/NixOS/nixpkgs/blob/master/doc/languages-frameworks/rust.section.md#importing-a-cargolock-file) to vendor dependencies from `Cargo.lock`,-so we can use this instead TOFU `cargoSha256` for Rust packageing. `nvfetcher` supports automating this process,-extracting the lock file to build and calculating `cargoLock.outputHashes`, as long as you set the config-* `cargo_lock = cargo_lock_path` - relative to the source root+so we can use this instead TOFU `cargoSha256` for Rust packaging. `nvfetcher` supports automating this process,+extracting the lock file to build and calculating `cargoLock.outputHashes`, as long as you set the config.+There can be many lock files in one source. +- `cargo_locks = [ "cargo_lock_path_1", "cargo_lock_path_2", ...]` - relative to the source root+ #### Passthru -*passthru* config, an additional set of attrs to be generated.+_passthru_ config, an additional set of attrs to be generated. - * `passthru = { k1 = "v1", k2 = "v2", ... }`+- `passthru = { k1 = "v1", k2 = "v2", ... }` +#### Pinned++If a package is pinned, we call nvchecker to check the new version iff there's no existing version.++- `pinned = true`+ ### Haskell library -nvfetcher itsetlf is a Haskell library as well, whereas the CLI program is just a trivial wrapper of the library.+nvfetcher itself is a Haskell library as well, whereas the CLI program is just a trivial wrapper of the library. You can create a Haskell program depending on it directly, by using the `runNvFetcher` entry point. In this case, we can define packages in Haskell language, getting rid of TOML constraints.
app/Config.hs view
@@ -1,6 +1,4 @@ {-# LANGUAGE DataKinds #-}-{-# LANGUAGE DeriveAnyClass #-}-{-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-}@@ -11,46 +9,70 @@ import Config.PackageFetcher import Config.VersionSource import Data.Coerce (coerce)+import Data.Either.Extra (mapLeft)+import qualified Data.HashMap.Strict as HMap+import Data.List (intersect) import Data.List.NonEmpty (NonEmpty ((:|))) import qualified Data.List.NonEmpty as NE+import Data.Text (Text)+import qualified Data.Text as T import NvFetcher.Types import Toml import Validation (validationToEither) -parseConfig :: TOML -> Either [Toml.TomlDecodeError] [Package]-parseConfig toml = go tables [] []+data PackageConfigParseError = TomlErrors [TomlDecodeError] | KeyConflicts [[Key]]++instance Semigroup PackageConfigParseError where+ TomlErrors e <> _ = TomlErrors e+ _ <> TomlErrors e = TomlErrors e+ KeyConflicts xs <> KeyConflicts ys = KeyConflicts $ xs <> ys++prettyPackageConfigParseError :: PackageConfigParseError -> Text+prettyPackageConfigParseError (TomlErrors e) = prettyTomlDecodeErrors e+prettyPackageConfigParseError (KeyConflicts xs) = "Skip parsing!\n" <> T.unlines ["Key conflict: " <> T.intercalate ", " [prettyKey k | k <- ks] | ks <- xs]++parseConfig :: TOML -> Either PackageConfigParseError [Package]+parseConfig toml = go tables Nothing [] where- go (Left errs : xs) se sp = go xs (se <> errs) sp+ go (Left errs : xs) (Just se) sp = go xs (Just (se <> errs)) sp+ go (Left errs : xs) Nothing sp = go xs (Just errs) sp go (Right x : xs) se sp = go xs se (x : sp)- go [] [] sp = Right sp- go [] se _ = Left se+ go [] Nothing sp = Right sp+ go [] (Just se) _ = Left se tables =- [ fmap (toPackage (coerce k)) $- validationToEither $- Toml.runTomlCodec packageConfigCodec v+ [ fmap (toPackage (coerce k)) $ validateKeys v >> mapLeft TomlErrors (validationToEither (Toml.runTomlCodec packageConfigCodec v)) | (Toml.unKey -> (Toml.unPiece -> k) :| _, v) <- Toml.toList $ Toml.tomlTables toml ] +validateKeys :: TOML -> Either PackageConfigParseError ()+validateKeys toml = if null e then Right () else Left $ foldl1 (<>) e+ where+ allKeys = HMap.keys $ Toml.tomlPairs toml+ go xs = let intersection = xs `intersect` allKeys in if length intersection > 1 then intersection else []+ e = [KeyConflicts [t] | k <- [versionSourceKeys, fetcherKeys], let t = go k, not $ null t]+ -------------------------------------------------------------------------------- data PackageConfig = PackageConfig { pcVersionSource :: VersionSource, pcFetcher :: PackageFetcher, pcExtractFiles :: Maybe PackageExtractSrc,- pcCargoLockPath :: Maybe PackageCargoFilePath,+ pcCargoLockFiles :: Maybe PackageCargoLockFiles, pcNvcheckerOptions :: NvcheckerOptions,- pcPassthru :: PackagePassthru+ pcPassthru :: PackagePassthru,+ pcUseStale :: UseStaleVersion } toPackage :: PackageKey -> PackageConfig -> Package toPackage k PackageConfig {..} = Package (coerce k)- (NvcheckerQ pcVersionSource pcNvcheckerOptions)+ (CheckVersion pcVersionSource pcNvcheckerOptions) pcFetcher pcExtractFiles- pcCargoLockPath+ pcCargoLockFiles pcPassthru+ pcUseStale packageConfigCodec :: TomlCodec PackageConfig packageConfigCodec =@@ -58,9 +80,10 @@ <$> versionSourceCodec .= pcVersionSource <*> fetcherCodec .= pcFetcher <*> extractFilesCodec .= pcExtractFiles- <*> cargoLockPathCodec .= pcCargoLockPath+ <*> cargoLockPathCodec .= pcCargoLockFiles <*> nvcheckerOptionsCodec .= pcNvcheckerOptions <*> passthruCodec .= pcPassthru+ <*> pinnedCodec .= pcUseStale -------------------------------------------------------------------------------- @@ -71,8 +94,12 @@ (\mxs -> coerce <$> (mxs >>= NE.nonEmpty)) $ dioptional $ arrayOf _String "extract" -cargoLockPathCodec :: TomlCodec (Maybe PackageCargoFilePath)-cargoLockPathCodec = dioptional $ diwrap (string "cargo_lock")+cargoLockPathCodec :: TomlCodec (Maybe PackageCargoLockFiles)+cargoLockPathCodec =+ dimap+ (fmap (NE.toList . coerce))+ (\mxs -> coerce <$> (mxs >>= NE.nonEmpty))+ $ dioptional $ arrayOf _String "cargo_locks" nvcheckerOptionsCodec :: TomlCodec NvcheckerOptions nvcheckerOptionsCodec =@@ -83,3 +110,14 @@ passthruCodec :: TomlCodec PackagePassthru passthruCodec = diwrap $ tableHashMap _KeyText text "passthru"++pinnedCodec :: TomlCodec UseStaleVersion+pinnedCodec =+ dimap+ ( \case+ PermanentStale -> Just True+ TemporaryStale -> error "Impossible!"+ NoStale -> Just False+ )+ (maybe NoStale (\x -> if x then PermanentStale else NoStale))+ $ dioptional $ bool "pinned"
app/Config/PackageFetcher.hs view
@@ -5,7 +5,11 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ViewPatterns #-} -module Config.PackageFetcher (fetcherCodec) where+module Config.PackageFetcher+ ( fetcherCodec,+ fetcherKeys,+ )+where import Data.Coerce (coerce) import Data.Default (Default, def)@@ -31,9 +35,21 @@ openVsxCodec, vscodeMarketplaceCodec, gitCodec,- urlCodec+ urlCodec,+ tarballCodec ] +fetcherKeys :: [Key]+fetcherKeys =+ [ "fetch.github",+ "fetch.pypi",+ "fetch.openvsx",+ "fetch.vsmarketplace",+ "fetch.git",+ "fetch.url",+ "fetch.tarball"+ ]+ -------------------------------------------------------------------------------- data GitOptions = GitOptions@@ -59,7 +75,15 @@ & leaveDotGit .~ fromMaybe False goLeaveDotGit ) <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit))-_GitOptions _ x@FetchUrl {} = pure x+_GitOptions f x@FetchGitHub {..} =+ ( \GitOptions {..} ->+ x+ & deepClone .~ fromMaybe False goDeepClone+ & fetchSubmodules .~ fromMaybe False goFetchSubmodules+ & leaveDotGit .~ fromMaybe False goLeaveDotGit+ )+ <$> f (GitOptions (Just _deepClone) (Just _fetchSubmodules) (Just _leaveDotGit))+_GitOptions _ x = pure x -------------------------------------------------------------------------------- @@ -138,3 +162,12 @@ unsupportError (\t -> Right $ \(coerce -> v) -> urlFetcher $ T.replace "$ver" v t) "fetch.url"++--------------------------------------------------------------------------------++tarballCodec :: TomlCodec PackageFetcher+tarballCodec =+ Toml.textBy+ unsupportError+ (\t -> Right $ \(coerce -> v) -> tarballFetcher $ T.replace "$ver" v t)+ "fetch.tarball"
app/Config/VersionSource.hs view
@@ -2,7 +2,11 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} -module Config.VersionSource (versionSourceCodec) where+module Config.VersionSource+ ( versionSourceCodec,+ versionSourceKeys,+ )+where import Data.Foldable (asum) import Data.Text (Text)@@ -28,9 +32,26 @@ webpageCodec, httpHeaderCodec, openVsxCodec,- vscodeMarketplaceCodec+ vscodeMarketplaceCodec,+ cmdCodec ] +versionSourceKeys :: [Key]+versionSourceKeys =+ [ "src.github",+ "src.github_tag",+ "src.git",+ "src.pypi",+ "src.archpkg",+ "src.aur",+ "src.manual",+ "src.webpage",+ "src.httpheader",+ "src.openvsx",+ "src.vsmarketplace",+ "src.cmd"+ ]+ -------------------------------------------------------------------------------- githubICodec :: Key -> TomlCodec (Text, Text)@@ -215,3 +236,11 @@ vscodeMarketplaceCodec :: TomlCodec VersionSource vscodeMarketplaceCodec = dimatch matchVscodeMarketplace (uncurry VscodeMarketplace) vscodeMarketplaceICodec++--------------------------------------------------------------------------------++matchCmd :: VersionSource -> Maybe Text+matchCmd x = x ^? vcmd++cmdCodec :: TomlCodec VersionSource+cmdCodec = dimatch matchCmd Cmd (text "src.cmd")
app/Main.hs view
@@ -1,11 +1,9 @@ {-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE TemplateHaskell #-}-{-# LANGUAGE ViewPatterns #-} module Main where import Config+import Data.Default (def) import qualified Data.Text as T import qualified Data.Text.IO as T import NvFetcher@@ -34,5 +32,5 @@ case toml of Left e -> error $ T.unpack $ Toml.prettyTomlDecodeError $ Toml.ParseError e Right x -> case parseConfig x of- Left e -> error $ T.unpack $ Toml.prettyTomlDecodeErrors e- Right pkgs -> runNvFetcherNoCLI (cliOptionsToArgs opt) $ purePackageSet pkgs+ Left e -> error $ T.unpack $ prettyPackageConfigParseError e+ Right pkgs -> runNvFetcherNoCLI (applyCliOptions def opt) (optTarget opt) $ purePackageSet pkgs
nvfetcher.cabal view
@@ -1,6 +1,6 @@ cabal-version: 2.4 name: nvfetcher-version: 0.4.0.0+version: 0.5.0.0 synopsis: Generate nix sources expr for the latest version of packages @@ -13,7 +13,7 @@ license-file: LICENSE author: berberman maintainer: berberman <berberman.yandex.com>-copyright: 2021 berberman+copyright: 2021-2022 berberman category: Nix build-type: Simple extra-doc-files:@@ -26,8 +26,9 @@ common common-options build-depends:- , aeson ^>=1.5.6- , base >=4.8 && <5+ , aeson >=1.5.6 && <2.1+ , aeson-pretty+ , base >=4.8 && <5 , binary , binary-instances ^>=1.0.1 , bytestring@@ -40,6 +41,8 @@ , neat-interpolation ^>=0.5.1 , optparse-simple ^>=0.1.1 , parsec+ , prettyprinter+ , regex-tdfa ^>=1.3.1.1 , shake ^>=0.19.4 , text , tomland ^>=1.3.2@@ -60,6 +63,7 @@ other-modules: NvFetcher.Utils exposed-modules: NvFetcher+ NvFetcher.Config NvFetcher.Core NvFetcher.ExtractSrc NvFetcher.FetchRustGitDeps
src/NvFetcher.hs view
@@ -1,10 +1,12 @@+{-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE TupleSections #-} {-# LANGUAGE TypeApplications #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -20,12 +22,12 @@ -- import NvFetcher -- -- main :: IO ()--- main = runNvFetcher defaultArgs packageSet+-- main = runNvFetcher packageSet -- -- packageSet :: PackageSet () -- packageSet = do -- define $ package "feeluown-core" `fromPypi` "feeluown"--- define $ package "qliveplayer" `fromGitHub` ("IsoaSFlus", "QLivePlayer")+-- define $ package "qliveplayer" `fromGitHub` ("THMonster", "QLivePlayer") -- @ -- -- You can find more examples of packages in @Main_example.hs@.@@ -35,95 +37,76 @@ -- * @main@ -- abbreviation of @main build@ -- * @main build@ -- build nix sources expr from given @packageSet@ -- * @main clean@ -- delete .shake dir and generated nix file--- * @main -j@ -- build with parallelism -- -- All shake options are inherited. module NvFetcher- ( Args (..),- defaultArgs,- runNvFetcher,+ ( runNvFetcher,+ runNvFetcher', runNvFetcherNoCLI,- cliOptionsToArgs,+ applyCliOptions,+ parseLastVersions, module NvFetcher.PackageSet, module NvFetcher.Types, module NvFetcher.Types.ShakeExtras, ) where -import Control.Monad.Extra (when, whenJust)+import Control.Monad.Extra (forM_, when, whenJust)+import qualified Data.Aeson as A+import qualified Data.Aeson.Encode.Pretty as A+import qualified Data.Aeson.Types as A+import qualified Data.ByteString.Lazy.Char8 as LBS+import Data.Default+import Data.List ((\\))+import qualified Data.Map.Strict as Map+import Data.Maybe (catMaybes, fromMaybe) import Data.Text (Text) import qualified Data.Text as T import Development.Shake import Development.Shake.FilePath import NeatInterpolation (trimming)+import NvFetcher.Config import NvFetcher.Core+import NvFetcher.NixExpr (ToNixExpr (toNixExpr)) import NvFetcher.NixFetcher import NvFetcher.Nvchecker import NvFetcher.Options import NvFetcher.PackageSet import NvFetcher.Types import NvFetcher.Types.ShakeExtras-import NvFetcher.Utils (getShakeDir)---- | Arguments for running nvfetcher-data Args = Args- { -- | Shake options- argShakeOptions :: ShakeOptions,- -- | Build target- argTarget :: String,- -- | Shake dir- argBuildDir :: FilePath,- -- | Custom rules- argRules :: Rules (),- -- | Action run after build rule- argActionAfterBuild :: Action (),- -- | Action run after clean rule- argActionAfterClean :: Action (),- -- | Retry times- argRetries :: Int- }---- | Default arguments of 'defaultMain'------ Build dir is @_sources@.-defaultArgs :: Args-defaultArgs =- Args- ( shakeOptions- { shakeProgress = progressSimple,- shakeThreads = 0- }- )- "build"- "_sources"- (pure ())- (pure ())- (pure ())- 3+import NvFetcher.Utils (aesonKey, getDataDir)+import qualified System.Directory.Extra as D+import Text.Regex.TDFA ((=~)) -- | Run nvfetcher with CLI options ----- This function calls 'runNvFetcherNoCLI', using 'Args' from 'CLIOptions'.+-- This function calls 'runNvFetcherNoCLI', using 'def' 'Config' overridden by 'CLIOptions'. -- Use this function to create your own Haskell executable program. runNvFetcher :: PackageSet () -> IO ()-runNvFetcher packageSet =- getCLIOptions cliOptionsParser >>= flip runNvFetcherNoCLI packageSet . cliOptionsToArgs+runNvFetcher = runNvFetcher' def --- | Apply 'CLIOptions' to 'defaultArgs'-cliOptionsToArgs :: CLIOptions -> Args-cliOptionsToArgs CLIOptions {..} =- defaultArgs- { argActionAfterBuild = do- whenJust logPath logChangesToFile- when commit commitChanges,- argTarget = target,- argShakeOptions =- (argShakeOptions defaultArgs)- { shakeTimings = timing,- shakeVerbosity = if verbose then Verbose else Info,- shakeThreads = threads,- shakeFiles = buildDir- }+-- | Similar to 'runNvFetcher', but uses custom @config@ instead of 'def' overridden by 'CLIOptions'+runNvFetcher' :: Config -> PackageSet () -> IO ()+runNvFetcher' config packageSet =+ getCLIOptions cliOptionsParser >>= \cli -> runNvFetcherNoCLI (applyCliOptions config cli) (optTarget cli) packageSet++-- | Apply 'CLIOptions' to 'Config'+applyCliOptions :: Config -> CLIOptions -> Config+applyCliOptions config CLIOptions {..} =+ config+ { buildDir = optBuildDir,+ actionAfterBuild = do+ whenJust optLogPath logChangesToFile+ when optCommit commitChanges+ actionAfterBuild config,+ shakeConfig =+ (shakeConfig config)+ { shakeTimings = optTiming,+ shakeVerbosity = if optVerbose then Verbose else Info,+ shakeThreads = optThreads+ },+ filterRegex = optPkgNameFilter,+ retry = optRetry } logChangesToFile :: FilePath -> Action ()@@ -140,54 +123,98 @@ [] -> Nothing whenJust commitMsg $ \msg -> do putInfo "Commiting changes"- getShakeDir >>= \dir -> command_ [] "git" ["add", dir]+ getBuildDir >>= \dir -> command_ [] "git" ["add", dir] command_ [] "git" ["commit", "-m", msg] +-- | @Parse generated.nix@+parseLastVersions :: FilePath -> IO (Maybe (Map.Map PackageKey Version))+parseLastVersions jsonFile =+ D.doesFileExist jsonFile >>= \case+ True -> do+ objs <- A.decodeFileStrict' jsonFile+ pure $+ flip fmap objs $+ ( \xs ->+ Map.fromList+ . catMaybes+ $ [(PackageKey k,) <$> A.parseMaybe (A..: "version") obj | (k, obj) <- xs]+ )+ . Map.toList+ _ -> pure mempty+ -- | Entry point of nvfetcher-runNvFetcherNoCLI :: Args -> PackageSet () -> IO ()-runNvFetcherNoCLI args@Args {..} packageSet = do- pkgs <- runPackageSet packageSet- shakeExtras <- initShakeExtras pkgs argRetries- let opts =- argShakeOptions- { shakeExtra = addShakeExtra shakeExtras (shakeExtra argShakeOptions)+runNvFetcherNoCLI :: Config -> Target -> PackageSet () -> IO ()+runNvFetcherNoCLI config@Config {..} target packageSet = do+ pkgs <- Map.map pinIfUnmatch <$> runPackageSet packageSet+ lastVersions <- parseLastVersions $ buildDir </> generatedJsonFileName+ shakeDir <- getDataDir+ -- Set shakeFiles+ let shakeOptions1 = shakeConfig {shakeFiles = shakeDir}+ -- shakeConfig in Config will be shakeOptions1 (not including shake extra)+ shakeExtras <- initShakeExtras (config {shakeConfig = shakeOptions1}) pkgs $ fromMaybe mempty lastVersions+ -- Set shakeExtra+ let shakeOptions2 = shakeOptions1 {shakeExtra = addShakeExtra shakeExtras (shakeExtra shakeConfig)}+ rules = mainRules config+ shake shakeOptions2 $ want [show target] >> rules+ where+ -- Don't touch already pinned packages+ pinIfUnmatch x@Package {..}+ | Just regex <- filterRegex =+ x+ { _ppinned = case _ppinned of+ PermanentStale -> PermanentStale+ _ ->+ if _pname =~ regex+ then NoStale+ else TemporaryStale }- rules = mainRules args- shake opts $ want [argTarget] >> rules+ | otherwise = x -------------------------------------------------------------------------------- -mainRules :: Args -> Rules ()-mainRules Args {..} = do+mainRules :: Config -> Rules ()+mainRules Config {..} = do "clean" ~> do- getShakeDir >>= flip removeFilesAfter ["//*"]- argActionAfterClean+ getBuildDir >>= flip removeFilesAfter ["//*"]+ actionAfterClean "build" ~> do allKeys <- getAllPackageKeys- body <- parallel $ generateNixSourceExpr <$> allKeys+ results <- parallel $ runPackage <$> allKeys+ -- Record removed packages to version changes+ getAllOnDiskVersions+ >>= \oldPkgs -> forM_ (Map.keys oldPkgs \\ allKeys) $+ \pkg -> recordVersionChange (coerce pkg) (oldPkgs Map.!? pkg) "∅" getVersionChanges >>= \changes -> if null changes then putInfo "Up to date" else do putInfo "Changes:" putInfo $ unlines $ show <$> changes- shakeDir <- getShakeDir- let genPath = shakeDir </> "generated.nix"- putVerbose $ "Generating " <> genPath- writeFileChanged genPath $ T.unpack $ srouces (T.unlines body) <> "\n"- need [genPath]- argActionAfterBuild+ buildDir <- getBuildDir+ let generatedNixPath = buildDir </> generatedNixFileName+ generatedJSONPath = buildDir </> generatedJsonFileName+ putVerbose $ "Generating " <> generatedNixPath+ writeFileChanged generatedNixPath $ T.unpack $ srouces (T.unlines $ toNixExpr <$> results) <> "\n"+ putVerbose $ "Generating " <> generatedJSONPath+ writeFileChanged generatedJSONPath $ LBS.unpack $ A.encodePretty $ A.object [aesonKey (_prname r) A..= r | r <- results]+ actionAfterBuild - argRules+ customRules coreRules srouces :: Text -> Text srouces body = [trimming| # This file was generated by nvfetcher, please do not modify it manually.- { fetchgit, fetchurl }:+ { fetchgit, fetchurl, fetchFromGitHub }: { $body } |]++generatedNixFileName :: String+generatedNixFileName = "generated.nix"++generatedJsonFileName :: String+generatedJsonFileName = "generated.json"
+ src/NvFetcher/Config.hs view
@@ -0,0 +1,38 @@+-- | Copyright: (c) 2021-2022 berberman+-- SPDX-License-Identifier: MIT+-- Maintainer: berberman <berberman@yandex.com>+-- Stability: experimental+-- Portability: portable+module NvFetcher.Config where++import Data.Default+import Development.Shake++-- | Nvfetcher configuration+data Config = Config+ { shakeConfig :: ShakeOptions,+ buildDir :: FilePath,+ customRules :: Rules (),+ actionAfterBuild :: Action (),+ actionAfterClean :: Action (),+ retry :: Int,+ filterRegex :: Maybe String,+ cacheNvchecker :: Bool+ }++instance Default Config where+ def =+ Config+ { shakeConfig =+ shakeOptions+ { shakeProgress = progressSimple,+ shakeThreads = 0+ },+ buildDir = "_sources",+ customRules = pure (),+ actionAfterBuild = pure (),+ actionAfterClean = pure (),+ retry = 3,+ filterRegex = Nothing,+ cacheNvchecker = True+ }
src/NvFetcher/Core.hs view
@@ -1,12 +1,9 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-}-{-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE TypeApplications #-}-{-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -14,26 +11,22 @@ module NvFetcher.Core ( Core (..), coreRules,- generateNixSourceExpr,+ runPackage, ) where import Data.Coerce (coerce) import qualified Data.HashMap.Strict as HMap-import Data.Text (Text) import qualified Data.Text as T import Development.Shake import Development.Shake.FilePath import Development.Shake.Rule-import NeatInterpolation (trimming) import NvFetcher.ExtractSrc import NvFetcher.FetchRustGitDeps-import NvFetcher.NixExpr import NvFetcher.NixFetcher import NvFetcher.Nvchecker import NvFetcher.Types import NvFetcher.Types.ShakeExtras-import NvFetcher.Utils -- | The core rule of nvchecker. -- all rules are wired here.@@ -43,126 +36,80 @@ prefetchRule extractSrcRule fetchRustGitDepsRule- -- TODO: maybe an oracle rule is enough- addBuiltinRule noLint noIdentity $ \(WithPackageKey (Core, pkg)) mOld mode -> case mode of- RunDependenciesSame- | Just old <- mOld,- (expr, _) <- decode' @(NixExpr, Version) old ->- pure $ RunResult ChangedNothing old expr- _ ->- lookupPackage pkg >>= \case- Nothing -> fail $ "Unkown package key: " <> show pkg- Just- Package- { _pversion = NvcheckerQ versionSource options,- _ppassthru = PackagePassthru passthruMap,- ..- } -> do- -- it's important to always rerun- -- since the package definition is not tracked at all- alwaysRerun- (NvcheckerA version mOldV) <- checkVersion versionSource options pkg- prefetched <- prefetch $ _pfetcher version- shakeDir <- getShakeDir- -- extract src- appending1 <-- case _pextract of- Just (PackageExtractSrc extract) -> do- result <- HMap.toList <$> extractSrcs prefetched extract- T.unlines- <$> sequence- [ do- -- write extracted files to shake dir- -- and read them in nix using 'builtins.readFile'- writeFile' (shakeDir </> path) (T.unpack v)- pure $ toNixExpr k <> " = builtins.readFile ./" <> T.pack path <> ";"- | (k, v) <- result,- let path =- T.unpack _pname- <> "-"- <> T.unpack (coerce version)- </> k- ]- _ -> pure ""- -- cargo lock- appending2 <-- case _pcargo of- Just (PackageCargoFilePath lockPath) -> do- (_, lockData) <- head . HMap.toList <$> extractSrc prefetched lockPath- result <- fetchRustGitDeps prefetched lockPath- let body = T.unlines [asString k <> " = " <> coerce (asString $ coerce v) <> ";" | (k, v) <- HMap.toList result]- lockPath' =- T.unpack _pname- <> "-"- <> T.unpack (coerce version)- </> lockPath- lockPathNix = "./" <> T.pack lockPath'- -- similar to extract src, write lock file to shake dir- writeFile' (shakeDir </> lockPath') $ T.unpack lockData- pure- [trimming|- cargoLock = {- lockFile = $lockPathNix;- outputHashes = {- $body- };- };- |]- _ -> pure ""- -- passthru- let appending3 = T.unlines [k <> " = " <> v <> ";" | (k, asString -> v) <- HMap.toList passthruMap]-- -- update changelog- case mOldV of- Nothing ->- recordVersionChange _pname Nothing version- Just old- | old /= version ->- recordVersionChange _pname (Just old) version- _ -> pure ()+ addBuiltinRule noLint noIdentity $ \(WithPackageKey (Core, pkg)) _ _ -> do+ -- it's important to always rerun+ -- since the package definition is not tracked at all+ alwaysRerun+ lookupPackage pkg >>= \case+ Nothing -> fail $ "Unkown package key: " <> show pkg+ Just+ Package+ { _pversion = CheckVersion versionSource options,+ _ppassthru = (PackagePassthru passthru),+ ..+ } -> do+ _prversion@(NvcheckerResult version _mOldV _isStale) <- checkVersion versionSource options pkg+ _prfetched <- prefetch $ _pfetcher version+ buildDir <- getBuildDir+ -- extract src+ _prextract <-+ case _pextract of+ Just (PackageExtractSrc extract) -> do+ result <- HMap.toList <$> extractSrcs _prfetched extract+ Just . HMap.fromList+ <$> sequence+ [ do+ -- write extracted files to build dir+ -- and read them in nix using 'builtins.readFile'+ writeFile' (buildDir </> path) (T.unpack v)+ pure (k, T.pack path)+ | (k, v) <- result,+ let path =+ "./"+ <> T.unpack _pname+ <> "-"+ <> T.unpack (coerce version)+ </> k+ ]+ _ -> pure Nothing+ -- cargo locks+ _prcargolock <-+ case _pcargo of+ Just (PackageCargoLockFiles lockPath) -> do+ lockFiles <- HMap.toList <$> extractSrcs _prfetched lockPath+ result <- parallel $+ flip fmap lockFiles $ \(lockPath, lockData) -> do+ result <- fetchRustGitDeps _prfetched lockPath+ let lockPath' =+ T.unpack _pname+ <> "-"+ <> T.unpack (coerce version)+ </> lockPath+ lockPathNix = "./" <> T.pack lockPath'+ -- similar to extract src, write lock file to build dir+ writeFile' (buildDir </> lockPath') $ T.unpack lockData+ pure (lockPath, (lockPathNix, result))+ pure . Just $ HMap.fromList result+ _ -> pure Nothing - let result = gen _pname version prefetched $ appending1 <> appending2 <> appending3- pure $ RunResult ChangedRecomputeDiff (encode' (result, version)) result+ -- update changelog+ -- always use on disk verion+ mOldV <- getLastVersionOnDisk pkg+ case mOldV of+ Nothing ->+ recordVersionChange _pname Nothing version+ Just old+ | old /= version ->+ recordVersionChange _pname (Just old) version+ _ -> pure () --- | Run the core rule.--- Given a 'PackageKey', run "NvFetcher.Nvchecker", "NvFetcher.NixFetcher"--- (may also run "NvFetcher.ExtractSrc" or "NvFetrcher.FetchRustGitDeps")------ @--- Package--- { _pname = "feeluown-core",--- _pversion = NvcheckerQ (Pypi "feeluown") def,--- _pfetcher = pypiFetcher "feeluown",--- _pextract = Nothing,--- _pcargo = Nothing,--- _ppassthru = PackagePassthru (HashMap.fromList [("a", "B")])--- }--- @------ resulting a nix exprs snippet like:------ @--- feeluown-core = {--- pname = "feeluown-core";--- version = "3.7.7";--- src = fetchurl {--- sha256 = "06d3j39ff9znqxkhp9ly81lcgajkhg30hyqxy2809yn23xixg3x2";--- url = "https://pypi.io/packages/source/f/feeluown/feeluown-3.7.7.tar.gz";--- };--- a = "B";--- };--- @-generateNixSourceExpr :: PackageKey -> Action NixExpr-generateNixSourceExpr k = apply1 $ WithPackageKey (Core, k)+ let _prpassthru = if HMap.null passthru then Nothing else Just passthru+ _prname = _pname+ _prpinned = _ppinned+ -- Since we don't save the previous result, we are not able to know if the result changes+ -- Depending on this rule leads to RunDependenciesChanged+ pure $ RunResult ChangedRecomputeDiff mempty PackageResult {..} -gen :: PackageName -> Version -> NixFetcher Fetched -> Text -> Text-gen name (coerce -> ver) (toNixExpr -> srcP) appending =- [trimming|- $name = {- pname = "$name";- version = "$ver";- src = $srcP;$appending'- };-|]- where- appending' = if T.null appending then "" else "\n" <> appending+-- | 'Core' rule take a 'PackageKey', find the corresponding 'Package', and run all needed rules to get 'PackageResult'+runPackage :: PackageKey -> Action PackageResult+runPackage k = apply1 $ WithPackageKey (Core, k)
src/NvFetcher/ExtractSrc.hs view
@@ -4,7 +4,7 @@ {-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -38,15 +38,18 @@ import NvFetcher.NixExpr import NvFetcher.Types import NvFetcher.Types.ShakeExtras+import Prettyprinter (pretty, (<+>)) -- | Rules of extract source extractSrcRule :: Rules () extractSrcRule = void $- addOracleCache $ \(q :: ExtractSrcQ) -> withTempFile $ \fp -> withRetries $ do- writeFile' fp $ T.unpack $ wrap $ toNixExpr q- need [fp]+ addOracleCache $ \(q :: ExtractSrcQ) -> withTempFile $ \fp -> withRetry $ do+ putInfo . show $ "#" <+> pretty q+ let nixExpr = T.unpack $ wrap $ toNixExpr q+ putVerbose $ "Generated nix expr:\n" <> nixExpr+ writeFile' fp nixExpr -- TODO: Avoid using NIX_PATH- (CmdTime t, StdoutTrim out, CmdLine c) <- cmd Shell $ "nix-instantiate --eval --strict --json --read-write-mode -E 'let pkgs = import <nixpkgs> { }; in ((import " <> fp <> ") pkgs)'"+ (CmdTime t, StdoutTrim out, CmdLine c) <- quietly $ cmd Shell $ "nix-instantiate --eval --strict --json --read-write-mode -E 'let pkgs = import <nixpkgs> { }; in ((import " <> fp <> ") pkgs)'" putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s" case A.decodeStrict out of Just x -> pure x
src/NvFetcher/FetchRustGitDeps.hs view
@@ -3,7 +3,7 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -33,6 +33,7 @@ import NvFetcher.ExtractSrc import NvFetcher.NixFetcher import NvFetcher.Types+import Prettyprinter (pretty, (<+>)) import Text.Parsec import Text.Parsec.Text import Toml (TomlCodec, (.=))@@ -41,7 +42,8 @@ -- | Rules of fetch rust git dependencies fetchRustGitDepsRule :: Rules () fetchRustGitDepsRule = void $- addOracleCache $ \(FetchRustGitDepsQ fetcher lockPath) -> do+ addOracleCache $ \key@(FetchRustGitDepsQ fetcher lockPath) -> do+ putInfo . show $ "#" <+> pretty key cargoLock <- head . HMap.elems <$> extractSrc fetcher lockPath deps <- case Toml.decode (Toml.list rustDepCodec "package") cargoLock of Right r -> pure $ nubOrdOn rrawSrc r
src/NvFetcher/NixExpr.hs view
@@ -4,9 +4,10 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE QuasiQuotes #-} {-# LANGUAGE RecordWildCards #-}+{-# LANGUAGE ScopedTypeVariables #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -21,12 +22,15 @@ where import Data.Coerce (coerce)+import Data.HashMap.Strict (HashMap)+import qualified Data.HashMap.Strict as HMap import qualified Data.List.NonEmpty as NE+import Data.Maybe (fromMaybe) import Data.Text (Text) import qualified Data.Text as T import NeatInterpolation (trimming) import NvFetcher.Types-import NvFetcher.Utils (asString)+import NvFetcher.Utils (quote, quoteIfNeeds) -- | Types can be converted into nix expr class ToNixExpr a where@@ -36,7 +40,7 @@ toNixExpr = nixFetcher "lib.fakeSha256" instance ToNixExpr (NixFetcher Fetched) where- toNixExpr f = nixFetcher (asString $ coerce $ _sha256 f) f+ toNixExpr f = nixFetcher (quote $ coerce $ _sha256 f) f instance ToNixExpr Bool where toNixExpr True = "true"@@ -61,11 +65,12 @@ nixFetcher sha256 = \case FetchGit { _sha256 = _,- _rev = asString . toNixExpr -> rev,+ _rev = quote . toNixExpr -> rev, _fetchSubmodules = toNixExpr -> fetchSubmodules, _deepClone = toNixExpr -> deepClone, _leaveDotGit = toNixExpr -> leaveDotGit,- _furl = asString -> url+ _furl = quote -> url,+ _name = nameField -> n } -> [trimming| fetchgit {@@ -73,17 +78,61 @@ rev = $rev; fetchSubmodules = $fetchSubmodules; deepClone = $deepClone;- leaveDotGit = $leaveDotGit;+ leaveDotGit = $leaveDotGit;$n sha256 = $sha256; } |]- (FetchUrl (asString -> url) _) ->+ FetchGitHub+ { _sha256 = _,+ _rev = quote . toNixExpr -> rev,+ _fetchSubmodules = toNixExpr -> fetchSubmodules,+ _deepClone = toNixExpr -> deepClone,+ _leaveDotGit = toNixExpr -> leaveDotGit,+ _fowner = quote -> owner,+ _frepo = quote -> repo,+ _name = nameField -> n+ } ->+ -- TODO: fix fetchFromGitHub in Nixpkgs so that deepClone and+ -- leaveDotGit won't get passed to fetchzip+ if (deepClone == "true") || (leaveDotGit == "true")+ then+ [trimming|+ fetchFromGitHub ({+ owner = $owner;+ repo = $repo;+ rev = $rev;+ fetchSubmodules = $fetchSubmodules;+ deepClone = $deepClone;+ leaveDotGit = $leaveDotGit;$n+ sha256 = $sha256;+ })+ |]+ else+ [trimming|+ fetchFromGitHub ({+ owner = $owner;+ repo = $repo;+ rev = $rev;+ fetchSubmodules = $fetchSubmodules;$n+ sha256 = $sha256;+ })+ |]+ (FetchUrl (quote -> url) (nameField -> n) _) -> [trimming| fetchurl {+ url = $url;$n+ sha256 = $sha256;+ }+ |]+ (FetchTarball (quote -> url) _) ->+ [trimming|+ fetchTarball { url = $url; sha256 = $sha256; } |]+ where+ nameField = maybe "" (\x -> "\nname = " <> quote x <> ";") instance ToNixExpr ExtractSrcQ where toNixExpr (ExtractSrcQ fetcher files) = extractFiles fetcher files@@ -104,3 +153,80 @@ value = toFile x; }) fileNames) |]++-- | nix expr snippet like:+--+-- @+-- feeluown-core = {+-- pname = "feeluown-core";+-- version = "3.7.7";+-- src = fetchurl {+-- sha256 = "06d3j39ff9znqxkhp9ly81lcgajkhg30hyqxy2809yn23xixg3x2";+-- url = "https://pypi.io/packages/source/f/feeluown/feeluown-3.7.7.tar.gz";+-- };+-- a = "B";+-- };+-- @+instance ToNixExpr PackageResult where+ toNixExpr PackageResult {..} =+ [trimming|+ $name = {+ pname = $nameString;+ version = $version;+ src = $src;$appending+ };+ |]+ where+ name = quoteIfNeeds _prname+ nameString = quote _prname+ version = quote . coerce . nvNow $ _prversion+ src = toNixExpr _prfetched+ extract =+ maybe+ ""+ ( \ex ->+ T.unlines+ [ quoteIfNeeds (T.pack name)+ <> " = builtins.readFile "+ <> fp+ <> ";"+ | (name, fp) <- HMap.toList ex+ ]+ )+ _prextract+ cargo = fromMaybe "" $ do+ cargoLocks <- _prcargolock+ let depsSnippet (deps :: HashMap Text Checksum) =+ T.unlines+ [ quoteIfNeeds name+ <> " = "+ <> quote (coerce sum)+ <> ";"+ | (name, sum) <- HMap.toList deps+ ]+ lockSnippet ((T.pack -> fp) :: FilePath, (nixFP :: NixExpr, deps :: HashMap Text Checksum)) =+ let hashes = depsSnippet deps+ in [trimming|+ cargoLock."$fp" = {+ lockFile = $nixFP;+ outputHashes = {+ $hashes+ };+ };+ |]+ pure . T.unlines $ lockSnippet <$> HMap.toList cargoLocks+ passthru =+ maybe+ ""+ ( \pt ->+ T.unlines+ [ quoteIfNeeds k+ <> " = "+ <> v+ <> ";"+ | (k, quote -> v) <- HMap.toList pt+ ]+ )+ _prpassthru+ joined = extract <> cargo <> passthru+ appending = if T.null joined then "" else "\n" <> joined
src/NvFetcher/NixFetcher.hs view
@@ -8,7 +8,7 @@ {-# LANGUAGE TypeApplications #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -16,10 +16,11 @@ -- -- 'NixFetcher' is used to describe how to fetch package sources. ----- There are two types of fetchers overall:+-- There are three types of fetchers overall: ----- 1. 'FetchGit' -- nix-prefetch-git--- 2. 'FetchUrl' -- nix-prefetch-url+-- 1. 'FetchGit' -- nix-prefetch fetchgit+-- 2. 'FetchGitHub' -- nix-prefetch fetchFromGitHub+-- 3. 'FetchUrl' -- nix-prefetch fetchurl -- -- As you can see the type signature of 'prefetch': -- a fetcher will be filled with the fetch result (hash) after the prefetch.@@ -41,12 +42,11 @@ urlFetcher, openVsxFetcher, vscodeMarketplaceFetcher,+ tarballFetcher, ) where -import Control.Monad (void, (<=<))-import qualified Data.Aeson as A-import qualified Data.Aeson.Types as A+import Control.Monad (void) import Data.Coerce (coerce) import Data.Text (Text) import qualified Data.Text as T@@ -55,31 +55,57 @@ import NeatInterpolation (trimming) import NvFetcher.Types import NvFetcher.Types.ShakeExtras+import Prettyprinter (pretty, (<+>)) -------------------------------------------------------------------------------- runFetcher :: NixFetcher Fresh -> Action Checksum runFetcher = \case FetchGit {..} -> do- let parser = A.withObject "nix-prefetch-git" $ \o -> Checksum <$> o A..: "sha256"- (CmdTime t, Stdout out, CmdLine c) <-- command [EchoStderr False] "nix-prefetch-git" $- [T.unpack _furl]- <> ["--rev", T.unpack $ coerce _rev]- <> ["--fetch-submodules" | _fetchSubmodules]- <> ["--deepClone" | _deepClone]- <> ["--leave-dotGit" | _leaveDotGit]+ (CmdTime t, Stdout (T.decodeUtf8 -> out), CmdLine c) <-+ quietly $+ command [EchoStderr False] "nix-prefetch" $+ ["fetchgit"]+ <> ["--url", T.unpack _furl]+ <> ["--rev", T.unpack $ coerce _rev]+ <> ["--fetchSubmodules" | _fetchSubmodules]+ <> ["--deepClone" | _deepClone]+ <> ["--leaveDotGit" | _leaveDotGit] putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"- let result = A.parseMaybe parser <=< A.decodeStrict $ out- case result of- Just x -> pure x- _ -> fail $ "Failed to parse output from nix-prefetch-git: " <> T.unpack (T.decodeUtf8 out)+ case takeWhile (not . T.null) $ reverse $ T.lines out of+ [x] -> pure $ coerce x+ _ -> fail $ "Failed to parse output from nix-prefetch: " <> T.unpack out+ FetchGitHub {..} -> do+ (CmdTime t, Stdout (T.decodeUtf8 -> out), CmdLine c) <-+ quietly $+ command [EchoStderr False] "nix-prefetch" $+ ["fetchFromGitHub"]+ <> ["--owner", T.unpack _fowner]+ <> ["--repo", T.unpack _frepo]+ <> ["--rev", T.unpack $ coerce _rev]+ <> ["--fetchSubmodules" | _fetchSubmodules]+ <> ["--deepClone" | _deepClone]+ <> ["--leaveDotGit" | _leaveDotGit]+ putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"+ case takeWhile (not . T.null) $ reverse $ T.lines out of+ [x] -> pure $ coerce x+ _ -> fail $ "Failed to parse output from nix-prefetch: " <> T.unpack out FetchUrl {..} -> do- (CmdTime t, Stdout (T.decodeUtf8 -> out), CmdLine c) <- command [EchoStderr False] "nix-prefetch-url" [T.unpack _furl]+ (CmdTime t, Stdout (T.decodeUtf8 -> out), CmdLine c) <-+ quietly $+ command [EchoStderr False] "nix-prefetch" ["fetchurl", "--url", T.unpack _furl] putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s" case takeWhile (not . T.null) $ reverse $ T.lines out of [x] -> pure $ coerce x- _ -> fail $ "Failed to parse output from nix-prefetch-url: " <> T.unpack out+ _ -> fail $ "Failed to parse output from nix-prefetch: " <> T.unpack out+ FetchTarball {..} -> do+ (CmdTime t, Stdout (T.decodeUtf8 -> out), CmdLine c) <-+ quietly $+ command [EchoStderr False] "nix-prefetch" ["fetchTarball", "--url", T.unpack _furl]+ putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"+ case takeWhile (not . T.null) $ reverse $ T.lines out of+ [x] -> pure $ coerce x+ _ -> fail $ "Failed to parse output from nix-prefetch: " <> T.unpack out pypiUrl :: Text -> Version -> Text pypiUrl pypi (coerce -> ver) =@@ -92,7 +118,8 @@ prefetchRule :: Rules () prefetchRule = void $ addOracleCache $ \(f :: NixFetcher Fresh) -> do- sha256 <- withRetries $ runFetcher f+ putInfo . show $ "#" <+> pretty f+ sha256 <- withRetry $ runFetcher f pure $ f {_sha256 = sha256} -- | Run nix fetcher@@ -103,14 +130,14 @@ -- | Create a fetcher from git url gitFetcher :: Text -> PackageFetcher-gitFetcher furl rev = FetchGit furl rev False False False ()+gitFetcher furl rev = FetchGit furl rev False False False Nothing () -- | Create a fetcher from github repo gitHubFetcher :: -- | owner and repo (Text, Text) -> PackageFetcher-gitHubFetcher (owner, repo) = gitFetcher [trimming|https://github.com/$owner/$repo|]+gitHubFetcher (owner, repo) rev = FetchGitHub owner repo rev False False False Nothing () -- | Create a fetcher from pypi pypiFetcher :: Text -> PackageFetcher@@ -129,7 +156,7 @@ -- | Create a fetcher from url urlFetcher :: Text -> NixFetcher Fresh-urlFetcher = flip FetchUrl ()+urlFetcher url = FetchUrl url Nothing () -- | Create a fetcher from openvsx openVsxFetcher ::@@ -137,8 +164,10 @@ (Text, Text) -> PackageFetcher openVsxFetcher (publisher, extName) (coerce -> ver) =- urlFetcher+ FetchUrl [trimming|https://open-vsx.org/api/$publisher/$extName/$ver/file/$publisher.$extName-$ver.vsix|]+ (Just [trimming|$extName-$ver.zip|])+ () -- | Create a fetcher from vscode marketplace vscodeMarketplaceFetcher ::@@ -146,5 +175,11 @@ (Text, Text) -> PackageFetcher vscodeMarketplaceFetcher (publisher, extName) (coerce -> ver) =- urlFetcher+ FetchUrl [trimming|https://$publisher.gallery.vsassets.io/_apis/public/gallery/publisher/$publisher/extension/$extName/$ver/assetbyname/Microsoft.VisualStudio.Services.VSIXPackage|]+ (Just [trimming|$extName-$ver.zip|])+ ()++-- | Create a fetcher from url, using fetchTarball+tarballFetcher :: Text -> NixFetcher Fresh+tarballFetcher url = FetchTarball url ()
src/NvFetcher/Nvchecker.hs view
@@ -1,9 +1,8 @@ {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-}-{-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -14,26 +13,29 @@ -- Now we call nvchecker for each 'VersionSource', which seems not to be efficient, but it's tolerable when running in parallel. -- -- Meanwhile, we lose the capabilities of tracking version updates, i.e. normally nvchecker will help us maintain a list of old versions,--- so that we are able to know which package's version is updated in this run. Fortunately, we can reimplement this using shake database,+-- so that we are able to know which package's version is updated in this run. Fortunately, we can reimplement this in shake, -- see 'nvcheckerRule' for details. module NvFetcher.Nvchecker ( -- * Types VersionSortMethod (..), ListOptions (..),- NvcheckerQ (..),+ CheckVersion (..), NvcheckerOptions (..), VersionSource (..),- NvcheckerA (..),+ NvcheckerResult (..), -- * Rules nvcheckerRule, checkVersion,+ checkVersion', ) where +import Control.Monad (void)+import Control.Monad.Extra (fromMaybeM) import qualified Data.Aeson as A import Data.Coerce (coerce)-import Data.Maybe (mapMaybe)+import Data.Maybe (fromJust, mapMaybe) import Data.String (fromString) import qualified Data.Text as T import qualified Data.Text.Encoding as T@@ -42,41 +44,75 @@ import NvFetcher.Types import NvFetcher.Types.ShakeExtras import NvFetcher.Utils-import Toml (Value (Bool, Text), pretty)+import Prettyprinter (pretty, (<+>))+import Toml (Value (Bool, Text))+import qualified Toml import Toml.Type.Edsl -- | Rules of nvchecker nvcheckerRule :: Rules ()-nvcheckerRule = addBuiltinRule noLint noIdentity $ \(WithPackageKey (NvcheckerQ versionSource options, pkg)) old _mode ->- -- If the package was removed after the last run,- -- shake still runs the nvchecker rule for this package.- -- So we record a version change here, indicating that the package has been removed.- -- Ideally, this should be done in the core rule- isPackageKeyTarget pkg >>= \case- False -> do- let oldVer = decode' <$> old- recordVersionChange (coerce pkg) oldVer "∅"- pure $ RunResult ChangedRecomputeDiff mempty undefined -- skip running, returning a never consumed result- _ ->- withTempFile $ \config -> withRetries $ do- writeFile' config $ T.unpack $ pretty $ mkToml $ genNvConfig pkg options versionSource- need [config]- (CmdTime t, Stdout out, CmdLine c) <- cmd $ "nvchecker --logger json -c " <> config- putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"- let out' = T.decodeUtf8 out- result = mapMaybe (A.decodeStrict . T.encodeUtf8) (T.lines out')- now <- case result of- [x] -> pure x- _ -> fail $ "Failed to parse output from nvchecker: " <> T.unpack out'- pure $ case old of- Just lastRun- | cachedResult <- decode' lastRun ->- if cachedResult == nvNow now- then -- try to get the version in last run from store, filling it into 'now'- RunResult ChangedRecomputeSame lastRun now {nvOld = Just cachedResult}- else RunResult ChangedRecomputeDiff (encode' $ nvNow now) now {nvOld = Just cachedResult}- Nothing -> RunResult ChangedRecomputeDiff (encode' $ nvNow now) now+nvcheckerRule = do+ persistedRule+ oneShotRule +-- | Nvchecker rule for packages, which is aware of version changes and supports using stale version.+-- nvchecker will be called at most one time given a package key. Follow-up using of this rule will return cached result.+-- 'PackageKey' is required for caching.+-- Run this rule by calling 'checkVersion'+persistedRule :: Rules ()+persistedRule = addBuiltinRule noLint noIdentity $ \(WithPackageKey (key@(CheckVersion versionSource options), pkg)) _old _mode -> do+ putInfo . show $ "#" <+> pretty key+ oldVer <- getRecentLastVersion pkg+ useStaleVersion <- _ppinned . fromJust <$> lookupPackage pkg+ let useStale = case useStaleVersion of+ PermanentStale -> True+ TemporaryStale -> True+ _ -> False+ case useStale of+ True+ | Just oldVer' <- oldVer -> do+ -- use the stale version if we have+ putInfo $ T.unpack $ "Skip running nvchecker, use stale version " <> coerce oldVer' <> " for " <> coerce pkg+ let result = NvcheckerResult {nvNow = oldVer', nvOld = oldVer, nvStale = True}+ pure $ RunResult ChangedRecomputeSame (encode' result) result++ -- run nvchecker+ _ -> do+ -- if we already run this rule for a package, we can recover the last result from getLastVersionUpdated+ -- (when cacheNvchecker is enabled)+ useCache <- nvcheckerCacheEnabled+ now <- fromMaybeM (coerce <$> runNvchecker pkg options versionSource) (if useCache then getLastVersionUpdated pkg else pure Nothing)+ let runChanged = case oldVer of+ Just oldVer'+ | oldVer' == now -> ChangedRecomputeSame+ _ -> ChangedRecomputeDiff+ result = NvcheckerResult {nvNow = now, nvOld = oldVer, nvStale = False}+ -- always update+ updateLastVersion pkg now+ pure $ RunResult runChanged mempty result++-- | Nvchecker rule without cache+-- Rule this rule by calling 'checkVersion''+oneShotRule :: Rules ()+oneShotRule = void $+ addOracle $ \key@(CheckVersion versionSource options) -> do+ putInfo . show $ pretty key+ NvcheckerRaw now <- runNvchecker (PackageKey "pkg") options versionSource+ pure $ NvcheckerResult now Nothing False++runNvchecker :: PackageKey -> NvcheckerOptions -> VersionSource -> Action NvcheckerRaw+runNvchecker pkg options versionSource = withTempFile $ \config -> withRetry $ do+ let nvcheckerConfig = T.unpack $ Toml.pretty $ mkToml $ genNvConfig pkg options versionSource+ putVerbose $ "Generated nvchecker config for " <> show pkg <> ":" <> nvcheckerConfig+ writeFile' config nvcheckerConfig+ (CmdTime t, Stdout out, CmdLine c) <- quietly . cmd $ "nvchecker --logger json -c " <> config+ putVerbose $ "Finishing running " <> c <> ", took " <> show t <> "s"+ let out' = T.decodeUtf8 out+ result = mapMaybe (A.decodeStrict . T.encodeUtf8) (T.lines out')+ case result of+ [x] -> pure x+ _ -> fail $ "Failed to parse output from nvchecker: " <> T.unpack out'+ genNvConfig :: PackageKey -> NvcheckerOptions -> VersionSource -> TDSL genNvConfig pkg options versionSource = table (fromString $ T.unpack $ coerce pkg) $ do genVersionSource versionSource@@ -133,6 +169,9 @@ VscodeMarketplace {..} -> do "source" =: "vsmarketplace" "vsmarketplace" =: Text (_vsmPublisher <> "." <> _vsmExtName)+ Cmd {..} -> do+ "source" =: "cmd"+ "cmd" =: Text _vcmd genListOptions ListOptions {..} = do "include_regex" =:? _includeRegex "exclude_regex" =:? _excludeRegex@@ -143,6 +182,11 @@ "from_pattern" =:? _fromPattern "to_pattern" =:? _toPattern --- | Run nvchecker-checkVersion :: VersionSource -> NvcheckerOptions -> PackageKey -> Action NvcheckerA-checkVersion v o k = apply1 $ WithPackageKey (NvcheckerQ v o, k)+-- | Run nvchecker given 'PackageKey'+-- Recording version changes and using stale version are available.+checkVersion :: VersionSource -> NvcheckerOptions -> PackageKey -> Action NvcheckerResult+checkVersion v o k = apply1 $ WithPackageKey (CheckVersion v o, k)++-- | Run nvchecker without cache+checkVersion' :: VersionSource -> NvcheckerOptions -> Action NvcheckerResult+checkVersion' v o = askOracle $ CheckVersion v o
src/NvFetcher/Options.hs view
@@ -1,6 +1,7 @@+{-# LANGUAGE LambdaCase #-} {-# LANGUAGE TemplateHaskell #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -9,6 +10,7 @@ -- CLI interface of nvfetcher module NvFetcher.Options ( CLIOptions (..),+ Target (..), cliOptionsParser, getCLIOptions, )@@ -17,16 +19,30 @@ import Options.Applicative.Simple import qualified Paths_nvfetcher as Paths +data Target = Build | Clean+ deriving (Eq)++instance Show Target where+ show Build = "build"+ show Clean = "clean"++targetParser :: ReadM Target+targetParser = maybeReader $ \case+ "build" -> Just Build+ "clean" -> Just Clean+ _ -> Nothing+ -- | Options for nvfetcher CLI data CLIOptions = CLIOptions- { buildDir :: FilePath,- commit :: Bool,- logPath :: Maybe FilePath,- threads :: Int,- retries :: Int,- timing :: Bool,- verbose :: Bool,- target :: String+ { optBuildDir :: FilePath,+ optCommit :: Bool,+ optLogPath :: Maybe FilePath,+ optThreads :: Int,+ optRetry :: Int,+ optTiming :: Bool,+ optVerbose :: Bool,+ optPkgNameFilter :: Maybe String,+ optTarget :: Target } deriving (Show) @@ -44,7 +60,7 @@ ) <*> switch ( long "commit-changes"- <> help "`git commit` changes in this run (with shake db)"+ <> help "`git commit` build dir with version changes as commit message" ) <*> optional ( strOption@@ -74,11 +90,20 @@ ) <*> switch (long "timing" <> short 't' <> help "Show build time") <*> switch (long "verbose" <> short 'v' <> help "Verbose mode")- <*> strArgument+ <*> optional+ ( strOption+ ( short 'f'+ <> long "filter"+ <> metavar "REGEX"+ <> help "Regex to filter packages to be updated"+ )+ )+ <*> argument+ targetParser ( metavar "TARGET" <> help "Two targets are available: 1.build 2.clean"- <> value "build"- <> completer (listCompleter ["build", "clean"])+ <> value Build+ <> completer (listCompleter [show Build, show Clean]) <> showDefault )
src/NvFetcher/PackageSet.hs view
@@ -1,9 +1,7 @@ {-# LANGUAGE DataKinds #-}-{-# LANGUAGE DeriveFunctor #-} {-# LANGUAGE FlexibleContexts #-} {-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GADTs #-}-{-# LANGUAGE KindSignatures #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE MultiParamTypeClasses #-} {-# LANGUAGE OverloadedStrings #-}@@ -15,7 +13,7 @@ {-# LANGUAGE UndecidableInstances #-} {-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -66,6 +64,7 @@ sourceHttpHeader, sourceOpenVsx, sourceVscodeMarketplace,+ sourceCmd, -- ** Fetchers fetchGitHub,@@ -77,12 +76,14 @@ fetchUrl, fetchOpenVsx, fetchVscodeMarketplace,+ fetchTarball, -- * Addons extractSource,- hasCargoLock,+ hasCargoLocks, tweakVersion, passthru,+ pinned, -- ** Miscellaneous Prod,@@ -139,7 +140,7 @@ -- 1. Carry defined packages -- 2. Run IO actions ----- Package set is evaluated be for shake runs.+-- Package set is evaluated before shake runs. -- Use 'newPackage' to add a new package, 'liftIO' to run an IO action. type PackageSet = Free PackageSetF @@ -149,13 +150,15 @@ -- | Add a package to package set newPackage :: PackageName ->- NvcheckerQ ->+ CheckVersion -> PackageFetcher -> Maybe PackageExtractSrc ->- Maybe PackageCargoFilePath ->+ Maybe PackageCargoLockFiles -> PackagePassthru ->+ UseStaleVersion -> PackageSet ()-newPackage name source fetcher extract cargo pasthru = liftF $ NewPackage (Package name source fetcher extract cargo pasthru) ()+newPackage name source fetcher extract cargo pasthru useStale =+ liftF $ NewPackage (Package name source fetcher extract cargo pasthru useStale) () -- | Add a list of packages into package set purePackageSet :: [Package] -> PackageSet ()@@ -250,9 +253,10 @@ r, OptionalMembers '[ PackageExtractSrc,- PackageCargoFilePath,+ PackageCargoLockFiles, NvcheckerOptions,- PackagePassthru+ PackagePassthru,+ UseStaleVersion ] r ) =>@@ -271,11 +275,12 @@ p <- e newPackage (proj p)- (NvcheckerQ (proj p) (fromMaybe def (projMaybe p)))+ (CheckVersion (proj p) (fromMaybe def (projMaybe p))) (proj p) (projMaybe p) (projMaybe p) (fromMaybe mempty (projMaybe p))+ (fromMaybe NoStale (projMaybe p)) -- | 'PkgDSL' version of 'newPackage' --@@ -293,9 +298,10 @@ r, OptionalMembers '[ PackageExtractSrc,- PackageCargoFilePath,+ PackageCargoLockFiles, PackagePassthru,- NvcheckerOptions+ NvcheckerOptions,+ UseStaleVersion ] r ) =>@@ -436,6 +442,12 @@ sourceVscodeMarketplace :: Attach VersionSource (Text, Text) sourceVscodeMarketplace e (_vsmPublisher, _vsmExtName) = src e VscodeMarketplace {..} +-- | This package follows a version from a shell command+--+-- Arg is the command to run+sourceCmd :: Attach VersionSource Text+sourceCmd e _vcmd = src e Cmd {..}+ -------------------------------------------------------------------------------- -- | This package is fetched from a github repo@@ -450,7 +462,7 @@ -- For example, you can enable fetch submodules like: -- -- @--- define $ package "qliveplayer" `sourceGitHub` ("IsoaSFlus", "QLivePlayer") `fetchGitHub'` ("IsoaSFlus", "QLivePlayer", fetchSubmodules .~ True)+-- define $ package "qliveplayer" `sourceGitHub` ("THMonster", "QLivePlayer") `fetchGitHub'` ("THMonster", "QLivePlayer", fetchSubmodules .~ True) -- @ fetchGitHub' :: Attach PackageFetcher (Text, Text, NixFetcher Fresh -> NixFetcher Fresh) fetchGitHub' e (owner, repo, f) = fetch e $ f . gitHubFetcher (owner, repo)@@ -498,17 +510,23 @@ fetchVscodeMarketplace :: Attach PackageFetcher (Text, Text) fetchVscodeMarketplace e = fetch e . vscodeMarketplaceFetcher +-- | This package is a tarball, fetched from url+--+-- Arg is a function which constructs the url from a version+fetchTarball :: Attach PackageFetcher (Version -> Text)+fetchTarball e f = fetch e (tarballFetcher . f)+ -------------------------------------------------------------------------------- -- | Extract files from fetched package source extractSource :: Attach PackageExtractSrc [FilePath] extractSource = (. pure . PackageExtractSrc . NE.fromList) . andThen --- | Run 'FetchRustGitDependencies' given the path to @Cargo.lock@+-- | Run 'FetchRustGitDependencies' given the path to @Cargo.lock@ files ----- The lock file will be extracted as well.-hasCargoLock :: Attach PackageCargoFilePath FilePath-hasCargoLock = (. pure . PackageCargoFilePath) . andThen+-- The lock files will be extracted as well.+hasCargoLocks :: Attach PackageCargoLockFiles [FilePath]+hasCargoLocks = (. pure . PackageCargoLockFiles . NE.fromList) . andThen -- | Set 'NvcheckerOptions' for a package, which can tweak the version number we obtain tweakVersion :: Attach NvcheckerOptions (NvcheckerOptions -> NvcheckerOptions)@@ -519,3 +537,9 @@ -- Arg is a list of kv pairs passthru :: Attach PackagePassthru [(Text, Text)] passthru = (. pure . PackagePassthru . HMap.fromList) . andThen++-- | Pin a package+--+-- new version won't be checked if we have a stale version+pinned :: PackageSet (Prod r) -> PackageSet (Prod (UseStaleVersion : r))+pinned = flip andThen . pure $ PermanentStale
src/NvFetcher/Types.hs view
@@ -3,17 +3,16 @@ {-# LANGUAGE DeriveGeneric #-} {-# LANGUAGE DerivingStrategies #-} {-# LANGUAGE FlexibleContexts #-}+{-# LANGUAGE FlexibleInstances #-} {-# LANGUAGE GeneralizedNewtypeDeriving #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE StandaloneDeriving #-}-{-# LANGUAGE TypeApplications #-} {-# LANGUAGE TypeFamilies #-} {-# LANGUAGE UndecidableInstances #-}-{-# LANGUAGE ViewPatterns #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -33,9 +32,11 @@ VersionSortMethod (..), ListOptions (..), VersionSource (..),- NvcheckerA (..),- NvcheckerQ (..),+ NvcheckerResult (..),+ NvcheckerRaw (..),+ CheckVersion (..), NvcheckerOptions (..),+ UseStaleVersion (..), -- * Nix fetcher types NixFetcher (..),@@ -55,10 +56,11 @@ PackageName, PackageFetcher, PackageExtractSrc (..),- PackageCargoFilePath (..),+ PackageCargoLockFiles (..), PackagePassthru (..), Package (..), PackageKey (..),+ PackageResult (..), ) where @@ -67,31 +69,32 @@ import Data.Default import Data.HashMap.Strict (HashMap) import qualified Data.List.NonEmpty as NE-import Data.Maybe (fromMaybe)+import Data.Maybe (fromMaybe, isNothing) import Data.String (IsString) import Data.Text (Text) import qualified Data.Text as T import Development.Shake import Development.Shake.Classes import GHC.Generics (Generic)+import Prettyprinter -------------------------------------------------------------------------------- -- | Package version newtype Version = Version Text- deriving newtype (Eq, Show, Ord, IsString, Semigroup, Monoid, A.FromJSON, A.ToJSON)+ deriving newtype (Eq, Show, Ord, IsString, Semigroup, Monoid, A.FromJSON, A.ToJSON, Pretty) deriving stock (Typeable, Generic) deriving anyclass (Hashable, Binary, NFData) -- | Check sum, sha256, sri or base32, etc. newtype Checksum = Checksum Text- deriving newtype (Show, Eq, Ord)+ deriving newtype (Show, Eq, Ord, A.FromJSON, A.ToJSON, Pretty) deriving stock (Typeable, Generic) deriving anyclass (Hashable, Binary, NFData) -- | Git branch ('Nothing': master) newtype Branch = Branch (Maybe Text)- deriving newtype (Show, Eq, Ord, Default)+ deriving newtype (Show, Eq, Ord, Default, Pretty) deriving stock (Typeable, Generic) deriving anyclass (Hashable, Binary, NFData) @@ -126,6 +129,10 @@ ParseVersion -> "parse_version" Vercmp -> "vercmp" +instance Pretty VersionSortMethod where+ pretty ParseVersion = "ParseVersion"+ pretty Vercmp = "Vercmp"+ instance Default VersionSortMethod where def = ParseVersion @@ -139,6 +146,27 @@ } deriving (Show, Typeable, Eq, Ord, Generic, Hashable, Binary, NFData, Default) +isEmptyListOptions :: ListOptions -> Bool+isEmptyListOptions ListOptions {..} =+ isNothing _includeRegex+ && isNothing _excludeRegex+ && isNothing _sortVersionKey+ && isNothing _includeRegex++instance Pretty ListOptions where+ pretty ListOptions {..} =+ "ListOptions"+ <> indent+ 2+ ( vsep $+ concat+ [ ppField "includeRegex" _includeRegex,+ ppField "excludeRegex" _excludeRegex,+ ppField "sortVersionKey" _sortVersionKey,+ ppField "ignored" _includeRegex+ ]+ )+ -- | Configuration available for evey version sourece. -- See <https://nvchecker.readthedocs.io/en/latest/usage.html#global-options> for details. data NvcheckerOptions = NvcheckerOptions@@ -148,6 +176,30 @@ } deriving (Show, Typeable, Eq, Ord, Generic, Hashable, Binary, NFData, Default) +isEmptyNvcheckerOptions :: NvcheckerOptions -> Bool+isEmptyNvcheckerOptions NvcheckerOptions {..} =+ isNothing _stripPrefix+ && isNothing _fromPattern+ && isNothing _toPattern++instance Pretty NvcheckerOptions where+ pretty NvcheckerOptions {..} =+ "NvcheckerOptions"+ <> line+ <> indent+ 2+ ( vsep $+ concat+ [ ppField "stripPrefix" _stripPrefix,+ ppField "fromPattern" _fromPattern,+ ppField "toPattern" _toPattern+ ]+ )++ppField :: Pretty a => Doc ann -> Maybe a -> [Doc ann]+ppField _ Nothing = []+ppField s (Just x) = [s <> colon <+> pretty x]+ -- | Upstream version source for nvchecker to check data VersionSource = GitHubRelease {_owner :: Text, _repo :: Text}@@ -162,26 +214,133 @@ | HttpHeader {_vurl :: Text, _regex :: Text, _listOptions :: ListOptions} | OpenVsx {_ovPublisher :: Text, _ovExtName :: Text} | VscodeMarketplace {_vsmPublisher :: Text, _vsmExtName :: Text}+ | Cmd {_vcmd :: Text} deriving (Show, Typeable, Eq, Ord, Generic, Hashable, Binary, NFData) +instance Pretty VersionSource where+ pretty GitHubRelease {..} =+ "CheckGitHubRelease"+ <> line+ <> indent+ 2+ ( vsep+ [ "owner" <> colon <+> pretty _owner,+ "repo" <> colon <+> pretty _repo+ ]+ )+ pretty GitHubTag {..} =+ "CheckGitHubTag"+ <> line+ <> indent+ 2+ ( vsep $+ [ "owner" <> colon <+> pretty _owner,+ "repo" <> colon <+> pretty _repo+ ]+ <> ["listOptions" <> colon <+> pretty _listOptions | not $ isEmptyListOptions _listOptions]+ )+ pretty Git {..} =+ "CheckGit"+ <> line+ <> indent+ 2+ ( vsep+ [ "url" <> colon <+> pretty _vurl,+ "branch" <> colon <+> pretty _vbranch+ ]+ )+ pretty Pypi {..} =+ "CheckPypi" <> colon <+> pretty _pypi+ pretty ArchLinux {..} =+ "CheckArchLinux" <> colon <+> pretty _archpkg+ pretty Aur {..} =+ "CheckAur" <> colon <+> pretty _aur+ pretty Manual {..} =+ "CheckManual" <> colon <+> pretty _manual+ pretty Repology {..} =+ "CheckRepology"+ <> line+ <> indent+ 2+ ( vsep+ [ "repology" <> colon <+> pretty _repology,+ "repo" <> colon <+> pretty _repo+ ]+ )+ pretty Webpage {..} =+ "CheckWebpage"+ <> line+ <> indent+ 2+ ( vsep $+ [ "url" <> colon <+> pretty _vurl,+ "regex" <> colon <+> pretty _regex+ ]+ <> ["listOptions" <> colon <+> pretty _listOptions | not $ isEmptyListOptions _listOptions]+ )+ pretty HttpHeader {..} =+ "CheckHttpHeader"+ <> line+ <> indent+ 2+ ( vsep $+ [ "url" <> colon <+> pretty _vurl,+ "regex" <> colon <+> pretty _regex+ ]+ <> ["listOptions" <> colon <+> pretty _listOptions | not $ isEmptyListOptions _listOptions]+ )+ pretty OpenVsx {..} =+ "CheckOpenVsx"+ <> line+ <> indent+ 2+ ( vsep+ [ "publisher" <> colon <+> pretty _ovPublisher,+ "extName" <> colon <+> pretty _ovExtName+ ]+ )+ pretty VscodeMarketplace {..} =+ "CheckVscodeMarketplace"+ <> line+ <> indent+ 2+ ( vsep+ [ "publisher" <> colon <+> pretty _vsmPublisher,+ "extName" <> colon <+> pretty _vsmExtName+ ]+ )+ pretty Cmd {..} =+ "CheckCmd" <> colon <+> pretty _vcmd+ -- | The input of nvchecker-data NvcheckerQ = NvcheckerQ VersionSource NvcheckerOptions+data CheckVersion = CheckVersion VersionSource NvcheckerOptions deriving (Show, Typeable, Eq, Ord, Generic, Hashable, Binary, NFData) --- | The result of running nvchecker-data NvcheckerA = NvcheckerA+instance Pretty CheckVersion where+ pretty (CheckVersion v n) = align (vsep $ [pretty v] <> [pretty n | not $ isEmptyNvcheckerOptions n])++-- | The result of nvchecker rule+data NvcheckerResult = NvcheckerResult { nvNow :: Version,- -- | nvchecker doesn't give this value, but shake restores it from last run- nvOld :: Maybe Version+ -- | last result of this nvchecker rule+ -- TODO: consider removing this field+ nvOld :: Maybe Version,+ -- | stale means even 'nvNow' comes from json file (last run)+ -- and we actually didn't run nvchecker this time. 'nvOld' will be 'Nothing' in this case.+ nvStale :: Bool } deriving (Show, Typeable, Eq, Generic, Hashable, Binary, NFData) -instance A.FromJSON NvcheckerA where- parseJSON = A.withObject "NvcheckerResult" $ \o ->- NvcheckerA <$> o A..: "version" <*> pure Nothing+-- | Parsed JSON output from nvchecker+newtype NvcheckerRaw = NvcheckerRaw Version+ deriving (Show, Typeable, Eq, Generic) -type instance RuleResult NvcheckerQ = NvcheckerA+instance A.FromJSON NvcheckerRaw where+ parseJSON = A.withObject "NvcheckerRaw" $ \o ->+ NvcheckerRaw <$> o A..: "version" +type instance RuleResult CheckVersion = NvcheckerResult+ -------------------------------------------------------------------------------- -- | If the package is prefetched, then we can obtain the SHA256@@ -192,9 +351,28 @@ _deepClone :: Bool, _fetchSubmodules :: Bool, _leaveDotGit :: Bool,+ _name :: Maybe Text, _sha256 :: FetchResult k }- | FetchUrl {_furl :: Text, _sha256 :: FetchResult k}+ | FetchGitHub+ { _fowner :: Text,+ _frepo :: Text,+ _rev :: Version,+ _deepClone :: Bool,+ _fetchSubmodules :: Bool,+ _leaveDotGit :: Bool,+ _name :: Maybe Text,+ _sha256 :: FetchResult k+ }+ | FetchUrl+ { _furl :: Text,+ _name :: Maybe Text,+ _sha256 :: FetchResult k+ }+ | FetchTarball+ { _furl :: Text,+ _sha256 :: FetchResult k+ } deriving (Typeable, Generic) -- | Fetch status@@ -219,6 +397,86 @@ deriving instance NFData (FetchResult k) => NFData (NixFetcher k) +instance A.ToJSON (NixFetcher Fetched) where+ toJSON FetchGit {..} =+ A.object+ [ "url" A..= _furl,+ "rev" A..= _rev,+ "deepClone" A..= _deepClone,+ "fetchSubmodules" A..= _fetchSubmodules,+ "leaveDotGit" A..= _leaveDotGit,+ "name" A..= _name,+ "sha256" A..= _sha256,+ "type" A..= A.String "git"+ ]+ toJSON FetchGitHub {..} =+ A.object+ [ "owner" A..= _fowner,+ "repo" A..= _frepo,+ "rev" A..= _rev,+ "deepClone" A..= _deepClone,+ "fetchSubmodules" A..= _fetchSubmodules,+ "leaveDotGit" A..= _leaveDotGit,+ "name" A..= _name,+ "sha256" A..= _sha256,+ "type" A..= A.String "github"+ ]+ toJSON FetchUrl {..} =+ A.object+ [ "url" A..= _furl,+ "name" A..= _name,+ "sha256" A..= _sha256,+ "type" A..= A.String "url"+ ]+ toJSON FetchTarball {..} =+ A.object+ [ "url" A..= _furl,+ "sha256" A..= _sha256,+ "type" A..= A.String "tarball"+ ]++instance Pretty (NixFetcher k) where+ pretty FetchGit {..} =+ "FetchGit"+ <> line+ <> indent+ 2+ ( vsep $+ [ "url" <> colon <+> pretty _furl,+ "rev" <> colon <+> pretty _rev,+ "deepClone" <> colon <+> pretty _deepClone,+ "fetchSubmodules" <> colon <+> pretty _fetchSubmodules,+ "leaveDotGit" <> colon <+> pretty _leaveDotGit+ ]+ <> ppField "name" _name+ )+ pretty FetchGitHub {..} =+ "FetchGitHub"+ <> line+ <> indent+ 2+ ( vsep $+ [ "owner" <> colon <+> pretty _fowner,+ "repo" <> colon <+> pretty _frepo,+ "rev" <> colon <+> pretty _rev,+ "deepClone" <> colon <+> pretty _deepClone,+ "fetchSubmodules" <> colon <+> pretty _fetchSubmodules,+ "leaveDotGit" <> colon <+> pretty _leaveDotGit+ ]+ <> ppField "name" _name+ )+ pretty FetchUrl {..} =+ "FetchUrl"+ <> line+ <> indent+ 2+ ( vsep $+ ["url" <> colon <+> pretty _furl]+ <> ppField "name" _name+ )+ pretty FetchTarball {..} =+ "FetchTarball" <> colon <+> pretty _furl+ -------------------------------------------------------------------------------- -- | Extract file contents from package source@@ -228,6 +486,17 @@ type instance RuleResult ExtractSrcQ = HashMap FilePath Text +instance Pretty ExtractSrcQ where+ pretty (ExtractSrcQ f n) =+ "ExtractSrc" <> line+ <> indent+ 2+ ( vsep+ [ "fetcher" <> colon <+> pretty f,+ "files" <> colon <+> pretty n+ ]+ )+ -------------------------------------------------------------------------------- -- | Fetch @outputHashes@ for git dependencies in @Cargo.lock@.@@ -239,6 +508,17 @@ -- | @outputHashes@, a mapping from nameVer -> output hash type instance RuleResult FetchRustGitDepsQ = HashMap Text Checksum +instance Pretty FetchRustGitDepsQ where+ pretty (FetchRustGitDepsQ f n) =+ "FetchRustGitDeps" <> line+ <> indent+ 2+ ( vsep+ [ "fetcher" <> colon <+> pretty f,+ "cargoLock" <> colon <+> pretty n+ ]+ )+ -------------------------------------------------------------------------------- -- | Package name, used in generating nix expr@@ -249,36 +529,51 @@ newtype PackageExtractSrc = PackageExtractSrc (NE.NonEmpty FilePath) -newtype PackageCargoFilePath = PackageCargoFilePath FilePath+newtype PackageCargoLockFiles = PackageCargoLockFiles (NE.NonEmpty FilePath) newtype PackagePassthru = PackagePassthru (HashMap Text Text) deriving newtype (Semigroup, Monoid) +-- | Using stale value indicates that we will /NOT/ check for new versions if+-- there is a known version recoverd from json file or last use of the rule.+-- Normally you don't want a stale version+-- unless you need pin a package.+data UseStaleVersion+ = -- | Specified in configuration file+ PermanentStale+ | -- | Specified by @--filter@ command+ TemporaryStale+ | NoStale+ deriving stock (Eq, Show, Ord, Typeable, Generic)+ deriving anyclass (Hashable, Binary, NFData)+ -- | A package is defined with: -- -- 1. its name -- 2. how to track its version -- 3. how to fetch it as we have the version--- 4. optional file paths to extract (dump to shake dir)+-- 4. optional file paths to extract (dump to build dir) -- 5. optional @Cargo.lock@ path (if it's a rust package) -- 6. an optional pass through map+-- 7. if the package version was pinned -- -- /INVARIANT: 'Version' passed to 'PackageFetcher' MUST be used textually,/ -- /i.e. can only be concatenated with other strings,/ -- /in case we can't check the equality between fetcher functions./ data Package = Package { _pname :: PackageName,- _pversion :: NvcheckerQ,+ _pversion :: CheckVersion, _pfetcher :: PackageFetcher, _pextract :: Maybe PackageExtractSrc,- _pcargo :: Maybe PackageCargoFilePath,- _ppassthru :: PackagePassthru+ _pcargo :: Maybe PackageCargoLockFiles,+ _ppassthru :: PackagePassthru,+ _ppinned :: UseStaleVersion } -- | Package key is the name of a package. -- We use this type to index packages. newtype PackageKey = PackageKey PackageName- deriving newtype (Eq, Show, Ord)+ deriving newtype (Eq, Show, Ord, Pretty) deriving stock (Typeable, Generic) deriving anyclass (Hashable, Binary, NFData) @@ -288,7 +583,7 @@ data Core = Core deriving (Eq, Show, Ord, Typeable, Generic, Hashable, Binary, NFData) -type instance RuleResult Core = NixExpr+type instance RuleResult Core = PackageResult -- | Decorate a rule's key with 'PackageKey' newtype WithPackageKey k = WithPackageKey (k, PackageKey)@@ -298,3 +593,31 @@ show (WithPackageKey (k, n)) = show k <> " (" <> show n <> ")" type instance RuleResult (WithPackageKey k) = RuleResult k++-- | Result type of 'Core'+data PackageResult = PackageResult+ { _prname :: PackageName,+ _prversion :: NvcheckerResult,+ _prfetched :: NixFetcher 'Fetched,+ _prpassthru :: Maybe (HashMap Text Text),+ -- | extracted file name -> file path in build dir+ _prextract :: Maybe (HashMap FilePath NixExpr),+ -- | cargo lock file path in build dir -> (file path in nix, git dependencies)+ _prcargolock :: Maybe (HashMap FilePath (NixExpr, HashMap Text Checksum)),+ _prpinned :: UseStaleVersion+ }+ deriving (Show, Typeable, Generic, NFData)++instance A.ToJSON PackageResult where+ toJSON PackageResult {..} =+ A.object+ [ "name" A..= _prname,+ "version" A..= nvNow _prversion,+ "src" A..= _prfetched,+ "extract" A..= _prextract,+ "passthru" A..= _prpassthru,+ "cargoLocks" A..= _prcargolock,+ "pinned" A..= case _prpinned of+ PermanentStale -> True+ _ -> False+ ]
src/NvFetcher/Types/Lens.hs view
@@ -1,7 +1,7 @@ {-# LANGUAGE DataKinds #-} {-# LANGUAGE TemplateHaskell #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental
src/NvFetcher/Types/ShakeExtras.hs view
@@ -2,7 +2,7 @@ {-# LANGUAGE RecordWildCards #-} {-# LANGUAGE TypeApplications #-} --- | Copyright: (c) 2021 berberman+-- | Copyright: (c) 2021-2022 berberman -- SPDX-License-Identifier: MIT -- Maintainer: berberman <berberman@yandex.com> -- Stability: experimental@@ -24,8 +24,21 @@ recordVersionChange, getVersionChanges, - -- * Retries- withRetries,+ -- * Retry+ withRetry,++ -- * Build dir+ getBuildDir,++ -- * Last versions+ getLastVersionOnDisk,+ getRecentLastVersion,+ updateLastVersion,+ getAllOnDiskVersions,+ getLastVersionUpdated,++ -- * Cache nvchecker+ nvcheckerCacheEnabled, ) where @@ -33,13 +46,22 @@ import Data.Map.Strict (Map) import qualified Data.Map.Strict as Map import Development.Shake+import NvFetcher.Config import NvFetcher.Types +data LastVersion+ = OnDisk Version+ | Updated+ (Maybe Version)+ -- ^ on disk if has+ Version+ -- | Values we use during the build. It's stored in 'shakeExtra' data ShakeExtras = ShakeExtras- { versionChanges :: Var [VersionChange],+ { config :: Config,+ versionChanges :: Var [VersionChange], targetPackages :: Map PackageKey Package,- retries :: Int+ lastVersions :: Var (Map PackageKey LastVersion) } -- | Get our values from shake@@ -49,10 +71,12 @@ Just x -> pure x _ -> fail "ShakeExtras is missing!" --- | Create an empty 'ShakeExtras' from packages to build and times to retry for each rule-initShakeExtras :: Map PackageKey Package -> Int -> IO ShakeExtras-initShakeExtras targetPackages retries = do+-- | Create an empty 'ShakeExtras' from packages to build, times to retry for each rule,+-- build dir, and on disk versions+initShakeExtras :: Config -> Map PackageKey Package -> Map PackageKey Version -> IO ShakeExtras+initShakeExtras config targetPackages lv = do versionChanges <- newVar mempty+ lastVersions <- newVar $ Map.map OnDisk lv pure ShakeExtras {..} -- | Get keys of all packages to build@@ -83,6 +107,64 @@ ShakeExtras {..} <- getShakeExtras liftIO $ readVar versionChanges --- | Run an action, retry at most 'retries' times if it throws an exception-withRetries :: Action a -> Action a-withRetries a = getShakeExtras >>= \ShakeExtras {..} -> actionRetry retries a+-- | Run an action, retry at most 'retry' times (defined in config) if it throws an exception+withRetry :: Action a -> Action a+withRetry a = getShakeExtras >>= \ShakeExtras {..} -> actionRetry (retry config) a++-- | Get build dir+getBuildDir :: Action FilePath+getBuildDir = buildDir . config <$> getShakeExtras++-- | Get initial version of a package+getLastVersionOnDisk :: PackageKey -> Action (Maybe Version)+getLastVersionOnDisk k = do+ ShakeExtras {..} <- getShakeExtras+ versions <- liftIO $ readVar lastVersions+ pure $ case versions Map.!? k of+ Just (Updated v _) -> v+ Just (OnDisk v) -> Just v+ _ -> Nothing++-- | Get version of a package, no matter it was initial one or rule result+getRecentLastVersion :: PackageKey -> Action (Maybe Version)+getRecentLastVersion k = do+ ShakeExtras {..} <- getShakeExtras+ versions <- liftIO $ readVar lastVersions+ pure $ case versions Map.!? k of+ Just (Updated _ v) -> Just v+ Just (OnDisk v) -> Just v+ _ -> Nothing++-- | Get updated version of a package+getLastVersionUpdated :: PackageKey -> Action (Maybe Version)+getLastVersionUpdated k = do+ ShakeExtras {..} <- getShakeExtras+ versions <- liftIO $ readVar lastVersions+ pure $ case versions Map.!? k of+ Just (Updated _ v) -> Just v+ _ -> Nothing++-- | Add nvchecker result of a package+updateLastVersion :: PackageKey -> Version -> Action ()+updateLastVersion k v = do+ ShakeExtras {..} <- getShakeExtras+ liftIO $+ modifyVar_ lastVersions $ \versions -> pure $ case versions Map.!? k of+ Just (Updated o _) -> Map.insert k (Updated o v) versions+ Just (OnDisk lv) -> Map.insert k (Updated (Just lv) v) versions+ _ -> Map.insert k (Updated Nothing v) versions++-- | Get all initial versions+getAllOnDiskVersions :: Action (Map PackageKey Version)+getAllOnDiskVersions = do+ ShakeExtras {..} <- getShakeExtras+ versions <- liftIO $ readVar lastVersions+ let xs = Map.toList $+ flip Map.map versions $ \case+ OnDisk v -> Just v+ Updated v _ -> v+ pure $ Map.fromList [(k, v) | (k, Just v) <- xs]++-- | Get if 'cacheNvchecker' is enabled+nvcheckerCacheEnabled :: Action Bool+nvcheckerCacheEnabled = cacheNvchecker . config <$> getShakeExtras
src/NvFetcher/Utils.hs view
@@ -1,3 +1,10 @@+{-# LANGUAGE CPP #-}++-- | Copyright: (c) 2021-2022 berberman+-- SPDX-License-Identifier: MIT+-- Maintainer: berberman <berberman@yandex.com>+-- Stability: experimental+-- Portability: portable module NvFetcher.Utils where import Data.Binary@@ -5,7 +12,11 @@ import qualified Data.ByteString.Lazy as LBS import Data.Text (Text) import qualified Data.Text as T-import Development.Shake+import System.Directory.Extra (XdgDirectory (XdgData), getXdgDirectory)+import Text.Regex.TDFA ((=~))+#if MIN_VERSION_aeson(2,0,0)+import qualified Data.Aeson.Key as A+#endif encode' :: Binary a => a -> BS.ByteString encode' = BS.concat . LBS.toChunks . encode@@ -13,8 +24,24 @@ decode' :: Binary a => BS.ByteString -> a decode' = decode . LBS.fromChunks . pure -asString :: Text -> Text-asString = T.pack . show+quote :: Text -> Text+quote = T.pack . show -getShakeDir :: Action FilePath-getShakeDir = shakeFiles <$> getShakeOptions+isLegalNixId :: Text -> Bool+isLegalNixId x = x =~ "^[a-zA-Z_][a-zA-Z0-9_'-]*$"++quoteIfNeeds :: Text -> Text+quoteIfNeeds x+ | isLegalNixId x = x+ | otherwise = quote x++getDataDir :: IO FilePath+getDataDir = getXdgDirectory XdgData "nvfetcher"++#if MIN_VERSION_aeson(2,0,0)+aesonKey :: Text -> A.Key+aesonKey = A.fromText+#else+aesonKey :: Text -> Text+aesonKey = id+#endif
test/CheckVersionSpec.hs view
@@ -2,20 +2,29 @@ module CheckVersionSpec where +import Control.Monad.IO.Class (liftIO) import Control.Monad.Trans.Reader import Data.Coerce (coerce) import Data.Default (def) import qualified Data.Map.Strict as Map+import qualified Data.Text as T import Lens.Micro+import NvFetcher.Config (Config (cacheNvchecker)) import NvFetcher.Nvchecker import NvFetcher.Types import NvFetcher.Types.Lens+import System.IO.Extra (newTempFile) import Test.Hspec import Utils --- | We need a fakePackageKey here; otherwise the nvchecker rule would be cutoff spec :: Spec-spec = aroundShake' (Map.singleton fakePackageKey fakePackage) $+spec = do+ versionSourcesSpec+ useStaleSpec+ cacheSpec++versionSourcesSpec :: Spec+versionSourcesSpec = aroundShake $ describe "nvchecker" $ do specifyChan "pypi" $ runNvcheckerRule (Pypi "example") `shouldReturnJust` Version "0.1.0"@@ -43,10 +52,14 @@ runNvcheckerRule (GitHubTag "harry-sanabria" "ReleaseTestRepo" $ def & ignored ?~ "second_release release3") `shouldReturnJust` Version "first_release" - specifyChan "http header" $ do+ specifyChan "http header" $ runNvcheckerRule (HttpHeader "https://www.unifiedremote.com/download/linux-x64-deb" "urserver-([\\d.]+).deb" def) >>= shouldBeJust + -- specifyChan "webpage" $+ -- runNvcheckerRule (Webpage "http://ftp.vim.org/pub/vim/patches/7.3/" "7\\.3\\.\\d+" def)+ -- `shouldReturnJust` Version "7.3.1314"+ specifyChan "manual" $ runNvcheckerRule (Manual "Meow") `shouldReturnJust` Version "Meow" @@ -59,11 +72,54 @@ specifyChan "vsmarketplace" $ runNvcheckerRule (VscodeMarketplace "usernamehw" "indent-one-space") `shouldReturnJust` Version "0.2.8" + specifyChan "cmd" $+ runNvcheckerRule (Cmd "echo Meow") `shouldReturnJust` Version "Meow"+ -------------------------------------------------------------------------------- +-- | We need disable nvchecker cache to see if use stale works+useStaleSpec :: Spec+useStaleSpec = aroundShake' (Map.singleton fakePackageKey fakePinnedPackage) def {cacheNvchecker = False} $+ describe "useStale" $ do+ (temp, cleanup) <- runIO newTempFile++ let versionSource = Cmd $ "cat " <> T.pack temp++ specifyChan "needs run" $ do+ liftIO $ writeFile temp "Meow"+ runNvcheckerRuleOnFakePackage versionSource `shouldReturnJust` NvcheckerResult {nvNow = "Meow", nvOld = Nothing, nvStale = False}++ specifyChan "stale" $ do+ liftIO $ writeFile temp "Bark"+ runNvcheckerRuleOnFakePackage versionSource `shouldReturnJust` NvcheckerResult {nvNow = "Meow", nvOld = Just "Meow", nvStale = True}++ runIO cleanup++cacheSpec :: Spec+cacheSpec = aroundShake' (Map.singleton fakePackageKey fakePackage) def {cacheNvchecker = True} $+ describe "cache" $ do+ (temp, cleanup) <- runIO newTempFile++ let versionSource = Cmd $ "cat " <> T.pack temp++ specifyChan "needs run" $ do+ liftIO $ writeFile temp "Meow"+ runNvcheckerRuleOnFakePackage versionSource `shouldReturnJust` NvcheckerResult {nvNow = "Meow", nvOld = Nothing, nvStale = False}++ specifyChan "cache" $ do+ liftIO $ writeFile temp "Bark"+ runNvcheckerRuleOnFakePackage versionSource `shouldReturnJust` NvcheckerResult {nvNow = "Meow", nvOld = Just "Meow", nvStale = False}++ runIO cleanup++--------------------------------------------------------------------------------+ runNvcheckerRule :: VersionSource -> ReaderT ActionQueue IO (Maybe Version)-runNvcheckerRule v = runActionChan $ nvNow <$> checkVersion v def fakePackageKey+runNvcheckerRule v = fmap nvNow <$> runActionChan (checkVersion' v def) +runNvcheckerRuleOnFakePackage :: VersionSource -> ReaderT ActionQueue IO (Maybe NvcheckerResult)+runNvcheckerRuleOnFakePackage v = runActionChan $ checkVersion v def fakePackageKey+ fakePackageKey :: PackageKey fakePackageKey = PackageKey "a-fake-package" @@ -75,5 +131,9 @@ _pfetcher = undefined, _pcargo = undefined, _pextract = undefined,- _ppassthru = undefined+ _ppassthru = undefined,+ _ppinned = NoStale }++fakePinnedPackage :: Package+fakePinnedPackage = fakePackage {_ppinned = PermanentStale}
test/FetchRustGitDepsSpec.hs view
@@ -26,7 +26,7 @@ shouldBeJust prefetched runFetchRustGitDepsRule (fromJust prefetched) "Cargo.lock" `shouldReturnJust` HMap.fromList- [ ("rand-0.8.3", Checksum "1khg0rnz8xxd389cprqmy9vq9sggzz78lb9n7hh2w6xfsl4xfyyc")+ [ ("rand-0.8.3", Checksum "sha256-zHvXCdWuGy4gPDYtis7/7+mEd/IV58sSGq139G0GD84=") ] runPrefetchRule :: NixFetcher Fresh -> ReaderT ActionQueue IO (Maybe (NixFetcher Fetched))
test/NixExprSpec.hs view
@@ -35,6 +35,18 @@ } |] + it "renders fresh gitHubFetcher" $+ toNixExpr (gitHubFetcher ("owner", "repo") "fake_rev")+ `shouldBe` [trimming|+ fetchFromGitHub ({+ owner = "owner";+ repo = "repo";+ rev = "fake_rev";+ fetchSubmodules = false;+ sha256 = lib.fakeSha256;+ })+ |]+ it "renders fresh urlFetcher" $ toNixExpr (urlFetcher "https://example.com") `shouldBe` [trimming|@@ -44,10 +56,20 @@ } |] + it "renders filename for vsc extension" $+ toNixExpr (openVsxFetcher ("publisher", "extension") "fake_version")+ `shouldBe` [trimming|+ fetchurl {+ url = "https://open-vsx.org/api/publisher/extension/fake_version/file/publisher.extension-fake_version.vsix";+ name = "extension-fake_version.zip";+ sha256 = lib.fakeSha256;+ }+ |]+ it "renders IFD of ExtractSrcQ" $ toNixExpr ( ExtractSrcQ- (FetchUrl "https://example.com" (Checksum "calculated sha256"))+ (FetchUrl "https://example.com" Nothing (Checksum "calculated sha256")) (NE.fromList ["a.txt", "b.txt"]) ) `shouldBe` [trimming|
test/PrefetchSpec.hs view
@@ -14,23 +14,27 @@ describe "fetchers" $ do specifyChan "pypi" $ runPrefetchRule (pypiFetcher "example" "0.1.0")- `shouldReturnJust` Checksum "1fy3zylvks372jz7fkgad3x14bscz19m2anlq2bn8xs1r1p3x1zm"+ `shouldReturnJust` Checksum "sha256-9Yc+bshBd2SXwNQqUVP4TC8S+mjqTXe+FGfouan/w7s=" specifyChan "openvsx" $ runPrefetchRule (openVsxFetcher ("usernamehw", "indent-one-space") "0.2.6")- `shouldReturnJust` Checksum "0smb75z10h1aiqmj2irn2vz1z99djfsfpkczwiqk447frx388bd1"+ `shouldReturnJust` Checksum "sha256-oS2ERs/uEDJx5J/N67STLaUf/hY2RyErjipAEH45q2o=" specifyChan "vsmarketplace" $ runPrefetchRule (vscodeMarketplaceFetcher ("usernamehw", "indent-one-space") "0.2.6")- `shouldReturnJust` Checksum "1vmq24hdbv8jwhvy85s7qq9gffcsndvsw8vmphxs95r7bc3439w7"+ `shouldReturnJust` Checksum "sha256-h6dBBlsnl6Q7vHUjrnezmjn3EsZHF+Q35BLt1SARuO4=" specifyChan "git" $- runPrefetchRule (gitFetcher "https://gitlab.com/gitlab-org/gitlab-test.git" "ddd0f15ae83993f5cb66a927a28673882e99100b")- `shouldReturnJust` Checksum "10d0yqa0h00pd5a36nwx3hb4apd4f9hki0pi545ffgrpf9nzzg92"+ runPrefetchRule (gitFetcher "https://github.com/git-up/test-repo-submodules" "5a1dfa807759c39e3df891b6b46dfb2cf776c6ef")+ `shouldReturnJust` Checksum "sha256-wCo1YobyatxSOE85xQNSJw6jvufghFNHlZl4ToQjRHA=" specifyChan "github" $ runPrefetchRule (gitHubFetcher ("harry-sanabria", "ReleaseTestRepo") "release3")- `shouldReturnJust` Checksum "1a8qv2h9ji6w5p4ljwwkgvk49w6hj9j7hgmw0ahw10y1i45s0b3i"+ `shouldReturnJust` Checksum "sha256-cSygC4nBg8ChArw+eGSS0PBE5n6Tc0nJLdxEmaDYGKk="++ specifyChan "tarball" $+ runPrefetchRule (tarballFetcher "https://github.com/nixos/nixpkgs/archive/3d35529a48d3ad50ad959463755b0b7fe392cfa7.tar.gz")+ `shouldReturnJust` Checksum "0la68sv52zz1kjw5s5sn6qslzz9mi9sakhzwi873gp4dhc8df1sg" --------------------------------------------------------------------------------
test/Utils.hs view
@@ -12,10 +12,12 @@ import Control.Monad (void) import Control.Monad.IO.Class import Control.Monad.Trans.Reader+import Data.Default (def) import Data.Map (Map) import Data.Maybe (isJust) import Development.Shake import Development.Shake.Database+import NvFetcher.Config import NvFetcher.Core (coreRules) import NvFetcher.Types import NvFetcher.Types.ShakeExtras@@ -35,9 +37,9 @@ type ActionQueue = TQueue (Action ()) -newAsyncActionQueue :: Map PackageKey Package -> IO (ActionQueue, Async ())-newAsyncActionQueue pkgs = Extra.withTempDir $ \dir -> do- shakeExtras <- liftIO $ initShakeExtras pkgs 3+newAsyncActionQueue :: Map PackageKey Package -> Config -> IO (ActionQueue, Async ())+newAsyncActionQueue pkgs config = Extra.withTempDir $ \dir -> do+ shakeExtras <- liftIO $ initShakeExtras config {buildDir = dir} pkgs mempty chan <- atomically newTQueue (getShakeDb, _) <- shakeOpenDatabase@@ -65,12 +67,12 @@ -------------------------------------------------------------------------------- aroundShake :: SpecWith ActionQueue -> Spec-aroundShake = aroundShake' mempty+aroundShake = aroundShake' mempty def -aroundShake' :: Map PackageKey Package -> SpecWith ActionQueue -> Spec-aroundShake' pkgs = aroundAll $ \f ->+aroundShake' :: Map PackageKey Package -> Config -> SpecWith ActionQueue -> Spec+aroundShake' pkgs config = aroundAll $ \f -> bracket- (newAsyncActionQueue pkgs)+ (newAsyncActionQueue pkgs config) (\(_, runnerTask) -> cancel runnerTask) (\(chan, _) -> f chan)