diff --git a/ChangeLog.md b/ChangeLog.md --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,12 @@ # Changelog for pantry +## v0.1.1.2 + +Bug fixes: + +* Module mapping insertions into the database are now atomic. Previously, if + you SIGTERMed at the wrong time while running a script, you could end up with + an inconsistent database state. ## v0.1.1.1 diff --git a/pantry.cabal b/pantry.cabal --- a/pantry.cabal +++ b/pantry.cabal @@ -1,18 +1,24 @@ cabal-version: 1.12 -name: pantry -version: 0.1.1.1 -license: BSD3 -license-file: LICENSE -copyright: 2018-2019 FP Complete -maintainer: michael@snoyman.com -author: Michael Snoyman -homepage: https://github.com/commercialhaskell/stack#readme -bug-reports: https://github.com/commercialhaskell/stack/issues -synopsis: Content addressable Haskell package management -description: - Please see the README on Github at -category: Development -build-type: Simple + +-- This file has been generated from package.yaml by hpack version 0.31.2. +-- +-- see: https://github.com/sol/hpack +-- +-- hash: e338943a1a53aa6c588e00a1b9eb2d38801ab6382e7ea4c33b31bd9eea0fb994 + +name: pantry +version: 0.1.1.2 +synopsis: Content addressable Haskell package management +description: Please see the README on Github at +category: Development +homepage: https://github.com/commercialhaskell/pantry#readme +bug-reports: https://github.com/commercialhaskell/pantry/issues +author: Michael Snoyman +maintainer: michael@snoyman.com +copyright: 2018-2019 FP Complete +license: BSD3 +license-file: LICENSE +build-type: Simple extra-source-files: README.md ChangeLog.md @@ -21,195 +27,198 @@ attic/symlink-to-dir.tar.gz source-repository head - type: git - location: https://github.com/commercialhaskell/stack + type: git + location: https://github.com/commercialhaskell/pantry library - exposed-modules: - Pantry - Pantry.SHA256 - Pantry.Internal - Pantry.Internal.StaticBytes - Pantry.Internal.Stackage - Pantry.Internal.Companion - Pantry.Internal.AesonExtended - hs-source-dirs: src/ + exposed-modules: + Pantry + Pantry.SHA256 + Pantry.Internal + Pantry.Internal.StaticBytes + Pantry.Internal.Stackage + Pantry.Internal.Companion + Pantry.Internal.AesonExtended + other-modules: + Hackage.Security.Client.Repository.HttpLib.HttpClient + Pantry.Archive + Pantry.HTTP + Pantry.HPack + Pantry.Hackage + Pantry.Repo + Pantry.SQLite + Pantry.Storage + Pantry.Tree + Pantry.Types + hs-source-dirs: + src/ + default-extensions: MonadFailDesugaring + ghc-options: -Wall + build-depends: + Cabal + , aeson + , ansi-terminal + , array + , base >=4.10 && <5 + , base-orphans + , base64-bytestring + , bytestring + , conduit + , conduit-extra + , containers + , contravariant + , cryptonite + , cryptonite-conduit + , deepseq + , digest + , directory + , filelock + , filepath + , generic-deriving + , ghc-prim + , hackage-security + , hashable + , hpack >=0.31.2 + , http-client + , http-client-tls + , http-conduit + , http-download + , http-types + , integer-gmp + , memory + , mono-traversable + , mtl + , network + , network-uri + , path + , path-io + , persistent + , persistent-sqlite >=2.9.3 + , persistent-template + , primitive + , resourcet + , rio + , rio-orphans + , rio-prettyprint + , safe + , syb + , tar-conduit + , template-haskell + , text + , text-metrics + , th-lift + , th-lift-instances + , th-orphans + , th-reify-many + , th-utilities + , time + , transformers + , unix-compat + , unliftio + , unordered-containers + , vector + , yaml + , zip-archive + if os(windows) other-modules: - Hackage.Security.Client.Repository.HttpLib.HttpClient - Pantry.Archive - Pantry.HTTP - Pantry.HPack - Pantry.Hackage - Pantry.Repo - Pantry.SQLite - Pantry.Storage - Pantry.Tree - Pantry.Types - default-language: Haskell2010 - default-extensions: MonadFailDesugaring - ghc-options: -Wall - build-depends: - Cabal >=2.4.0.1, - aeson >=1.2.4.0, - ansi-terminal >=0.9, - array >=0.5.2.0, - base >=4.10 && <5, - base-orphans >=0.7, - base64-bytestring >=1.0.0.1, - bytestring >=0.10.8.2, - conduit >=1.3.0.3, - conduit-extra >=1.3.0, - containers >=0.5.10.2, - contravariant >=1.4.1, - cryptonite >=0.25, - cryptonite-conduit >=0.2.2, - deepseq >=1.4.3.0, - digest >=0.0.1.2, - directory >=1.3.0.2, - filelock >=0.1.1.2, - filepath >=1.4.1.2, - generic-deriving >=1.12.2, - ghc-prim >=0.5.1.1, - hackage-security >=0.5.3.0, - hashable >=1.2.7.0, - hpack >=0.31.2, - http-client >=0.5.13.1, - http-client-tls >=0.3.5.3, - http-conduit >=2.3.2, - http-download >=0.1.0.0, - http-types >=0.12.1, - integer-gmp >=1.0.1.0, - memory >=0.14.16, - mono-traversable >=1.0.9.0, - mtl >=2.2.2, - network >=2.6.3.6, - network-uri >=2.6.1.0, - path >=0.6.1, - path-io >=1.3.3, - persistent >=2.9.2, - persistent-sqlite >=2.9.3, - persistent-template >=2.5.4, - primitive >=0.6.4.0, - resourcet >=1.2.1, - rio >=0.1.10.0, - rio-orphans >=0.1.1.0, - rio-prettyprint >=0.1.0.0, - safe >=0.3.17, - syb >=0.7, - tar-conduit >=0.3.0, - template-haskell >=2.12.0.0, - text >=1.2.3.0, - text-metrics >=0.3.0, - th-lift >=0.7.10, - th-lift-instances >=0.1.11, - th-orphans >=0.13.6, - th-reify-many >=0.1.8, - th-utilities >=0.2.0.1, - time >=1.8.0.2, - transformers >=0.5.2.0, - unix-compat >=0.5.0.1, - unliftio >=0.2.8.0, - unordered-containers >=0.2.9.0, - vector >=0.12.0.1, - yaml >=0.10.4.0, - zip-archive >=0.3.3 - - if os(windows) - hs-source-dirs: src/windows/ - other-modules: - System.IsWindows - else - hs-source-dirs: src/unix/ - other-modules: - System.IsWindows + System.IsWindows + hs-source-dirs: + src/windows/ + else + other-modules: + System.IsWindows + hs-source-dirs: + src/unix/ + default-language: Haskell2010 test-suite spec - type: exitcode-stdio-1.0 - main-is: Spec.hs - hs-source-dirs: test - other-modules: - Pantry.ArchiveSpec - Pantry.BuildPlanSpec - Pantry.CabalSpec - Pantry.FileSpec - Pantry.GlobalHintsSpec - Pantry.HackageSpec - Pantry.Internal.StaticBytesSpec - Pantry.InternalSpec - Pantry.TreeSpec - Pantry.TypesSpec - Paths_pantry - default-language: Haskell2010 - default-extensions: MonadFailDesugaring - ghc-options: -Wall - build-depends: - Cabal >=2.4.0.1, - QuickCheck >=2.10.1, - aeson >=1.2.4.0, - ansi-terminal >=0.9, - array >=0.5.2.0, - base >=4.10 && <5, - base-orphans >=0.7, - base64-bytestring >=1.0.0.1, - bytestring >=0.10.8.2, - conduit >=1.3.0.3, - conduit-extra >=1.3.0, - containers >=0.5.10.2, - contravariant >=1.4.1, - cryptonite >=0.25, - cryptonite-conduit >=0.2.2, - deepseq >=1.4.3.0, - digest >=0.0.1.2, - directory >=1.3.0.2, - exceptions >=0.8.3, - filelock >=0.1.1.2, - filepath >=1.4.1.2, - generic-deriving >=1.12.2, - ghc-prim >=0.5.1.1, - hackage-security >=0.5.3.0, - hashable >=1.2.7.0, - hedgehog >=0.6.1, - hpack >=0.31.2, - hspec >=2.4.8, - http-client >=0.5.13.1, - http-client-tls >=0.3.5.3, - http-conduit >=2.3.2, - http-download >=0.1.0.0, - http-types >=0.12.1, - integer-gmp >=1.0.1.0, - memory >=0.14.16, - mono-traversable >=1.0.9.0, - mtl >=2.2.2, - network >=2.6.3.6, - network-uri >=2.6.1.0, - pantry -any, - path >=0.6.1, - path-io >=1.3.3, - persistent >=2.9.2, - persistent-sqlite >=2.9.3, - persistent-template >=2.5.4, - primitive >=0.6.4.0, - raw-strings-qq >=1.1, - resourcet >=1.2.1, - rio >=0.1.10.0, - rio-orphans >=0.1.1.0, - rio-prettyprint >=0.1.0.0, - safe >=0.3.17, - syb >=0.7, - tar-conduit >=0.3.0, - template-haskell >=2.12.0.0, - text >=1.2.3.0, - text-metrics >=0.3.0, - th-lift >=0.7.10, - th-lift-instances >=0.1.11, - th-orphans >=0.13.6, - th-reify-many >=0.1.8, - th-utilities >=0.2.0.1, - time >=1.8.0.2, - transformers >=0.5.2.0, - unix-compat >=0.5.0.1, - unliftio >=0.2.8.0, - unordered-containers >=0.2.9.0, - vector >=0.12.0.1, - yaml >=0.10.4.0, - zip-archive >=0.3.3 + type: exitcode-stdio-1.0 + main-is: Spec.hs + other-modules: + Pantry.ArchiveSpec + Pantry.BuildPlanSpec + Pantry.CabalSpec + Pantry.FileSpec + Pantry.GlobalHintsSpec + Pantry.HackageSpec + Pantry.Internal.StaticBytesSpec + Pantry.InternalSpec + Pantry.TreeSpec + Pantry.TypesSpec + Paths_pantry + hs-source-dirs: + test + default-extensions: MonadFailDesugaring + ghc-options: -Wall + build-depends: + Cabal + , QuickCheck + , aeson + , ansi-terminal + , array + , base >=4.10 && <5 + , base-orphans + , base64-bytestring + , bytestring + , conduit + , conduit-extra + , containers + , contravariant + , cryptonite + , cryptonite-conduit + , deepseq + , digest + , directory + , exceptions + , filelock + , filepath + , generic-deriving + , ghc-prim + , hackage-security + , hashable + , hedgehog + , hpack >=0.31.2 + , hspec + , http-client + , http-client-tls + , http-conduit + , http-download + , http-types + , integer-gmp + , memory + , mono-traversable + , mtl + , network + , network-uri + , pantry + , path + , path-io + , persistent + , persistent-sqlite >=2.9.3 + , persistent-template + , primitive + , raw-strings-qq + , resourcet + , rio + , rio-orphans + , rio-prettyprint + , safe + , syb + , tar-conduit + , template-haskell + , text + , text-metrics + , th-lift + , th-lift-instances + , th-orphans + , th-reify-many + , th-utilities + , time + , transformers + , unix-compat + , unliftio + , unordered-containers + , vector + , yaml + , zip-archive + default-language: Haskell2010 diff --git a/src/Pantry.hs b/src/Pantry.hs --- a/src/Pantry.hs +++ b/src/Pantry.hs @@ -1593,10 +1593,11 @@ mres <- withStorage $ getSnapshotCacheByHash hash cacheId <- case mres of Nothing -> do - scId <- withStorage $ getSnapshotCacheId hash - packageModules <- getModuleMapping logWarn "Populating snapshot module name cache" - withStorage $ storeSnapshotModuleCache scId packageModules - return scId + packageModules <- getModuleMapping + withStorage $ do + scId <- getSnapshotCacheId hash + storeSnapshotModuleCache scId packageModules + return scId Just scId -> pure scId f $ withStorage . loadExposedModulePackages cacheId diff --git a/src/Pantry/Internal/Stackage.hs b/src/Pantry/Internal/Stackage.hs --- a/src/Pantry/Internal/Stackage.hs +++ b/src/Pantry/Internal/Stackage.hs @@ -29,6 +29,8 @@ , getTreeForKey , getVersionId , loadBlobById + , allBlobsSource + , allBlobsCount , migrateAll , treeCabal , Key(unBlobKey) diff --git a/src/Pantry/Storage.hs b/src/Pantry/Storage.hs --- a/src/Pantry/Storage.hs +++ b/src/Pantry/Storage.hs @@ -1,3 +1,4 @@ +{-# LANGUAGE TypeApplications #-} {-# LANGUAGE NamedFieldPuns #-} {-# LANGUAGE NoImplicitPrelude #-} {-# LANGUAGE BangPatterns #-} @@ -21,6 +22,8 @@ , loadBlob , loadBlobById , loadBlobBySHA + , allBlobsSource + , allBlobsCount , getBlobKey , loadURLBlob , storeURLBlob @@ -96,7 +99,7 @@ import Database.Persist import Database.Persist.Sqlite import Database.Persist.TH -import RIO.Orphans () +import RIO.Orphans (HasResourceMap) import qualified Pantry.SHA256 as SHA256 import qualified RIO.Map as Map import qualified RIO.Text as T @@ -355,6 +358,18 @@ case mbt of Nothing -> error "loadBlobById: ID doesn't exist in database" Just bt -> pure $ blobContents bt + +allBlobsSource :: + HasResourceMap env + => Maybe BlobId + -- ^ For some x, yield blob whose id>x. + -> ConduitT () (BlobId, ByteString) (ReaderT SqlBackend (RIO env)) () +allBlobsSource mblobId = + selectSource [BlobId >. blobId | Just blobId <- [mblobId]] [Asc BlobId] .| + mapC ((entityKey &&& blobContents . entityVal)) + +allBlobsCount :: Maybe BlobId -> ReaderT SqlBackend (RIO env) Int +allBlobsCount mblobId = count [BlobId >. blobId | Just blobId <- [mblobId]] getBlobKey :: BlobId -> ReaderT SqlBackend (RIO env) BlobKey getBlobKey bid = do