gridfs (empty) → 0.1.0.1
raw patch · 4 files changed
+420/−0 lines, 4 filesdep +basedep +bsondep +bytestringsetup-changed
Dependencies added: base, bson, bytestring, conduit, conduit-extra, monad-control, mongoDB, mtl, pureMD5, resourcet, tagged, text, time, transformers
Files
- Database/MongoDB/GridFS.hs +179/−0
- LICENSE +202/−0
- Setup.hs +2/−0
- gridfs.cabal +37/−0
+ Database/MongoDB/GridFS.hs view
@@ -0,0 +1,179 @@+-- | MongoDB GridFS implementation+{-# LANGUAGE OverloadedStrings, RecordWildCards, NamedFieldPuns, TupleSections, FlexibleContexts, FlexibleInstances, UndecidableInstances, MultiParamTypeClasses, GeneralizedNewtypeDeriving, StandaloneDeriving, TypeSynonymInstances, TypeFamilies, CPP, RankNTypes #-}++module Database.MongoDB.GridFS + -- * GridFS+ -- ** Data types+ ( Bucket+ , files, chunks+ , File+ , document, bucket+ -- ** Setup+ , openDefaultBucket+ , openBucket+ -- ** Query+ , findFile+ , findOneFile+ , fetchFile+ -- ** Delete+ , deleteFile+ -- ** Conduits+ , sourceFile+ , sinkFile+ )+ where++import Control.Applicative((<$>))+import Control.Concurrent(forkIO)+import Control.Monad(when)+import Control.Monad.IO.Class+import Control.Monad.Trans(MonadTrans, lift)+import Control.Monad.Trans.Control(MonadBaseControl)+import Control.Monad.Trans.Resource(MonadResource(..))+import Data.Conduit+import Data.Digest.Pure.MD5+import Data.Int+import Data.Tagged(Tagged, untag)+import Data.Text(Text, append)+import Data.Time.Clock(getCurrentTime)+import Database.MongoDB+import Prelude+import qualified Data.Bson as B+import qualified Data.ByteString as S+import qualified Data.ByteString.Lazy as L+import qualified Data.Conduit.Binary as BI+import qualified Data.Conduit.List as CL++defaultChunkSize :: Int64+-- ^ The default chunk size is 256 kB+defaultChunkSize = 256 * 1024++data Bucket = Bucket {files :: Text, chunks :: Text}+-- ^ Files are stored in "buckets". You open a bucket with openDefaultBucket or openBucket++openDefaultBucket :: (Monad m, MonadIO m) => Action m Bucket+-- ^ Open the default 'Bucket' (named "fs")+openDefaultBucket = openBucket "fs"++openBucket :: (Monad m, MonadIO m) => Text -> Action m Bucket+-- ^ Open a 'Bucket'+openBucket name = do+ let filesCollection = name `append` ".files"+ let chunksCollection = name `append` ".chunks"+ ensureIndex $ (index filesCollection ["filename" =: (1::Int), "uploadDate" =: (1::Int)])+ ensureIndex $ (index chunksCollection ["files_id" =: (1::Int), "n" =: (1::Int)]) { iUnique = True, iDropDups = True }+ return $ Bucket filesCollection chunksCollection++data File = File {bucket :: Bucket, document :: Document}++getChunk :: (Monad m, MonadIO m) => File -> Int -> Action m (Maybe S.ByteString)+-- ^ Get a chunk of a file+getChunk (File bucket doc) i = do+ files_id <- B.look "_id" doc+ result <- findOne $ select ["files_id" := files_id, "n" =: i] $ chunks bucket+ let content = at "data" <$> result+ case content of+ Just (Binary b) -> return (Just b)+ _ -> return Nothing++findFile :: (MonadIO m, MonadBaseControl IO m) => Bucket -> Selector -> Action m [File]+-- ^ Find files in the bucket+findFile bucket sel = do+ cursor <- find $ select sel $ files bucket+ results <- rest cursor+ return $ File bucket <$> results++findOneFile :: MonadIO m => Bucket -> Selector -> Action m (Maybe File)+-- ^ Find one file in the bucket+findOneFile bucket sel = do+ mdoc <- findOne $ select sel $ files bucket+ return $ File bucket <$> mdoc++fetchFile :: MonadIO m => Bucket -> Selector -> Action m File+-- ^ Fetch one file in the bucket+fetchFile bucket sel = do+ doc <- fetch $ select sel $ files bucket+ return $ File bucket doc++deleteFile :: (MonadIO m) => File -> Action m ()+-- ^ Delete files in the bucket+deleteFile (File bucket doc) = do+ files_id <- B.look "_id" doc+ delete $ select ["_id" := files_id] $ files bucket+ delete $ select ["files_id" := files_id] $ chunks bucket++putChunk :: (Monad m, MonadIO m) => Bucket -> ObjectId -> Int -> L.ByteString -> Action m ()+-- ^ Put a chunk in the bucket+putChunk bucket files_id i chunk = do+ insert_ (chunks bucket) ["files_id" =: files_id, "n" =: i, "data" =: Binary (L.toStrict chunk)]++sourceFile :: (Monad m, MonadIO m) => File -> Producer (Action m) S.ByteString+-- ^ A producer for the contents of a file+sourceFile file = yieldChunk 0 where+ yieldChunk i = do+ mbytes <- lift $ getChunk file i+ case mbytes of+ Just bytes -> yield bytes >> yieldChunk (i+1)+ Nothing -> return ()++-- Used to keep data during writing+data FileWriter = FileWriter+ { fwChunkSize :: Int64+ , fwBucket :: Bucket+ , fwFilesId :: ObjectId+ , fwChunkIndex :: Int+ , fwSize :: Int64+ , fwAcc :: L.ByteString+ , fwMd5Context :: MD5Context+ , fwMd5acc :: L.ByteString+ } ++-- Finalize file, calculating md5 digest, saving the last chunk, and creating the file in the bucket+finalizeFile :: (Monad m, MonadIO m) => Text -> FileWriter -> Action m File+finalizeFile filename (FileWriter chunkSize bucket files_id i size acc md5context md5acc) = do+ let md5digest = md5Finalize md5context (L.toStrict md5acc)+ when (L.length acc > 0) $ putChunk bucket files_id i acc+ timestamp <- liftIO $ getCurrentTime+ let doc = [ "_id" =: files_id+ , "length" =: size+ , "uploadDate" =: timestamp+ , "md5" =: show (md5digest)+ , "chunkSize" =: chunkSize+ , "filename" =: filename+ ]+ insert_ (files bucket) doc+ return $ File bucket doc++-- Write as many chunks as can be written from the file writer+writeChunks :: (Monad m, MonadIO m) => FileWriter -> L.ByteString -> Action m FileWriter+writeChunks (FileWriter chunkSize bucket files_id i size acc md5context md5acc) chunk = do+ -- Update md5 context+ let md5BlockLength = fromIntegral $ untag (blockLength :: Tagged MD5Digest Int)+ let md5acc_temp = (md5acc `L.append` chunk)+ let (md5context', md5acc') = + if (L.length md5acc_temp < md5BlockLength)+ then (md5context, md5acc_temp)+ else let numBlocks = L.length md5acc_temp `div` md5BlockLength+ (current, rest) = L.splitAt (md5BlockLength * numBlocks) md5acc_temp+ in (md5Update md5context (L.toStrict current), rest)+ -- Update chunks+ let size' = (size + L.length chunk)+ let acc_temp = (acc `L.append` chunk)+ if (L.length acc_temp < chunkSize)+ then return (FileWriter chunkSize bucket files_id i size' acc_temp md5context' md5acc')+ else do+ let (chunk, acc') = L.splitAt chunkSize acc_temp+ putChunk bucket files_id i chunk+ writeChunks (FileWriter chunkSize bucket files_id (i+1) size' acc' md5context' md5acc') L.empty++sinkFile :: (Monad m, MonadIO m) => Bucket -> Text -> Consumer S.ByteString (Action m) File+-- ^ A consumer that creates a file in the bucket and puts all consumed data in it+sinkFile bucket filename = do+ files_id <- liftIO $ genObjectId+ awaitChunk $ FileWriter defaultChunkSize bucket files_id 0 0 L.empty md5InitialContext L.empty+ where+ awaitChunk fw = do+ mchunk <- await+ case mchunk of+ Nothing -> lift (finalizeFile filename fw)+ Just chunk -> lift (writeChunks fw (L.fromStrict chunk)) >>= awaitChunk
+ LICENSE view
@@ -0,0 +1,202 @@+ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+
+ Setup.hs view
@@ -0,0 +1,2 @@+import Distribution.Simple+main = defaultMain
+ gridfs.cabal view
@@ -0,0 +1,37 @@+name: gridfs +version: 0.1.0.1+synopsis: GridFS (MongoDB file storage) implementation +description: Please see README.md+homepage: http://github.com/btubbs/haskell-gridfs#readme+license: OtherLicense+license-file: LICENSE+author: Martin Norbäck Olivers+maintainer: brent.tubbs@gmail.com +copyright: 2016 Martin Norbäck Olivers+category: Database +build-type: Simple+stability: alpha+-- extra-source-files:+cabal-version: >=1.10++library+ exposed-modules: Database.MongoDB.GridFS + default-language: Haskell2010+ build-depends: base >= 4.7 && < 5+ , bson+ , bytestring+ , conduit+ , conduit-extra+ , monad-control+ , mongoDB+ , mtl+ , pureMD5+ , resourcet+ , tagged+ , text+ , time+ , transformers++source-repository head+ type: git+ location: https://github.com/btubbs/haskell-gridfs