fei-dataiter (empty) → 0.2.0.0
raw patch · 9 files changed
+1358/−0 lines, 9 filesdep +basedep +conduitdep +conduit-combinators
Dependencies added: base, conduit, conduit-combinators, directory, fei-base, fei-dataiter, fei-nn, filepath, haskell-src-exts, hslogger, hspec, mtl, optparse-applicative, streaming, template-haskell, text
Files
- LICENSE +29/−0
- README.md +78/−0
- cmdline/Main.hs +191/−0
- dataiter/src/MXNet/NN/DataIter/Conduit.hs +77/−0
- dataiter/src/MXNet/NN/DataIter/Raw.hs +741/−0
- dataiter/src/MXNet/NN/DataIter/Streaming.hs +78/−0
- dataiter/test/conduit.hs +38/−0
- dataiter/test/streaming.hs +39/−0
- fei-dataiter.cabal +87/−0
+ LICENSE view
@@ -0,0 +1,29 @@+BSD 3-Clause License++Copyright (c) 2018, Jiasen Wu+All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++* Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++* Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++* Neither the name of the copyright holder nor the names of its+ contributors may be used to endorse or promote products derived from+ this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ README.md view
@@ -0,0 +1,78 @@+# mxnet-dataiter++Here is an example of making a Conduit from MNIST dataset.++```haskell+mnistIter (add @"image" "data/train-images-idx3-ubyte" $ + add @"label" "data/train-labels-idx1-ubyte" $+ add @"batch_size" 128 + nil) :: ConduitData IO (NDArray Float, NDArray Float)+```++The first argument is provides named parameters for the MXNet Data Iterators. Detailed specification can be +found in MXNet API 's python [document](https://mxnet.incubator.apache.org/api/python/io/io.html).++Below is a snapshot of current support in this package.++```haskell+type CSVIter_Args = + '[ "data_csv" := String, "data_shape" := [Int], "label_csv" := String, "label_shape" := [Int]+ , "batch_size" := Int, "round_batch" := Bool, "prefetch_buffer" := Integer, "dtype" := String] ++type MNISTIter_Args = + '[ "image" := String, "label" := String, "batch_size" := Int, "shuffle" := Bool, "flat" := Bool+ , "seed" := Int, "silent" := Bool, "num_parts" := Int, "part_index" := Int+ , "prefetch_buffer" := Integer, "dtype" := String]++type ImageRecordIter_Args = + '[ "path_imglist" := String, "path_imgrec" := String, "path_imgidx" := String, "aug_seq" := String+ , "label_width" := Int, "data_shape" := [Int], "preprocess_threads" := Int, "verbose" := Bool+ , "num_parts" := Int, "part_index" := Int, "shuffle_chunk_size" := Integer+ , "shuffle_chunk_seed" := Int, "shuffle" := Bool, "seed" := Int, "batch_size" := Int+ , "round_batch" := Bool, "prefetch_buffer" := Integer, "dtype" := String, "resize" := Int+ , "rand_crop" := Bool, "max_rotate_angle" := Int, "max_aspect_ratio" := Float+ , "max_shear_ratio" := Float, "max_crop_size" := Int, "min_crop_size" := Int+ , "max_random_scale" := Float, "min_random_scale" := Float, "max_img_size" := Float+ , "min_img_size" := Float, "random_h" := Int, "random_s" := Int, "random_l" := Int, "rotate" := Int+ , "fill_value" := Int, "inter_method" := Int, "pad" := Int, "mirror" := Bool, "rand_mirror" := Bool+ , "mean_img" := String, "mean_r" := Float, "mean_g" := Float, "mean_b" := Float, "mean_a" := Float+ , "std_r" := Float, "std_g" := Float, "std_b" := Float, "std_a" := Float, "scale" := Float+ , "max_random_contrast" := Float, "max_random_illumination" := Float]+ +type ImageDetRecordIter_Args = + '[ "path_imglist" := String, "path_imgrec" := String, "aug_seq" := String, "label_width" := Int+ , "data_shape" := [Int], "preprocess_threads" := Int, "verbose" := Bool, "num_parts" := Int+ , "part_index" := Int, "shuffle_chunk_size" := Integer, "shuffle_chunk_seed" := Int+ , "label_pad_width" := Int, "label_pad_value" := Float, "shuffle" := Bool, "seed" := Int+ , "batch_size" := Int, "round_batch" := Bool, "prefetch_buffer" := Integer, "dtype" := String+ , "resize" := Int, "rand_crop_prob" := Float, "min_crop_scales" := [Float]+ , "max_crop_scales" := [Float], "min_crop_aspect_ratios" := [Float]+ , "max_crop_aspect_ratios" := [Float], "min_crop_overlaps" := [Float], "max_crop_overlaps" := [Float]+ , "min_crop_sample_coverages" := [Float], "max_crop_sample_coverages" := [Float]+ , "min_crop_object_coverages" := [Float], "max_crop_object_coverages" := [Float]+ , "num_crop_sampler" := Int, "crop_emit_mode" := String, "emit_overlap_thresh" := Float+ , "max_crop_trials" := [Int], "rand_pad_prob" := Float, "max_pad_scale" := Float+ , "max_random_hue" := Int, "random_hue_prob" := Float, "max_random_saturation" := Int+ , "random_saturation_prob" := Float, "max_random_illumination" := Int+ , "random_illumination_prob" := Float, "max_random_contrast" := Float, "random_contrast_prob" := Float+ , "rand_mirror_prob" := Float, "fill_value" := Int, "inter_method" := Int, "resize_mode" := String+ , "mean_img" := String, "mean_r" := Float, "mean_g" := Float, "mean_b" := Float, "mean_a" := Float+ , "std_r" := Float, "std_g" := Float, "std_b" := Float, "std_a" := Float, "scale" := Float]++type ImageRecordUInt8Iter_Args = + '[ "path_imglist" := String, "path_imgrec" := String, "path_imgidx" := String, "aug_seq" := String+ , "label_width" := Int, "data_shape" := [Int], "preprocess_threads" := Int, "verbose" := Bool+ , "num_parts" := Int, "part_index" := Int, "shuffle_chunk_size" := Integer+ , "shuffle_chunk_seed" := Int, "shuffle" := Bool, "seed" := Int, "batch_size" := Int+ , "round_batch" := Bool, "prefetch_buffer" := Integer, "dtype" := String, "resize" := Int+ , "rand_crop" := Bool, "max_rotate_angle" := Int, "max_aspect_ratio" := Float+ , "max_shear_ratio" := Float, "max_crop_size" := Int, "min_crop_size" := Int+ , "max_random_scale" := Float, "min_random_scale" := Float, "max_img_size" := Float+ , "min_img_size" := Float, "random_h" := Int, "random_s" := Int, "random_l" := Int, "rotate" := Int+ , "fill_value" := Int, "inter_method" := Int, "pad" := Int]++type LibSVMIter_Args = + '[ "data_libsvm" := String, "data_shape" := [Int], "label_libsvm" := String, "label_shape" := [Int]+ , "num_parts" := Int, "part_index" := Int, "batch_size" := Int, "round_batch" := Bool+ , "prefetch_buffer" := Integer, "dtype" := String]+```
+ cmdline/Main.hs view
@@ -0,0 +1,191 @@+module Main where++import Options.Applicative+import Data.Semigroup ((<>))+import Language.Haskell.Exts+import qualified Data.Text as T+import System.Log.Logger+import Control.Monad+import Control.Monad.Writer (Writer, execWriter, tell)+import Data.Either+import Data.Char (toLower, isUpper, isSpace, isAlphaNum)+import Text.Printf (printf)+import Text.ParserCombinators.ReadP+import System.FilePath+import System.Directory++import MXNet.Base.Raw++_module_ = "Main"++data Arguments = Arguments {+ output_dir :: FilePath+}++args_spec = Arguments + <$> strOption (long "output" <> short 'o' <> value "dataiter/src" <> metavar "OUTPUT-DIR")++main = do+ updateGlobalLogger _module_ (setLevel INFO)+ args <- execParser opts+ let base = output_dir args </> "MXNet" </> "NN" </> "DataIter"+ createDirectoryIfMissing True base++ dataitercreators <- mxListDataIters++ infoM _module_ "Generating DataIters..."+ dataiters <- concat <$> mapM genDataIter (zip dataitercreators [0..])+ writeFile (base </> "Raw.hs") $ prettyPrint (modDataIter dataiters)+ + where+ opts = info (args_spec <**> helper) (fullDesc <> progDesc "Generate MXNet dataiters")+ modDataIter = Module () (Just $ ModuleHead () (ModuleName () "MXNet.NN.DataIter.Raw") Nothing Nothing) [] + [ simpleImport "MXNet.Base.Raw"+ , simpleImport "MXNet.Base.Spec.Operator"+ , simpleImport "MXNet.Base.Spec.HMap"+ , simpleImportVars "Data.Maybe" ["catMaybes", "fromMaybe"]]++simpleImport mod = ImportDecl {+ importAnn = (),+ importModule = ModuleName () mod,+ importQualified = False,+ importSrc = False,+ importSafe = False,+ importPkg = Nothing,+ importAs = Nothing,+ importSpecs = Nothing+}++simpleImportVars mod vars = ImportDecl {+ importAnn = (),+ importModule = ModuleName () mod,+ importQualified = False,+ importSrc = False,+ importSafe = False,+ importPkg = Nothing,+ importAs = Nothing,+ importSpecs = Just $ ImportSpecList () False [IVar () $ Ident () var | var <- vars]+}++genDataIter :: (DataIterCreator, Integer) -> IO [Decl ()]+genDataIter (dataitercreator, index) = do+ (diname, didesc, argnames, argtypes, argdescs) <- mxDataIterGetIterInfo dataitercreator+ let diname_ = normalizeName diname+ (errs, scalarTypes) = execWriter $ zipWithM_ resolveHaskellType argnames argtypes++ -- parameter list+ paramList = map (\(name, typ1, typ2) -> tyPromotedTuple [tyPromotedStr name, tyApp typ1 typ2]) scalarTypes+ paramInst = TypeInsDecl () (tyApp (tyCon $ unQual $ name "ParameterList") (tyPromotedStr diname))+ (tyPromotedList paramList)++ -- signature+ cxfullfill = appA (name "Fullfilled") [tyPromotedStr diname, tyVarIdent "args"]+ tyfun = tyFun (tyApp (tyApp (tyCon $ unQual $ name "ArgsHMap") (tyPromotedStr diname)) (tyVarIdent "args")) + (tyApp (tyCon $ unQual $ name "IO") (tyCon $ unQual $ name "DataIterHandle"))+ tysig = tySig [name diname_] $ tyForall [unkindedVar (name "args")] (cxSingle cxfullfill) tyfun++ -- function+ fun = sfun (name diname_) [name "args"] (UnGuardedRhs () body) Nothing+ body = letE ([+ patBind (pvar $ name "allargs") (function "catMaybes" + `app` listE [+ infixApp (infixApp (tupleSection [Just $ strE argkey, Nothing]) (op $ sym ".") (function "showValue")) (op $ sym "<$>") $ + ExpTypeSig () (infixApp (var $ name "args") (op $ sym "!?") (OverloadedLabel () argkey)) (tyApp (tyCon $ unQual $ name "Maybe") typ) | (argkey, _, typ) <- scalarTypes])+ , patBind (pTuple [pvar $ name "keys", pvar $ name "vals"]) (app (function "unzip") $ var $ name "allargs")+ ]) (doE $ [+ genStmt (pvar $ name "dis") $ function "mxListDataIters",+ genStmt (pvar $ name "di") $ function "return" `app` (infixApp (var $ name "dis") (op $ sym "!!") (intE index)),+ qualStmt $ function "mxDataIterCreateIter" `app` (var $ name "di") `app` (var $ name "keys") `app` (var $ name "vals")+ ])++ return [paramInst, tysig, fun]+++ where+ normalizeName :: String -> String+ normalizeName name@(c:cs) + | isUpper c = '_' : name+ | otherwise = name++data ParamDesc = ParamDescItem String | ParamDescList Bool [String] deriving (Eq, Show)++type ResolvedType = (String, Type (), Type ())+resolveHaskellType :: String -> String -> Writer ([(String, String)], [ResolvedType]) ()+resolveHaskellType argname desc =+ case head fields of + ParamDescItem "Shape(tuple)" -> scalar $ tyList $ tyCon $ unQual $ name "Int"+ ParamDescItem "int" -> scalar $ tyCon $ unQual $ name "Int"+ ParamDescItem "int (non-negative)" -> scalar $ tyCon $ unQual $ name "Int"+ ParamDescItem "long (non-negative)" -> scalar $ tyCon $ unQual $ name "Int"+ ParamDescItem "boolean" -> scalar $ tyCon $ unQual $ name "Bool"+ ParamDescItem "float" -> scalar $ tyCon $ unQual $ name "Float"+ ParamDescItem "double" -> scalar $ tyCon $ unQual $ name "Double"+ ParamDescItem "float32" -> scalar $ tyCon $ unQual $ name "Float"+ ParamDescItem "string" -> scalar $ tyCon $ unQual $ name "String"+ ParamDescItem "int or None" -> scalar $ tyApp (tyCon $ unQual $ name "Maybe") (tyCon $ unQual $ name "Int")+ ParamDescItem "double or None" -> scalar $ tyApp (tyCon $ unQual $ name "Maybe") (tyCon $ unQual $ name "Double")+ ParamDescList hasnone vs -> do+ let vsprom = map tyPromotedStr vs+ typ1 = tyApp (tyCon $ unQual $ name "EnumType") (tyPromotedList vsprom)+ typ2 = tyApp (tyCon $ unQual $ name "Maybe") typ1+ scalar $ if hasnone then typ2 else typ1++ t -> fail $ printf "unsupported arg: %s" (show t)+ where+ typedesc = do+ ds <- sepBy (skipSpaces >> (list1 +++ list2 +++ item)) (char ',')+ eof+ return ds+ list1 = ParamDescList True <$> between (string "{None,") (char '}') (sepBy (skipSpaces >> listItem) (char ','))+ list2 = ParamDescList False <$> between (string "{") (char '}') (sepBy (skipSpaces >> listItem) (char ','))+ listItem = between (char '\'') (char '\'') (munch1 (\c -> isAlphaNum c || c `elem` "_"))+ item = ParamDescItem <$> munch1 (\c -> isAlphaNum c || c `elem` " _-()=[]<>'./+")+ runP str = case readP_to_S typedesc str of + [(xs, "")] -> xs+ other -> error ("cannot parse type description: " ++ str)++ fields = runP desc+ required = ParamDescItem "required" `elem` fields+ attr = tyCon $ unQual $ name $ if required then "AttrReq" else "AttrOpt"+ scalar hstyp = tell ([], [(argname, attr, hstyp)])+ fail msg = tell ([(argname, msg)], [])++makeParamInst :: String -> [ResolvedType] -> Bool -> Decl ()+makeParamInst symname typs symbolapi = + TypeInsDecl () (tyApp (tyCon $ unQual $ name "ParameterList") (tyPromotedStr symname_with_appendix))+ (tyPromotedList paramList)+ where+ symname_with_appendix = symname ++ (if symbolapi then "(symbol)" else "(ndarray)")+ paramList = map (\(name, typ1, typ2) -> tyPromotedTuple [tyPromotedStr name, tyApp typ1 typ2]) typs+++unQual = UnQual ()+unkindedVar = UnkindedVar ()++tyCon = TyCon ()+tyVarSymbol = TyVar () . Symbol ()+tyVarIdent = TyVar () . Ident ()+tyApp = TyApp ()+tyFun = TyFun ()+tySig names types = TypeSig () names types+tyList = TyList ()+tyVar = TyVar ()++tyPromotedInteger s = TyPromoted () (PromotedInteger () s (show s))+tyPromotedStr s = TyPromoted () (PromotedString () s s)+tyPromotedList s = TyPromoted () (PromotedList () True s)+tyPromotedTuple s = TyPromoted () (PromotedTuple () s)++tyForall vars cxt typ = TyForall () vars_ cxt_ typ+ where+ vars_ = if null vars then Nothing else Just vars+ cxt_ = if cxt == CxEmpty () then Nothing else Just cxt++cxSingle = CxSingle ()+cxTuple = CxTuple ()++appA = AppA ()++tupleSection = TupleSection () Boxed++con = Con ()
+ dataiter/src/MXNet/NN/DataIter/Conduit.hs view
@@ -0,0 +1,77 @@+{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} +module MXNet.NN.DataIter.Conduit ( + ConduitData(..), + Dataset(..), + imageRecordIter, mnistIter, csvIter, libSVMIter +) where + +import Data.Conduit +import qualified Data.Conduit.Combinators as C +import qualified Data.Conduit.List as CL +import Control.Applicative +import Control.Monad.IO.Class + +import MXNet.Base +import qualified MXNet.NN.DataIter.Raw as I +import MXNet.NN.DataIter.Class + +data ConduitData m a = ConduitData { + iter_batch_size :: Maybe Int, + getConduit :: ConduitM () a m () +} + +imageRecordIter :: (Fullfilled "ImageRecordIter" args, DType a, MonadIO m) + => ArgsHMap "ImageRecordIter" args -> ConduitData m (NDArray a, NDArray a) +imageRecordIter args = ConduitData { + getConduit = makeIter I._ImageRecordIter args, + iter_batch_size = Just (args ! #batch_size) +} + +mnistIter :: (Fullfilled "MNISTIter" args, DType a, MonadIO m) + => ArgsHMap "MNISTIter" args -> ConduitData m (NDArray a, NDArray a) +mnistIter args = ConduitData { + getConduit = makeIter I._MNISTIter args, + iter_batch_size = (args !? #batch_size) <|> Just 1 +} + +csvIter :: (Fullfilled "CSVIter" args, DType a, MonadIO m) + => ArgsHMap "CSVIter" args -> ConduitData m (NDArray a, NDArray a) +csvIter args = ConduitData { + getConduit = makeIter I._CSVIter args, + iter_batch_size = Just (args ! #batch_size) +} + +libSVMIter :: (Fullfilled "LibSVMIter" args, DType a, MonadIO m) + => ArgsHMap "LibSVMIter" args -> ConduitData m (NDArray a, NDArray a) +libSVMIter args = ConduitData { + getConduit = makeIter I._LibSVMIter args, + iter_batch_size = Just (args ! #batch_size) +} + +makeIter creator args = do + iter <- liftIO (creator args) + let loop = do valid <- liftIO $ mxDataIterNext iter + if valid == 0 + then liftIO (finalizeDataIterHandle iter) + else do + yieldM $ liftIO $ do + dat <- mxDataIterGetData iter + lbl <- mxDataIterGetLabel iter + return (NDArray dat, NDArray lbl) + loop + loop + +type instance DatasetConstraint (ConduitData m1) m2 = m1 ~ m2 + +instance Monad m => Dataset (ConduitData m) where + fromListD = ConduitData Nothing . CL.sourceList + zipD d1 d2 = ConduitData Nothing $ getZipSource $ (,) <$> ZipSource (getConduit d1) <*> ZipSource (getConduit d2) + sizeD d = runConduit (getConduit d .| C.length) + forEachD d proc = sourceToList $ getConduit d .| CL.mapM proc + foldD proc elem d = runConduit (getConduit d .| C.foldM proc elem) + takeD n d = d {getConduit = getConduit d .| C.take n} + +instance DatasetProp (ConduitData m) a where + batchSizeD = return . iter_batch_size
+ dataiter/src/MXNet/NN/DataIter/Raw.hs view
@@ -0,0 +1,741 @@+module MXNet.NN.DataIter.Raw where+import MXNet.Base.Raw+import MXNet.Base.Spec.Operator+import MXNet.Base.Spec.HMap+import Data.Maybe (catMaybes, fromMaybe)++type instance ParameterList "CSVIter" =+ '[ '("data_csv", AttrReq String), '("data_shape", AttrReq [Int]),+ '("label_csv", AttrOpt String), '("label_shape", AttrOpt [Int]),+ '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool),+ '("prefetch_buffer", AttrOpt Int),+ '("dtype",+ AttrOpt+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"])))]++_CSVIter ::+ forall args . Fullfilled "CSVIter" args =>+ ArgsHMap "CSVIter" args -> IO DataIterHandle+_CSVIter args+ = let allargs+ = catMaybes+ [("data_csv",) . showValue <$> (args !? #data_csv :: Maybe String),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("label_csv",) . showValue <$>+ (args !? #label_csv :: Maybe String),+ ("label_shape",) . showValue <$>+ (args !? #label_shape :: Maybe [Int]),+ ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int),+ ("round_batch",) . showValue <$>+ (args !? #round_batch :: Maybe Bool),+ ("prefetch_buffer",) . showValue <$>+ (args !? #prefetch_buffer :: Maybe Int),+ ("dtype",) . showValue <$>+ (args !? #dtype ::+ Maybe+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"])))]+ (keys, vals) = unzip allargs+ in+ do dis <- mxListDataIters+ di <- return (dis !! 0)+ mxDataIterCreateIter di keys vals++type instance ParameterList "ImageDetRecordIter" =+ '[ '("path_imglist", AttrOpt String),+ '("path_imgrec", AttrOpt String), '("aug_seq", AttrOpt String),+ '("label_width", AttrOpt Int), '("data_shape", AttrReq [Int]),+ '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int),+ '("shuffle_chunk_size", AttrOpt Int),+ '("shuffle_chunk_seed", AttrOpt Int),+ '("label_pad_width", AttrOpt Int),+ '("label_pad_value", AttrOpt Float), '("shuffle", AttrOpt Bool),+ '("seed", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool),+ '("prefetch_buffer", AttrOpt Int),+ '("dtype",+ AttrOpt+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ '("resize", AttrOpt Int), '("rand_crop_prob", AttrOpt Float),+ '("num_crop_sampler", AttrOpt Int),+ '("crop_emit_mode", AttrOpt (EnumType '["center", "overlap"])),+ '("emit_overlap_thresh", AttrOpt Float),+ '("max_crop_trials", AttrOpt [Int]),+ '("rand_pad_prob", AttrOpt Float),+ '("max_pad_scale", AttrOpt Float),+ '("max_random_hue", AttrOpt Int),+ '("random_hue_prob", AttrOpt Float),+ '("max_random_saturation", AttrOpt Int),+ '("random_saturation_prob", AttrOpt Float),+ '("max_random_illumination", AttrOpt Int),+ '("random_illumination_prob", AttrOpt Float),+ '("max_random_contrast", AttrOpt Float),+ '("random_contrast_prob", AttrOpt Float),+ '("rand_mirror_prob", AttrOpt Float), '("fill_value", AttrOpt Int),+ '("inter_method", AttrOpt Int), '("data_shape", AttrReq [Int]),+ '("resize_mode", AttrOpt (EnumType '["fit", "force", "shrink"])),+ '("seed", AttrOpt Int), '("mean_img", AttrOpt String),+ '("mean_r", AttrOpt Float), '("mean_g", AttrOpt Float),+ '("mean_b", AttrOpt Float), '("mean_a", AttrOpt Float),+ '("std_r", AttrOpt Float), '("std_g", AttrOpt Float),+ '("std_b", AttrOpt Float), '("std_a", AttrOpt Float),+ '("scale", AttrOpt Float), '("verbose", AttrOpt Bool)]++_ImageDetRecordIter ::+ forall args . Fullfilled "ImageDetRecordIter" args =>+ ArgsHMap "ImageDetRecordIter" args -> IO DataIterHandle+_ImageDetRecordIter args+ = let allargs+ = catMaybes+ [("path_imglist",) . showValue <$>+ (args !? #path_imglist :: Maybe String),+ ("path_imgrec",) . showValue <$>+ (args !? #path_imgrec :: Maybe String),+ ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String),+ ("label_width",) . showValue <$>+ (args !? #label_width :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("preprocess_threads",) . showValue <$>+ (args !? #preprocess_threads :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int),+ ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int),+ ("shuffle_chunk_size",) . showValue <$>+ (args !? #shuffle_chunk_size :: Maybe Int),+ ("shuffle_chunk_seed",) . showValue <$>+ (args !? #shuffle_chunk_seed :: Maybe Int),+ ("label_pad_width",) . showValue <$>+ (args !? #label_pad_width :: Maybe Int),+ ("label_pad_value",) . showValue <$>+ (args !? #label_pad_value :: Maybe Float),+ ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int),+ ("round_batch",) . showValue <$>+ (args !? #round_batch :: Maybe Bool),+ ("prefetch_buffer",) . showValue <$>+ (args !? #prefetch_buffer :: Maybe Int),+ ("dtype",) . showValue <$>+ (args !? #dtype ::+ Maybe+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ ("resize",) . showValue <$> (args !? #resize :: Maybe Int),+ ("rand_crop_prob",) . showValue <$>+ (args !? #rand_crop_prob :: Maybe Float),+ ("num_crop_sampler",) . showValue <$>+ (args !? #num_crop_sampler :: Maybe Int),+ ("crop_emit_mode",) . showValue <$>+ (args !? #crop_emit_mode ::+ Maybe (EnumType '["center", "overlap"])),+ ("emit_overlap_thresh",) . showValue <$>+ (args !? #emit_overlap_thresh :: Maybe Float),+ ("max_crop_trials",) . showValue <$>+ (args !? #max_crop_trials :: Maybe [Int]),+ ("rand_pad_prob",) . showValue <$>+ (args !? #rand_pad_prob :: Maybe Float),+ ("max_pad_scale",) . showValue <$>+ (args !? #max_pad_scale :: Maybe Float),+ ("max_random_hue",) . showValue <$>+ (args !? #max_random_hue :: Maybe Int),+ ("random_hue_prob",) . showValue <$>+ (args !? #random_hue_prob :: Maybe Float),+ ("max_random_saturation",) . showValue <$>+ (args !? #max_random_saturation :: Maybe Int),+ ("random_saturation_prob",) . showValue <$>+ (args !? #random_saturation_prob :: Maybe Float),+ ("max_random_illumination",) . showValue <$>+ (args !? #max_random_illumination :: Maybe Int),+ ("random_illumination_prob",) . showValue <$>+ (args !? #random_illumination_prob :: Maybe Float),+ ("max_random_contrast",) . showValue <$>+ (args !? #max_random_contrast :: Maybe Float),+ ("random_contrast_prob",) . showValue <$>+ (args !? #random_contrast_prob :: Maybe Float),+ ("rand_mirror_prob",) . showValue <$>+ (args !? #rand_mirror_prob :: Maybe Float),+ ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int),+ ("inter_method",) . showValue <$>+ (args !? #inter_method :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("resize_mode",) . showValue <$>+ (args !? #resize_mode ::+ Maybe (EnumType '["fit", "force", "shrink"])),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("mean_img",) . showValue <$> (args !? #mean_img :: Maybe String),+ ("mean_r",) . showValue <$> (args !? #mean_r :: Maybe Float),+ ("mean_g",) . showValue <$> (args !? #mean_g :: Maybe Float),+ ("mean_b",) . showValue <$> (args !? #mean_b :: Maybe Float),+ ("mean_a",) . showValue <$> (args !? #mean_a :: Maybe Float),+ ("std_r",) . showValue <$> (args !? #std_r :: Maybe Float),+ ("std_g",) . showValue <$> (args !? #std_g :: Maybe Float),+ ("std_b",) . showValue <$> (args !? #std_b :: Maybe Float),+ ("std_a",) . showValue <$> (args !? #std_a :: Maybe Float),+ ("scale",) . showValue <$> (args !? #scale :: Maybe Float),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool)]+ (keys, vals) = unzip allargs+ in+ do dis <- mxListDataIters+ di <- return (dis !! 1)+ mxDataIterCreateIter di keys vals++type instance ParameterList "ImageRecordIter_v1" =+ '[ '("path_imglist", AttrOpt String),+ '("path_imgrec", AttrOpt String), '("path_imgidx", AttrOpt String),+ '("aug_seq", AttrOpt String), '("label_width", AttrOpt Int),+ '("data_shape", AttrReq [Int]),+ '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int),+ '("shuffle_chunk_size", AttrOpt Int),+ '("shuffle_chunk_seed", AttrOpt Int), '("shuffle", AttrOpt Bool),+ '("seed", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool),+ '("prefetch_buffer", AttrOpt Int),+ '("dtype",+ AttrOpt+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ '("resize", AttrOpt Int), '("rand_crop", AttrOpt Bool),+ '("max_rotate_angle", AttrOpt Int),+ '("max_aspect_ratio", AttrOpt Float),+ '("max_shear_ratio", AttrOpt Float),+ '("max_crop_size", AttrOpt Int), '("min_crop_size", AttrOpt Int),+ '("max_random_scale", AttrOpt Float),+ '("min_random_scale", AttrOpt Float),+ '("max_img_size", AttrOpt Float), '("min_img_size", AttrOpt Float),+ '("random_h", AttrOpt Int), '("random_s", AttrOpt Int),+ '("random_l", AttrOpt Int), '("rotate", AttrOpt Int),+ '("fill_value", AttrOpt Int), '("data_shape", AttrReq [Int]),+ '("inter_method", AttrOpt Int), '("pad", AttrOpt Int),+ '("seed", AttrOpt Int), '("mirror", AttrOpt Bool),+ '("rand_mirror", AttrOpt Bool), '("mean_img", AttrOpt String),+ '("mean_r", AttrOpt Float), '("mean_g", AttrOpt Float),+ '("mean_b", AttrOpt Float), '("mean_a", AttrOpt Float),+ '("std_r", AttrOpt Float), '("std_g", AttrOpt Float),+ '("std_b", AttrOpt Float), '("std_a", AttrOpt Float),+ '("scale", AttrOpt Float), '("max_random_contrast", AttrOpt Float),+ '("max_random_illumination", AttrOpt Float),+ '("verbose", AttrOpt Bool)]++_ImageRecordIter_v1 ::+ forall args . Fullfilled "ImageRecordIter_v1" args =>+ ArgsHMap "ImageRecordIter_v1" args -> IO DataIterHandle+_ImageRecordIter_v1 args+ = let allargs+ = catMaybes+ [("path_imglist",) . showValue <$>+ (args !? #path_imglist :: Maybe String),+ ("path_imgrec",) . showValue <$>+ (args !? #path_imgrec :: Maybe String),+ ("path_imgidx",) . showValue <$>+ (args !? #path_imgidx :: Maybe String),+ ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String),+ ("label_width",) . showValue <$>+ (args !? #label_width :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("preprocess_threads",) . showValue <$>+ (args !? #preprocess_threads :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int),+ ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int),+ ("shuffle_chunk_size",) . showValue <$>+ (args !? #shuffle_chunk_size :: Maybe Int),+ ("shuffle_chunk_seed",) . showValue <$>+ (args !? #shuffle_chunk_seed :: Maybe Int),+ ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int),+ ("round_batch",) . showValue <$>+ (args !? #round_batch :: Maybe Bool),+ ("prefetch_buffer",) . showValue <$>+ (args !? #prefetch_buffer :: Maybe Int),+ ("dtype",) . showValue <$>+ (args !? #dtype ::+ Maybe+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ ("resize",) . showValue <$> (args !? #resize :: Maybe Int),+ ("rand_crop",) . showValue <$> (args !? #rand_crop :: Maybe Bool),+ ("max_rotate_angle",) . showValue <$>+ (args !? #max_rotate_angle :: Maybe Int),+ ("max_aspect_ratio",) . showValue <$>+ (args !? #max_aspect_ratio :: Maybe Float),+ ("max_shear_ratio",) . showValue <$>+ (args !? #max_shear_ratio :: Maybe Float),+ ("max_crop_size",) . showValue <$>+ (args !? #max_crop_size :: Maybe Int),+ ("min_crop_size",) . showValue <$>+ (args !? #min_crop_size :: Maybe Int),+ ("max_random_scale",) . showValue <$>+ (args !? #max_random_scale :: Maybe Float),+ ("min_random_scale",) . showValue <$>+ (args !? #min_random_scale :: Maybe Float),+ ("max_img_size",) . showValue <$>+ (args !? #max_img_size :: Maybe Float),+ ("min_img_size",) . showValue <$>+ (args !? #min_img_size :: Maybe Float),+ ("random_h",) . showValue <$> (args !? #random_h :: Maybe Int),+ ("random_s",) . showValue <$> (args !? #random_s :: Maybe Int),+ ("random_l",) . showValue <$> (args !? #random_l :: Maybe Int),+ ("rotate",) . showValue <$> (args !? #rotate :: Maybe Int),+ ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("inter_method",) . showValue <$>+ (args !? #inter_method :: Maybe Int),+ ("pad",) . showValue <$> (args !? #pad :: Maybe Int),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("mirror",) . showValue <$> (args !? #mirror :: Maybe Bool),+ ("rand_mirror",) . showValue <$>+ (args !? #rand_mirror :: Maybe Bool),+ ("mean_img",) . showValue <$> (args !? #mean_img :: Maybe String),+ ("mean_r",) . showValue <$> (args !? #mean_r :: Maybe Float),+ ("mean_g",) . showValue <$> (args !? #mean_g :: Maybe Float),+ ("mean_b",) . showValue <$> (args !? #mean_b :: Maybe Float),+ ("mean_a",) . showValue <$> (args !? #mean_a :: Maybe Float),+ ("std_r",) . showValue <$> (args !? #std_r :: Maybe Float),+ ("std_g",) . showValue <$> (args !? #std_g :: Maybe Float),+ ("std_b",) . showValue <$> (args !? #std_b :: Maybe Float),+ ("std_a",) . showValue <$> (args !? #std_a :: Maybe Float),+ ("scale",) . showValue <$> (args !? #scale :: Maybe Float),+ ("max_random_contrast",) . showValue <$>+ (args !? #max_random_contrast :: Maybe Float),+ ("max_random_illumination",) . showValue <$>+ (args !? #max_random_illumination :: Maybe Float),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool)]+ (keys, vals) = unzip allargs+ in+ do dis <- mxListDataIters+ di <- return (dis !! 2)+ mxDataIterCreateIter di keys vals++type instance ParameterList "ImageRecordUInt8Iter_v1" =+ '[ '("path_imglist", AttrOpt String),+ '("path_imgrec", AttrOpt String), '("path_imgidx", AttrOpt String),+ '("aug_seq", AttrOpt String), '("label_width", AttrOpt Int),+ '("data_shape", AttrReq [Int]),+ '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int),+ '("shuffle_chunk_size", AttrOpt Int),+ '("shuffle_chunk_seed", AttrOpt Int), '("shuffle", AttrOpt Bool),+ '("seed", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool),+ '("prefetch_buffer", AttrOpt Int),+ '("dtype",+ AttrOpt+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ '("resize", AttrOpt Int), '("rand_crop", AttrOpt Bool),+ '("max_rotate_angle", AttrOpt Int),+ '("max_aspect_ratio", AttrOpt Float),+ '("max_shear_ratio", AttrOpt Float),+ '("max_crop_size", AttrOpt Int), '("min_crop_size", AttrOpt Int),+ '("max_random_scale", AttrOpt Float),+ '("min_random_scale", AttrOpt Float),+ '("max_img_size", AttrOpt Float), '("min_img_size", AttrOpt Float),+ '("random_h", AttrOpt Int), '("random_s", AttrOpt Int),+ '("random_l", AttrOpt Int), '("rotate", AttrOpt Int),+ '("fill_value", AttrOpt Int), '("data_shape", AttrReq [Int]),+ '("inter_method", AttrOpt Int), '("pad", AttrOpt Int)]++_ImageRecordUInt8Iter_v1 ::+ forall args . Fullfilled "ImageRecordUInt8Iter_v1" args =>+ ArgsHMap "ImageRecordUInt8Iter_v1" args -> IO DataIterHandle+_ImageRecordUInt8Iter_v1 args+ = let allargs+ = catMaybes+ [("path_imglist",) . showValue <$>+ (args !? #path_imglist :: Maybe String),+ ("path_imgrec",) . showValue <$>+ (args !? #path_imgrec :: Maybe String),+ ("path_imgidx",) . showValue <$>+ (args !? #path_imgidx :: Maybe String),+ ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String),+ ("label_width",) . showValue <$>+ (args !? #label_width :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("preprocess_threads",) . showValue <$>+ (args !? #preprocess_threads :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int),+ ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int),+ ("shuffle_chunk_size",) . showValue <$>+ (args !? #shuffle_chunk_size :: Maybe Int),+ ("shuffle_chunk_seed",) . showValue <$>+ (args !? #shuffle_chunk_seed :: Maybe Int),+ ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int),+ ("round_batch",) . showValue <$>+ (args !? #round_batch :: Maybe Bool),+ ("prefetch_buffer",) . showValue <$>+ (args !? #prefetch_buffer :: Maybe Int),+ ("dtype",) . showValue <$>+ (args !? #dtype ::+ Maybe+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ ("resize",) . showValue <$> (args !? #resize :: Maybe Int),+ ("rand_crop",) . showValue <$> (args !? #rand_crop :: Maybe Bool),+ ("max_rotate_angle",) . showValue <$>+ (args !? #max_rotate_angle :: Maybe Int),+ ("max_aspect_ratio",) . showValue <$>+ (args !? #max_aspect_ratio :: Maybe Float),+ ("max_shear_ratio",) . showValue <$>+ (args !? #max_shear_ratio :: Maybe Float),+ ("max_crop_size",) . showValue <$>+ (args !? #max_crop_size :: Maybe Int),+ ("min_crop_size",) . showValue <$>+ (args !? #min_crop_size :: Maybe Int),+ ("max_random_scale",) . showValue <$>+ (args !? #max_random_scale :: Maybe Float),+ ("min_random_scale",) . showValue <$>+ (args !? #min_random_scale :: Maybe Float),+ ("max_img_size",) . showValue <$>+ (args !? #max_img_size :: Maybe Float),+ ("min_img_size",) . showValue <$>+ (args !? #min_img_size :: Maybe Float),+ ("random_h",) . showValue <$> (args !? #random_h :: Maybe Int),+ ("random_s",) . showValue <$> (args !? #random_s :: Maybe Int),+ ("random_l",) . showValue <$> (args !? #random_l :: Maybe Int),+ ("rotate",) . showValue <$> (args !? #rotate :: Maybe Int),+ ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("inter_method",) . showValue <$>+ (args !? #inter_method :: Maybe Int),+ ("pad",) . showValue <$> (args !? #pad :: Maybe Int)]+ (keys, vals) = unzip allargs+ in+ do dis <- mxListDataIters+ di <- return (dis !! 3)+ mxDataIterCreateIter di keys vals++type instance ParameterList "MNISTIter" =+ '[ '("image", AttrOpt String), '("label", AttrOpt String),+ '("batch_size", AttrOpt Int), '("shuffle", AttrOpt Bool),+ '("flat", AttrOpt Bool), '("seed", AttrOpt Int),+ '("silent", AttrOpt Bool), '("num_parts", AttrOpt Int),+ '("part_index", AttrOpt Int), '("prefetch_buffer", AttrOpt Int),+ '("dtype",+ AttrOpt+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"])))]++_MNISTIter ::+ forall args . Fullfilled "MNISTIter" args =>+ ArgsHMap "MNISTIter" args -> IO DataIterHandle+_MNISTIter args+ = let allargs+ = catMaybes+ [("image",) . showValue <$> (args !? #image :: Maybe String),+ ("label",) . showValue <$> (args !? #label :: Maybe String),+ ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int),+ ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool),+ ("flat",) . showValue <$> (args !? #flat :: Maybe Bool),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("silent",) . showValue <$> (args !? #silent :: Maybe Bool),+ ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int),+ ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int),+ ("prefetch_buffer",) . showValue <$>+ (args !? #prefetch_buffer :: Maybe Int),+ ("dtype",) . showValue <$>+ (args !? #dtype ::+ Maybe+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"])))]+ (keys, vals) = unzip allargs+ in+ do dis <- mxListDataIters+ di <- return (dis !! 4)+ mxDataIterCreateIter di keys vals++type instance ParameterList "ImageRecordIter" =+ '[ '("path_imglist", AttrOpt String),+ '("path_imgrec", AttrOpt String), '("path_imgidx", AttrOpt String),+ '("aug_seq", AttrOpt String), '("label_width", AttrOpt Int),+ '("data_shape", AttrReq [Int]),+ '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int),+ '("shuffle_chunk_size", AttrOpt Int),+ '("shuffle_chunk_seed", AttrOpt Int), '("shuffle", AttrOpt Bool),+ '("seed", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool),+ '("prefetch_buffer", AttrOpt Int),+ '("dtype",+ AttrOpt+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ '("resize", AttrOpt Int), '("rand_crop", AttrOpt Bool),+ '("max_rotate_angle", AttrOpt Int),+ '("max_aspect_ratio", AttrOpt Float),+ '("max_shear_ratio", AttrOpt Float),+ '("max_crop_size", AttrOpt Int), '("min_crop_size", AttrOpt Int),+ '("max_random_scale", AttrOpt Float),+ '("min_random_scale", AttrOpt Float),+ '("max_img_size", AttrOpt Float), '("min_img_size", AttrOpt Float),+ '("random_h", AttrOpt Int), '("random_s", AttrOpt Int),+ '("random_l", AttrOpt Int), '("rotate", AttrOpt Int),+ '("fill_value", AttrOpt Int), '("data_shape", AttrReq [Int]),+ '("inter_method", AttrOpt Int), '("pad", AttrOpt Int),+ '("seed", AttrOpt Int), '("mirror", AttrOpt Bool),+ '("rand_mirror", AttrOpt Bool), '("mean_img", AttrOpt String),+ '("mean_r", AttrOpt Float), '("mean_g", AttrOpt Float),+ '("mean_b", AttrOpt Float), '("mean_a", AttrOpt Float),+ '("std_r", AttrOpt Float), '("std_g", AttrOpt Float),+ '("std_b", AttrOpt Float), '("std_a", AttrOpt Float),+ '("scale", AttrOpt Float), '("max_random_contrast", AttrOpt Float),+ '("max_random_illumination", AttrOpt Float),+ '("verbose", AttrOpt Bool)]++_ImageRecordIter ::+ forall args . Fullfilled "ImageRecordIter" args =>+ ArgsHMap "ImageRecordIter" args -> IO DataIterHandle+_ImageRecordIter args+ = let allargs+ = catMaybes+ [("path_imglist",) . showValue <$>+ (args !? #path_imglist :: Maybe String),+ ("path_imgrec",) . showValue <$>+ (args !? #path_imgrec :: Maybe String),+ ("path_imgidx",) . showValue <$>+ (args !? #path_imgidx :: Maybe String),+ ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String),+ ("label_width",) . showValue <$>+ (args !? #label_width :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("preprocess_threads",) . showValue <$>+ (args !? #preprocess_threads :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int),+ ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int),+ ("shuffle_chunk_size",) . showValue <$>+ (args !? #shuffle_chunk_size :: Maybe Int),+ ("shuffle_chunk_seed",) . showValue <$>+ (args !? #shuffle_chunk_seed :: Maybe Int),+ ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int),+ ("round_batch",) . showValue <$>+ (args !? #round_batch :: Maybe Bool),+ ("prefetch_buffer",) . showValue <$>+ (args !? #prefetch_buffer :: Maybe Int),+ ("dtype",) . showValue <$>+ (args !? #dtype ::+ Maybe+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ ("resize",) . showValue <$> (args !? #resize :: Maybe Int),+ ("rand_crop",) . showValue <$> (args !? #rand_crop :: Maybe Bool),+ ("max_rotate_angle",) . showValue <$>+ (args !? #max_rotate_angle :: Maybe Int),+ ("max_aspect_ratio",) . showValue <$>+ (args !? #max_aspect_ratio :: Maybe Float),+ ("max_shear_ratio",) . showValue <$>+ (args !? #max_shear_ratio :: Maybe Float),+ ("max_crop_size",) . showValue <$>+ (args !? #max_crop_size :: Maybe Int),+ ("min_crop_size",) . showValue <$>+ (args !? #min_crop_size :: Maybe Int),+ ("max_random_scale",) . showValue <$>+ (args !? #max_random_scale :: Maybe Float),+ ("min_random_scale",) . showValue <$>+ (args !? #min_random_scale :: Maybe Float),+ ("max_img_size",) . showValue <$>+ (args !? #max_img_size :: Maybe Float),+ ("min_img_size",) . showValue <$>+ (args !? #min_img_size :: Maybe Float),+ ("random_h",) . showValue <$> (args !? #random_h :: Maybe Int),+ ("random_s",) . showValue <$> (args !? #random_s :: Maybe Int),+ ("random_l",) . showValue <$> (args !? #random_l :: Maybe Int),+ ("rotate",) . showValue <$> (args !? #rotate :: Maybe Int),+ ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("inter_method",) . showValue <$>+ (args !? #inter_method :: Maybe Int),+ ("pad",) . showValue <$> (args !? #pad :: Maybe Int),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("mirror",) . showValue <$> (args !? #mirror :: Maybe Bool),+ ("rand_mirror",) . showValue <$>+ (args !? #rand_mirror :: Maybe Bool),+ ("mean_img",) . showValue <$> (args !? #mean_img :: Maybe String),+ ("mean_r",) . showValue <$> (args !? #mean_r :: Maybe Float),+ ("mean_g",) . showValue <$> (args !? #mean_g :: Maybe Float),+ ("mean_b",) . showValue <$> (args !? #mean_b :: Maybe Float),+ ("mean_a",) . showValue <$> (args !? #mean_a :: Maybe Float),+ ("std_r",) . showValue <$> (args !? #std_r :: Maybe Float),+ ("std_g",) . showValue <$> (args !? #std_g :: Maybe Float),+ ("std_b",) . showValue <$> (args !? #std_b :: Maybe Float),+ ("std_a",) . showValue <$> (args !? #std_a :: Maybe Float),+ ("scale",) . showValue <$> (args !? #scale :: Maybe Float),+ ("max_random_contrast",) . showValue <$>+ (args !? #max_random_contrast :: Maybe Float),+ ("max_random_illumination",) . showValue <$>+ (args !? #max_random_illumination :: Maybe Float),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool)]+ (keys, vals) = unzip allargs+ in+ do dis <- mxListDataIters+ di <- return (dis !! 5)+ mxDataIterCreateIter di keys vals++type instance ParameterList "ImageRecordUInt8Iter" =+ '[ '("path_imglist", AttrOpt String),+ '("path_imgrec", AttrOpt String), '("path_imgidx", AttrOpt String),+ '("aug_seq", AttrOpt String), '("label_width", AttrOpt Int),+ '("data_shape", AttrReq [Int]),+ '("preprocess_threads", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int),+ '("shuffle_chunk_size", AttrOpt Int),+ '("shuffle_chunk_seed", AttrOpt Int), '("shuffle", AttrOpt Bool),+ '("seed", AttrOpt Int), '("verbose", AttrOpt Bool),+ '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool),+ '("prefetch_buffer", AttrOpt Int),+ '("dtype",+ AttrOpt+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ '("resize", AttrOpt Int), '("rand_crop", AttrOpt Bool),+ '("max_rotate_angle", AttrOpt Int),+ '("max_aspect_ratio", AttrOpt Float),+ '("max_shear_ratio", AttrOpt Float),+ '("max_crop_size", AttrOpt Int), '("min_crop_size", AttrOpt Int),+ '("max_random_scale", AttrOpt Float),+ '("min_random_scale", AttrOpt Float),+ '("max_img_size", AttrOpt Float), '("min_img_size", AttrOpt Float),+ '("random_h", AttrOpt Int), '("random_s", AttrOpt Int),+ '("random_l", AttrOpt Int), '("rotate", AttrOpt Int),+ '("fill_value", AttrOpt Int), '("data_shape", AttrReq [Int]),+ '("inter_method", AttrOpt Int), '("pad", AttrOpt Int)]++_ImageRecordUInt8Iter ::+ forall args . Fullfilled "ImageRecordUInt8Iter" args =>+ ArgsHMap "ImageRecordUInt8Iter" args -> IO DataIterHandle+_ImageRecordUInt8Iter args+ = let allargs+ = catMaybes+ [("path_imglist",) . showValue <$>+ (args !? #path_imglist :: Maybe String),+ ("path_imgrec",) . showValue <$>+ (args !? #path_imgrec :: Maybe String),+ ("path_imgidx",) . showValue <$>+ (args !? #path_imgidx :: Maybe String),+ ("aug_seq",) . showValue <$> (args !? #aug_seq :: Maybe String),+ ("label_width",) . showValue <$>+ (args !? #label_width :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("preprocess_threads",) . showValue <$>+ (args !? #preprocess_threads :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int),+ ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int),+ ("shuffle_chunk_size",) . showValue <$>+ (args !? #shuffle_chunk_size :: Maybe Int),+ ("shuffle_chunk_seed",) . showValue <$>+ (args !? #shuffle_chunk_seed :: Maybe Int),+ ("shuffle",) . showValue <$> (args !? #shuffle :: Maybe Bool),+ ("seed",) . showValue <$> (args !? #seed :: Maybe Int),+ ("verbose",) . showValue <$> (args !? #verbose :: Maybe Bool),+ ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int),+ ("round_batch",) . showValue <$>+ (args !? #round_batch :: Maybe Bool),+ ("prefetch_buffer",) . showValue <$>+ (args !? #prefetch_buffer :: Maybe Int),+ ("dtype",) . showValue <$>+ (args !? #dtype ::+ Maybe+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"]))),+ ("resize",) . showValue <$> (args !? #resize :: Maybe Int),+ ("rand_crop",) . showValue <$> (args !? #rand_crop :: Maybe Bool),+ ("max_rotate_angle",) . showValue <$>+ (args !? #max_rotate_angle :: Maybe Int),+ ("max_aspect_ratio",) . showValue <$>+ (args !? #max_aspect_ratio :: Maybe Float),+ ("max_shear_ratio",) . showValue <$>+ (args !? #max_shear_ratio :: Maybe Float),+ ("max_crop_size",) . showValue <$>+ (args !? #max_crop_size :: Maybe Int),+ ("min_crop_size",) . showValue <$>+ (args !? #min_crop_size :: Maybe Int),+ ("max_random_scale",) . showValue <$>+ (args !? #max_random_scale :: Maybe Float),+ ("min_random_scale",) . showValue <$>+ (args !? #min_random_scale :: Maybe Float),+ ("max_img_size",) . showValue <$>+ (args !? #max_img_size :: Maybe Float),+ ("min_img_size",) . showValue <$>+ (args !? #min_img_size :: Maybe Float),+ ("random_h",) . showValue <$> (args !? #random_h :: Maybe Int),+ ("random_s",) . showValue <$> (args !? #random_s :: Maybe Int),+ ("random_l",) . showValue <$> (args !? #random_l :: Maybe Int),+ ("rotate",) . showValue <$> (args !? #rotate :: Maybe Int),+ ("fill_value",) . showValue <$> (args !? #fill_value :: Maybe Int),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("inter_method",) . showValue <$>+ (args !? #inter_method :: Maybe Int),+ ("pad",) . showValue <$> (args !? #pad :: Maybe Int)]+ (keys, vals) = unzip allargs+ in+ do dis <- mxListDataIters+ di <- return (dis !! 6)+ mxDataIterCreateIter di keys vals++type instance ParameterList "LibSVMIter" =+ '[ '("data_libsvm", AttrReq String), '("data_shape", AttrReq [Int]),+ '("label_libsvm", AttrOpt String), '("label_shape", AttrOpt [Int]),+ '("num_parts", AttrOpt Int), '("part_index", AttrOpt Int),+ '("batch_size", AttrReq Int), '("round_batch", AttrOpt Bool),+ '("prefetch_buffer", AttrOpt Int),+ '("dtype",+ AttrOpt+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"])))]++_LibSVMIter ::+ forall args . Fullfilled "LibSVMIter" args =>+ ArgsHMap "LibSVMIter" args -> IO DataIterHandle+_LibSVMIter args+ = let allargs+ = catMaybes+ [("data_libsvm",) . showValue <$>+ (args !? #data_libsvm :: Maybe String),+ ("data_shape",) . showValue <$>+ (args !? #data_shape :: Maybe [Int]),+ ("label_libsvm",) . showValue <$>+ (args !? #label_libsvm :: Maybe String),+ ("label_shape",) . showValue <$>+ (args !? #label_shape :: Maybe [Int]),+ ("num_parts",) . showValue <$> (args !? #num_parts :: Maybe Int),+ ("part_index",) . showValue <$> (args !? #part_index :: Maybe Int),+ ("batch_size",) . showValue <$> (args !? #batch_size :: Maybe Int),+ ("round_batch",) . showValue <$>+ (args !? #round_batch :: Maybe Bool),+ ("prefetch_buffer",) . showValue <$>+ (args !? #prefetch_buffer :: Maybe Int),+ ("dtype",) . showValue <$>+ (args !? #dtype ::+ Maybe+ (Maybe+ (EnumType '["float16", "float32", "float64", "int32", "uint8"])))]+ (keys, vals) = unzip allargs+ in+ do dis <- mxListDataIters+ di <- return (dis !! 7)+ mxDataIterCreateIter di keys vals
+ dataiter/src/MXNet/NN/DataIter/Streaming.hs view
@@ -0,0 +1,78 @@+{-# LANGUAGE TypeFamilies #-} +{-# LANGUAGE FlexibleInstances #-} +{-# LANGUAGE MultiParamTypeClasses #-} +module MXNet.NN.DataIter.Streaming ( + StreamData(..), + Dataset(..), + imageRecordIter, mnistIter, csvIter, libSVMIter +) where + +import Streaming +import Streaming.Prelude (Of(..), yield, length_, toList_) +import qualified Streaming.Prelude as S + +import MXNet.Base +import qualified MXNet.NN.DataIter.Raw as I +import MXNet.NN.DataIter.Class + +data StreamData m a = StreamData { + iter_batch_size :: Maybe Int, + getStream :: Stream (Of a) m () +} + +imageRecordIter :: (Fullfilled "ImageRecordIter" args, DType a, MonadIO m) + => ArgsHMap "ImageRecordIter" args -> StreamData m (NDArray a, NDArray a) +imageRecordIter args = StreamData { + getStream = makeIter I._ImageRecordIter args, + iter_batch_size = Just (args ! #batch_size) +} + +mnistIter :: (Fullfilled "MNISTIter" args, DType a, MonadIO m) + => ArgsHMap "MNISTIter" args -> StreamData m (NDArray a, NDArray a) +mnistIter args = StreamData { + getStream = makeIter I._MNISTIter args, + iter_batch_size = (args !? #batch_size) <|> Just 1 +} + +csvIter :: (Fullfilled "CSVIter" args, DType a, MonadIO m) + => ArgsHMap "CSVIter" args -> StreamData m (NDArray a, NDArray a) +csvIter args = StreamData { + getStream = makeIter I._CSVIter args, + iter_batch_size = Just (args ! #batch_size) +} + +libSVMIter :: (Fullfilled "LibSVMIter" args, DType a, MonadIO m) + => ArgsHMap "LibSVMIter" args -> StreamData m (NDArray a, NDArray a) +libSVMIter args = StreamData { + getStream = makeIter I._LibSVMIter args, + iter_batch_size = Just (args ! #batch_size) +} + + +makeIter creator args = do + iter <- liftIO (creator args) + let loop = do valid <- liftIO $ mxDataIterNext iter + if valid == 0 + then liftIO (finalizeDataIterHandle iter) + else do + item <- liftIO $ do + dat <- mxDataIterGetData iter + lbl <- mxDataIterGetLabel iter + return (NDArray dat, NDArray lbl) + yield item + loop + loop + +type instance DatasetConstraint (StreamData m1) m2 = m1 ~ m2 + +instance Monad m => Dataset (StreamData m) where + fromListD = StreamData Nothing . S.each + zipD s1 s2 = StreamData Nothing $ S.zip (getStream s1) (getStream s2) + sizeD = length_ . getStream + forEachD dat proc = toList_ $ void $ S.mapM proc (getStream dat) + foldD proc elem dat = S.foldM_ proc (return elem) return (getStream dat) + takeD n dat = dat { getStream = S.take n (getStream dat) } + +instance DatasetProp (StreamData m) a where + batchSizeD = return . iter_batch_size +
+ dataiter/test/conduit.hs view
@@ -0,0 +1,38 @@+module Main where + +import Test.Hspec + +import MXNet.Base +import MXNet.NN.DataIter.Conduit + +type DS = ConduitData IO (NDArray Float, NDArray Float) + +main :: IO () +main = hspec $ do + describe "MNISTIter" $ do + it "batch-size = 1" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 1 .& Nil) + sizeD sr `shouldReturn` 60000 + it "batch-size = 32" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 32 .& Nil) + sizeD sr `shouldReturn` 1875 + it "batch-size = 128" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 128 .& Nil) + sizeD sr `shouldReturn` 468 + describe "ImageRecordIter" $ do + it "batch-size = 32" $ do + let sr = imageRecordIter (#path_imgrec := "dataiter/test/data/cifar10_val.rec" .& + #data_shape := [3,28,28] .& + #batch_size := 32 .& Nil) + sizeD sr `shouldReturn` 313 + it "batch-size = 128" $ do + let sr = imageRecordIter (#path_imgrec := "dataiter/test/data/cifar10_val.rec" .& + #data_shape := [3,28,28] .& + #batch_size := 128 .& Nil) + sizeD sr `shouldReturn` 79
+ dataiter/test/streaming.hs view
@@ -0,0 +1,39 @@+module Main where + +import Test.Hspec +import Streaming.Prelude +import MXNet.Core.Base + +import MXNet.Core.IO.DataIter.Streaming + +type DS = StreamData IO (NDArray Float, NDArray Float) + +main :: IO () +main = hspec $ do + describe "MNISTIter" $ do + it "batch-size = 1" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 1 .& Nil) :: DS + sizeD sr `shouldReturn` 60000 + it "batch-size = 32" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 32 .& Nil) :: DS + sizeD sr `shouldReturn` 1875 + it "batch-size = 128" $ do + let sr = mnistIter (#image := "dataiter/test/data/train-images-idx3-ubyte" .& + #label := "dataiter/test/data/train-labels-idx1-ubyte" .& + #batch_size := 128 .& Nil) :: DS + sizeD sr `shouldReturn` 468 + describe "ImageRecordIter" $ do + it "batch-size = 32" $ do + let sr = imageRecordIter (#path_imgrec := "dataiter/test/data/cifar10_val.rec" .& + #data_shape := [3,28,28] .& + #batch_size := 32 .& Nil) :: DS + sizeD sr `shouldReturn` 313 + it "batch-size = 128" $ do + let sr = imageRecordIter (#path_imgrec := "dataiter/test/data/cifar10_val.rec" .& + #data_shape := [3,28,28] .& + #batch_size := 128 .& Nil) :: DS + sizeD sr `shouldReturn` 79
+ fei-dataiter.cabal view
@@ -0,0 +1,87 @@+name: fei-dataiter+version: 0.2.0.0+synopsis: mxnet dataiters+description: Providing the mxnet dataiters as Stream or Conduit+homepage: https://github.com/pierric/fei-dataiter#readme+bug-reports: https://github.com/pierric/fei-dataiter/issues+author: Jiasen Wu+maintainer: jiasenwu@hotmail.com+copyright: 2018 Jiasen Wu+license: BSD3+license-file: LICENSE+category: Machine Learning, AI+build-type: Simple+cabal-version: >= 1.10++extra-source-files:+ README.md++source-repository head+ type: git+ location: https://github.com/pierric/fei-dataiter++library+ hs-source-dirs: dataiter/src+ build-depends: base >=4.7 && <5,+ template-haskell >= 2.10.0.0,+ streaming >= 0.1.4.5,+ conduit >= 1.2 && < 1.4,+ conduit-combinators >= 1.1.2 && < 1.4,+ fei-base,+ fei-nn+ exposed-modules: MXNet.NN.DataIter.Streaming,+ MXNet.NN.DataIter.Conduit+ other-modules: Paths_fei_dataiter+ MXNet.NN.DataIter.Raw+ default-extensions: FlexibleContexts,+ DataKinds,+ TypeOperators,+ ExplicitForAll,+ TupleSections,+ OverloadedLabels,+ TypeFamilies+ default-language: Haskell2010++executable mxnet-dataiter-gen+ hs-source-dirs: cmdline+ main-is: Main.hs+ build-depends: base >= 4.7 && < 5,+ optparse-applicative >= 0.14 && < 0.15,+ haskell-src-exts >= 1.20,+ hslogger >= 1.2,+ text >= 1.2.0 && < 1.3,+ mtl,+ filepath >= 1.4 && < 1.5,+ directory >= 1.3 && < 1.4,+ fei-base+ default-language: Haskell2010+ default-extensions: FlexibleContexts+ extra-libraries: mxnet++test-suite streaming+ type: exitcode-stdio-1.0+ main-is: streaming.hs+ hs-source-dirs: dataiter/test+ build-depends: base>=4.7 && <5,+ hspec==2.*,+ streaming >= 0.1.4.5,+ fei-base,+ fei-dataiter+ default-language: Haskell2010+ default-extensions: FlexibleContexts,+ DataKinds,+ TypeApplications++test-suite conduit+ type: exitcode-stdio-1.0+ main-is: conduit.hs+ hs-source-dirs: dataiter/test+ build-depends: base>=4.7 && <5,+ hspec==2.*,+ fei-base,+ fei-dataiter+ default-language: Haskell2010+ default-extensions: FlexibleContexts,+ DataKinds,+ TypeApplications+